diff --git a/.clang-format b/.clang-format index 70a4c48..04b49ca 100644 --- a/.clang-format +++ b/.clang-format @@ -1,7 +1,11 @@ BasedOnStyle: Microsoft +Standard: Latest ColumnLimit: 160 IndentWidth: 4 +TabWidth: 4 UseTab: Always BreakBeforeBraces: Allman AllowShortFunctionsOnASingleLine: Empty -SortIncludes: CaseSensitive +SortIncludes: + Enabled: true + IgnoreCase: false diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8a30b08 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +** +!containers/ +!containers/** diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86af06b..16658a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,167 +3,122 @@ name: SimdLib CI on: push: pull_request: + workflow_dispatch: permissions: contents: read jobs: - windows: - name: MSVC ${{ matrix.arch }} ${{ matrix.config }} + native-msvc: + name: MSVC x64 validation runs-on: windows-2022 - strategy: - fail-fast: false - matrix: - arch: [x64, Win32] - config: [Debug, Release] steps: - uses: actions/checkout@v4 - - name: Configure - shell: pwsh - run: | - cmake -S . -B build -G 'Visual Studio 17 2022' -A '${{ matrix.arch }}' -T v143 ` - -DSIMDLIB_BUILD_TESTS=ON ` - -DSIMDLIB_BUILD_TESTS_OPTIONAL=OFF ` - -DSIMDLIB_BUILD_EXAMPLES=ON ` - -DSIMDLIB_STRICT_WARNINGS=ON - - name: Build - run: cmake --build build --config ${{ matrix.config }} --parallel - - name: Test - run: ctest --test-dir build -C ${{ matrix.config }} --output-on-failure + - name: Build every MSVC validation cell + run: tools/Build.ps1 -Scope Native -Compiler Msvc + - name: Test the exact MSVC build receipt + run: tools/Run-Tests.ps1 -Scope Native -Compiler Msvc + - name: Build MSVC benchmark artifacts explicitly + run: tools/Build-Benchmarks.ps1 -Scope Native -Compiler Msvc + - name: Upload MSVC evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: msvc-evidence + path: | + out/pipeline/windows-msvc/**/provenance + out/pipeline/windows-msvc/**/reports + out/pipeline/windows-msvc/**/validation-build.manifest + out/pipeline/windows-msvc/**/benchmark-build.manifest + out/pipeline/windows-msvc/**/build/register-codegen/**/*.json + out/pipeline/windows-msvc/**/build/register-codegen/**/*.txt + out/pipeline/windows-msvc/**/build/method-flags-codegen/**/*.json + out/pipeline/windows-msvc/**/build/method-flags-codegen/**/*.txt + out/pipeline/logs + out/pipeline/provenance + if-no-files-found: error - clang-cl: - name: clang-cl ${{ matrix.arch }} ${{ matrix.config }} + native-clangcl: + name: clang-cl and Clang coverage x64 validation runs-on: windows-2022 - strategy: - fail-fast: false - matrix: - arch: [x64, x86] - config: [Debug, Release] steps: - uses: actions/checkout@v4 - - uses: ilammy/msvc-dev-cmd@v1 + - name: Build every Clang validation cell + run: tools/Build.ps1 -Scope Native -Compiler ClangCl,ClangCoverage + - name: Test the exact Clang build receipt + run: tools/Run-Tests.ps1 -Scope Native -Compiler ClangCl,ClangCoverage + - name: Build clang-cl benchmark artifacts explicitly + run: tools/Build-Benchmarks.ps1 -Scope Native -Compiler ClangCl + - name: Upload Clang evidence + if: always() + uses: actions/upload-artifact@v4 with: - arch: ${{ matrix.arch }} - - name: Configure standalone clang-cl - run: >- - cmake -S . -B build -G Ninja - -DCMAKE_BUILD_TYPE=${{ matrix.config }} - -DCMAKE_CXX_COMPILER=clang-cl - -DSIMDLIB_BUILD_TESTS=ON - -DSIMDLIB_BUILD_TESTS_OPTIONAL=OFF - -DSIMDLIB_BUILD_EXAMPLES=ON - -DSIMDLIB_STRICT_WARNINGS=ON - - name: Build - run: cmake --build build --parallel - - name: Test - run: ctest --test-dir build --output-on-failure + name: clang-evidence + path: | + out/pipeline/windows-clangcl/**/provenance + out/pipeline/windows-clangcl/**/reports + out/pipeline/windows-clangcl/**/validation-build.manifest + out/pipeline/windows-clangcl/**/benchmark-build.manifest + out/pipeline/windows-clangcl/**/build/register-codegen/**/*.json + out/pipeline/windows-clangcl/**/build/register-codegen/**/*.txt + out/pipeline/windows-clangcl/**/build/method-flags-codegen/**/*.json + out/pipeline/windows-clangcl/**/build/method-flags-codegen/**/*.txt + out/pipeline/windows-clang-coverage/**/provenance + out/pipeline/windows-clang-coverage/**/reports + out/pipeline/windows-clang-coverage/**/validation-build.manifest + out/pipeline/windows-clang-coverage/**/build/method-flags-codegen/**/*.json + out/pipeline/windows-clang-coverage/**/build/method-flags-codegen/**/*.txt + out/pipeline/logs + out/pipeline/provenance + if-no-files-found: error - linux: - name: ${{ matrix.compiler }} ${{ matrix.arch }} ${{ matrix.config }} + container-compilers: + name: GCC 13, GCC 14, and Clang 22 containers runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - compiler: [gcc, clang] - arch: [x64, x86] - config: [Debug, Release] steps: - uses: actions/checkout@v4 - - name: Install x86 multilib support - if: matrix.arch == 'x86' - run: sudo apt-get update && sudo apt-get install -y g++-multilib - - name: Select compiler and architecture + - name: Install required Docker Compose shell: bash - run: | - if [[ '${{ matrix.compiler }}' == 'clang' ]]; then - echo 'CXX=clang++' >> "$GITHUB_ENV" - else - echo 'CXX=g++' >> "$GITHUB_ENV" - fi - if [[ '${{ matrix.arch }}' == 'x86' ]]; then - echo 'ARCH_FLAGS=-m32' >> "$GITHUB_ENV" - else - echo 'ARCH_FLAGS=' >> "$GITHUB_ENV" - fi - - name: Configure - run: >- - cmake -S . -B build -G Ninja - -DCMAKE_BUILD_TYPE=${{ matrix.config }} - -DCMAKE_CXX_FLAGS="${ARCH_FLAGS}" - -DCMAKE_EXE_LINKER_FLAGS="${ARCH_FLAGS}" - -DSIMDLIB_BUILD_TESTS=ON - -DSIMDLIB_BUILD_TESTS_OPTIONAL=OFF - -DSIMDLIB_BUILD_EXAMPLES=ON - -DSIMDLIB_STRICT_WARNINGS=ON - - name: Build - run: cmake --build build --parallel - - name: Test - run: ctest --test-dir build --output-on-failure - - feature-matrix: - name: AVX2 FMA BMI1 BMI2 enabled and disabled - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: Configure strict Release matrix - run: >- - cmake -S . -B build -G Ninja - -DCMAKE_BUILD_TYPE=Release - -DSIMDLIB_BUILD_TESTS=ON - -DSIMDLIB_BUILD_TESTS_OPTIONAL=ON - -DSIMDLIB_BUILD_EXAMPLES=ON - -DSIMDLIB_STRICT_WARNINGS=ON - - name: Build - run: cmake --build build --parallel - - name: Run feature profiles - run: ctest --test-dir build --output-on-failure -L 'AVX2|FMA|BMI|SCALAR' - - sanitizer: - name: Clang ASan and UBSan - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: Configure env: - CXX: clang++ - run: >- - cmake -S . -B build -G Ninja - -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_FLAGS='-fsanitize=address,undefined -fno-omit-frame-pointer' - -DCMAKE_EXE_LINKER_FLAGS='-fsanitize=address,undefined' - -DSIMDLIB_BUILD_TESTS=ON - -DSIMDLIB_BUILD_TESTS_OPTIONAL=OFF - -DSIMDLIB_BUILD_EXAMPLES=ON - -DSIMDLIB_STRICT_WARNINGS=ON - - name: Build - run: cmake --build build --parallel - - name: Test - run: ctest --test-dir build --output-on-failure - - contract-gates: - name: constexpr, header hygiene, ODR, and consumer - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: Configure header and constexpr probes - run: >- - cmake -S . -B build/contracts -G Ninja - -DCMAKE_BUILD_TYPE=Release - -DSIMDLIB_BUILD_TESTS=OFF - -DSIMDLIB_BUILD_CONFIGURATION_TESTS=ON - -DSIMDLIB_BUILD_HEADER_TESTS=ON - -DSIMDLIB_BUILD_SMOKE_TESTS=ON - -DSIMDLIB_STRICT_WARNINGS=ON - - name: Build compile-time contracts - run: cmake --build build/contracts --parallel - - name: Run multi-translation-unit ODR smoke test - run: ctest --test-dir build/contracts --output-on-failure -R SimdLib.HeaderOnlySmoke - - name: Configure add_subdirectory consumer - run: >- - cmake -S tests/consumer -B build/consumer -G Ninja - -DCMAKE_BUILD_TYPE=Release - -DSIMDLIB_SOURCE_DIR=${{ github.workspace }} - - name: Build and test header-only consumer + DOCKER_COMPOSE_VERSION: v2.39.0 run: | - cmake --build build/consumer --parallel - ctest --test-dir build/consumer --output-on-failure + plugin_dir="${HOME}/.docker/cli-plugins" + asset_name="docker-compose-linux-x86_64" + release_url="https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}" + mkdir -p "${plugin_dir}" + curl --fail --location --silent --show-error \ + "${release_url}/${asset_name}" \ + --output "${plugin_dir}/docker-compose" + expected_sha256="$(curl --fail --location --silent --show-error \ + "${release_url}/${asset_name}.sha256" | awk '{print $1}')" + actual_sha256="$(sha256sum "${plugin_dir}/docker-compose" | awk '{print $1}')" + test -n "${expected_sha256}" + test "${actual_sha256}" = "${expected_sha256}" + chmod +x "${plugin_dir}/docker-compose" + docker compose version + - name: Build every Linux validation cell + shell: pwsh + run: tools/Build.ps1 -Scope Containers + - name: Test the exact Linux build receipt + shell: pwsh + run: tools/Run-Tests.ps1 -Scope Containers + - name: Build Linux benchmark artifacts explicitly + shell: pwsh + run: tools/Build-Benchmarks.ps1 -Scope Containers + - name: Upload container evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: linux-container-evidence + path: | + out/pipeline/linux-*/**/provenance + out/pipeline/linux-*/**/reports + out/pipeline/linux-*/**/validation-build.manifest + out/pipeline/linux-*/**/benchmark-build.manifest + out/pipeline/linux-*/**/build/register-codegen/**/*.json + out/pipeline/linux-*/**/build/register-codegen/**/*.txt + out/pipeline/linux-*/**/build/method-flags-codegen/**/*.json + out/pipeline/linux-*/**/build/method-flags-codegen/**/*.txt + out/pipeline/logs + out/pipeline/provenance + if-no-files-found: error diff --git a/.github/workflows/container-reproducibility.yml b/.github/workflows/container-reproducibility.yml new file mode 100644 index 0000000..1ddd3ea --- /dev/null +++ b/.github/workflows/container-reproducibility.yml @@ -0,0 +1,29 @@ +name: Container reproducibility + +on: + workflow_dispatch: + schedule: + - cron: '17 9 * * 1' + +permissions: + contents: read + +jobs: + container-reproducibility: + name: Rebuild pinned images without cache + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Rebuild pinned environments without compiling the project + shell: pwsh + run: tools/Run-ContainerMatrix.ps1 -Action InspectEnvironment -NoImageCache + - name: Record image identities and sizes + shell: pwsh + run: docker image inspect simdlib/gcc13:local simdlib/gcc14:local simdlib/clang22:local | Out-File -Encoding utf8 out/pipeline/image-inspect.json + - name: Upload reproducibility evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: container-reproducibility-evidence + path: out/pipeline + if-no-files-found: error diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 983a0fb..ea05ecb 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,5 +1,6 @@ { "recommendations": [ - "ms-vscode.cmake-tools" + "ms-vscode.cmake-tools", + "ms-vscode.cpptools" ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 057df9d..ea19a71 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,9 +9,17 @@ "cmake.ctest.allowParallelJobs": true, "cmake.ctest.testSuiteDelimiter": "\\.", "cmake.ctest.testSuiteDelimiterMaxOccurrence": 0, - "cmake.preRunCoverageTarget": "SimdLibCoverageReset", - "cmake.postRunCoverageTarget": "SimdLibCoverageReport", - "cmake.coverageInfoFiles": [ - "${workspaceFolder}/build-coverage/coverage.info" - ] + "C_Cpp.formatting": "clangFormat", + "C_Cpp.clang_format_style": "file", + "C_Cpp.clang_format_fallbackStyle": "none", + "[c]": { + "editor.defaultFormatter": "ms-vscode.cpptools", + "editor.formatOnSave": true, + "editor.formatOnSaveMode": "modificationsIfAvailable" + }, + "[cpp]": { + "editor.defaultFormatter": "ms-vscode.cpptools", + "editor.formatOnSave": true, + "editor.formatOnSaveMode": "modificationsIfAvailable" + } } diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..159e599 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,121 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Build", + "type": "process", + "command": "pwsh", + "args": [ + "-NoProfile", + "-File", + "${workspaceFolder}/tools/Build.ps1", + "-Scope", + "All" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": "$msCompile", + "presentation": { + "clear": true, + "reveal": "always", + "panel": "dedicated" + }, + "group": { + "kind": "build", + "isDefault": true + }, + "detail": "Builds the complete native and container validation matrix without benchmark artifacts." + }, + { + "label": "Run-Tests", + "type": "process", + "command": "pwsh", + "args": [ + "-NoProfile", + "-File", + "${workspaceFolder}/tools/Run-Tests.ps1", + "-Scope", + "All" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [], + "presentation": { + "clear": true, + "reveal": "always", + "panel": "dedicated" + }, + "group": "test", + "detail": "Builds the complete matrix once, then runs every assigned test-only cell." + }, + { + "label": "Build-Benchmarks", + "type": "process", + "command": "pwsh", + "args": [ + "-NoProfile", + "-File", + "${workspaceFolder}/tools/Build-Benchmarks.ps1", + "-Scope", + "All" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": "$msCompile", + "presentation": { + "clear": true, + "reveal": "always", + "panel": "dedicated" + }, + "group": "build", + "detail": "Builds only benchmark artifacts in completed exhaustive Release trees." + }, + { + "label": "Run-Benchmarks", + "type": "process", + "command": "pwsh", + "args": [ + "-NoProfile", + "-File", + "${workspaceFolder}/tools/Run-Benchmarks.ps1", + "-Scope", + "All" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [], + "presentation": { + "clear": true, + "reveal": "always", + "panel": "dedicated" + }, + "group": "test", + "detail": "Runs supplemental benchmarks from completed Release fingerprint manifests without building." + }, + { + "label": "Format: All C/C++ Files", + "type": "process", + "command": "pwsh", + "args": [ + "-NoProfile", + "-Command", + "& clang-format -i --style=file --fallback-style=none @(& git ls-files -- '*.c' '*.cc' '*.cpp' '*.cxx' '*.h' '*.hh' '*.hpp' '*.hxx' '*.inl' '*.ipp' '*.cu' '*.cuh')" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [], + "presentation": { + "clear": true, + "reveal": "always", + "panel": "dedicated" + }, + "group": "build", + "detail": "Formats every tracked C and C++ source file with the repository .clang-format file." + } + ] +} diff --git a/CMakeLists.txt b/CMakeLists.txt index e021236..bb8074e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,621 +1,46 @@ -cmake_minimum_required(VERSION 4.4) +cmake_minimum_required(VERSION 3.31) project(SimdLib VERSION 0.2.0 LANGUAGES CXX) -option(SIMDLIB_BUILD_SMOKE_TESTS "Build header-only ODR smoke tests" ON) -option(SIMDLIB_BUILD_TESTS "Build SimdLib Catch2 tests" OFF) -option(SIMDLIB_BUILD_TESTS_128 "Build 128-bit SSE4.2 tests" ON) -option(SIMDLIB_BUILD_TESTS_256 "Build 256-bit AVX2 tests" ON) -option(SIMDLIB_BUILD_TESTS_FMA "Build FMA tests" ON) -option(SIMDLIB_BUILD_TESTS_OPTIONAL "Build optional BMI-family tests" OFF) -option(SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS "Build SimdVector, SimdAlgo, and resampling parity tests" ON) -option(SIMDLIB_BUILD_BENCHMARKS "Build SimdLib Catch2 benchmarks" OFF) -option(SIMDLIB_BUILD_EXAMPLES "Build the executable API example" OFF) -option(SIMDLIB_BUILD_CONFIGURATION_TESTS "Build compile-only configuration probes" ON) -option(SIMDLIB_BUILD_HEADER_TESTS "Build first-and-only public-header probes" ON) -option(SIMDLIB_FETCH_TEST_DEPENDENCIES "Fetch missing test-only dependencies" ON) -option(SIMDLIB_STRICT_WARNINGS "Treat warnings in SimdLib-owned targets as errors" OFF) -option(SIMDLIB_ENABLE_COVERAGE "Instrument SimdLib-owned targets for source coverage" OFF) - -# CTest 4.4 uses this setting during its dashboard Test step to assign a -# collision-free LLVM_PROFILE_FILE to every discovered test invocation. -if(SIMDLIB_ENABLE_COVERAGE) - set(CTEST_TEST_COVERAGE_TOOL "LLVM-COV") -endif() -include(CTest) - add_library(SimdLib INTERFACE) add_library(SimdLib::SimdLib ALIAS SimdLib) - -set(SIMDLIB_MSVC_STYLE_DRIVER ${MSVC}) -if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") - set(SIMDLIB_MSVC_STYLE_DRIVER OFF) -endif() - target_compile_features(SimdLib INTERFACE cxx_std_20) target_include_directories(SimdLib INTERFACE $) target_sources(SimdLib INTERFACE - $) - -# Consumer-facing examples and probes may use focused public headers, but must -# never depend on implementation-only Detail declarations or include paths. -file(GLOB_RECURSE SIMDLIB_PUBLIC_CONSUMER_SOURCES CONFIGURE_DEPENDS - "${CMAKE_CURRENT_SOURCE_DIR}/examples/*.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/tests/consumer/*.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/tests/headers/*.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/tests/smoke/*.cpp") -foreach(consumer_source IN LISTS SIMDLIB_PUBLIC_CONSUMER_SOURCES) - file(READ "${consumer_source}" consumer_source_text) - if(consumer_source_text MATCHES "SimdLib::Detail| --target SimdLibConstexprProbes) - set_tests_properties(SimdLib.ConstexprProbes.Build PROPERTIES LABELS "CONSTEXPR;COMPILE_ONLY" RUN_SERIAL TRUE) -endif() -if(SIMDLIB_BUILD_HEADER_TESTS) - foreach(header_probe IN ITEMS - Config - TemplateTools - Api - SimdApi - SimdVector - SimdAlgo - SimdResample - Bmi - UInt128 - Format - SimdLib - PublicSurface) - add_library(SimdLibHeader${header_probe}Probe OBJECT tests/headers/${header_probe}HeaderProbe.cpp) - target_link_libraries(SimdLibHeader${header_probe}Probe PRIVATE SimdLib::SimdLib) - simdlib_enable_development_warnings(SimdLibHeader${header_probe}Probe) - endforeach() -endif() - -add_library(SimdLibAvailabilityDisabledProbe OBJECT tests/availability/ApiDisabledProbe.cpp) -target_link_libraries(SimdLibAvailabilityDisabledProbe PRIVATE SimdLib::SimdLib) -simdlib_enable_development_warnings(SimdLibAvailabilityDisabledProbe) - -add_library(SimdLibAvailabilityEnabledProbe OBJECT tests/availability/ApiEnabledProbe.cpp) -target_link_libraries(SimdLibAvailabilityEnabledProbe PRIVATE SimdLib::SimdLib) -simdlib_enable_development_warnings(SimdLibAvailabilityEnabledProbe) -if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(SimdLibAvailabilityEnabledProbe PRIVATE /arch:AVX2) -else() - target_compile_options(SimdLibAvailabilityEnabledProbe PRIVATE -mavx2) -endif() - -if(SIMDLIB_BUILD_SMOKE_TESTS) - add_executable(SimdLibHeaderOnlySmoke - tests/smoke/main.cpp - tests/smoke/second_translation_unit.cpp) - target_link_libraries(SimdLibHeaderOnlySmoke PRIVATE SimdLib::SimdLib) - simdlib_enable_development_warnings(SimdLibHeaderOnlySmoke) - add_test(NAME SimdLib.HeaderOnlySmoke COMMAND SimdLibHeaderOnlySmoke) - simdlib_set_coverage_profile_prefix(SimdLibHeaderOnlySmoke - "SimdLib.HeaderOnlySmoke") -endif() - -if(SIMDLIB_BUILD_TESTS) - find_package(Catch2 3 CONFIG QUIET) - if(NOT Catch2_FOUND AND SIMDLIB_FETCH_TEST_DEPENDENCIES) - include(FetchContent) - FetchContent_Declare(Catch2 - GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v3.8.1 - GIT_SHALLOW TRUE) - FetchContent_MakeAvailable(Catch2) - endif() - if(NOT TARGET Catch2::Catch2WithMain) - message(FATAL_ERROR "Catch2 3 is required; install it or enable SIMDLIB_FETCH_TEST_DEPENDENCIES") - endif() - include(Catch) - - # @brief Applies labels after Catch2 has populated its deferred discovery list. - # @param test_list_variable Name of the Catch2-generated test-list variable. - # @param labels Semicolon-separated labels applied to every discovered test. - function(simdlib_label_discovered_tests test_list_variable labels) - set(label_file "${CMAKE_CURRENT_BINARY_DIR}/${test_list_variable}-labels.cmake") - file(WRITE "${label_file}" - "foreach(discovered_test IN LISTS ${test_list_variable})\n" - " set_tests_properties(\"\${discovered_test}\" PROPERTIES LABELS \"${labels}\")\n" - "endforeach()\n") - set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES "${label_file}") - endfunction() - - function(simdlib_add_catch_test target source test_prefix labels) - add_executable(${target} ${source}) - target_link_libraries(${target} PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) - simdlib_enable_development_warnings(${target}) - simdlib_set_coverage_profile_prefix(${target} "${test_prefix}") - set(test_list_variable "${target}_DISCOVERED_TESTS") - catch_discover_tests(${target} - TEST_PREFIX "${test_prefix}." - TEST_LIST ${test_list_variable}) - simdlib_label_discovered_tests(${test_list_variable} "${labels}") - endfunction() - - simdlib_add_catch_test(SimdLibTestsBmiPortable tests/Bmi.tests.cpp - SimdLib.Tests.BmiPortable "BMI;PORTABLE") - target_compile_definitions(SimdLibTestsBmiPortable PRIVATE - SIMDLIB_HAS_BMI1=0 SIMDLIB_HAS_BMI2=0 - SIMDLIB_BMI_EXPECT_BMI1=0 SIMDLIB_BMI_EXPECT_BMI2=0) - if(NOT SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(SimdLibTestsBmiPortable PRIVATE -mno-bmi -mno-bmi2) - endif() - - simdlib_add_catch_test(SimdLibTestsFormat tests/Format.tests.cpp - SimdLib.Tests.Format "FORMAT;SSE42") - add_executable(SimdLibFormatOdr - tests/format_odr/main.cpp - tests/format_odr/second_translation_unit.cpp) - target_link_libraries(SimdLibFormatOdr PRIVATE SimdLib::SimdLib) - simdlib_enable_development_warnings(SimdLibFormatOdr) - add_test(NAME SimdLib.FormatOdr COMMAND SimdLibFormatOdr) - set_tests_properties(SimdLib.FormatOdr PROPERTIES LABELS "FORMAT;ODR") - simdlib_set_coverage_profile_prefix(SimdLibFormatOdr "SimdLib.FormatOdr") - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_definitions(SimdLibTestsFormat PRIVATE - SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) - target_compile_definitions(SimdLibFormatOdr PRIVATE - SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(SimdLibTestsFormat PRIVATE /arch:AVX2) - target_compile_options(SimdLibFormatOdr PRIVATE /arch:AVX2) - endif() - else() - target_compile_options(SimdLibTestsFormat PRIVATE -msse4.2) - target_compile_options(SimdLibFormatOdr PRIVATE -msse4.2) - endif() - - if(SIMDLIB_BUILD_TESTS_128) - simdlib_add_catch_test(SimdLibTests128 tests/Api128.tests.cpp - SimdLib.Tests.SSE42 "SSE42") - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_definitions(SimdLibTests128 PRIVATE - SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(SimdLibTests128 PRIVATE /arch:AVX2) - endif() - else() - target_compile_options(SimdLibTests128 PRIVATE -msse4.2) - endif() - - simdlib_add_catch_test(SimdLibTestsUInt128Optimized tests/UInt128.tests.cpp - SimdLib.Tests.UInt128Optimized "UINT128;OPTIMIZED;SSE42") - simdlib_add_catch_test(SimdLibTestsUInt128Portable tests/UInt128.tests.cpp - SimdLib.Tests.UInt128Portable "UINT128;PORTABLE;SSE42") - simdlib_add_catch_test(SimdLibTestsUInt128Scalar tests/UInt128.tests.cpp - SimdLib.Tests.UInt128Scalar "UINT128;PORTABLE;SCALAR") - target_compile_definitions(SimdLibTestsUInt128Portable PRIVATE - SIMDLIB_USE_COMPILER_CARRY_INTRINSICS=0 SIMDLIB_EXPECT_CARRY_PATH=0) - target_compile_definitions(SimdLibTestsUInt128Scalar PRIVATE SIMDLIB_EXPECT_CARRY_PATH=0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - target_compile_definitions(SimdLibTestsUInt128Optimized PRIVATE SIMDLIB_EXPECT_CARRY_PATH=1) - elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") - target_compile_definitions(SimdLibTestsUInt128Optimized PRIVATE SIMDLIB_EXPECT_CARRY_PATH=2) - endif() - target_compile_definitions(SimdLibTestsUInt128Scalar PRIVATE - SIMDLIB_USE_COMPILER_CARRY_INTRINSICS=0 - SIMDLIB_HAS_SSE=0 SIMDLIB_HAS_SSE2=0 SIMDLIB_HAS_SSE3=0 SIMDLIB_HAS_SSSE3=0 - SIMDLIB_HAS_SSE41=0 SIMDLIB_HAS_SSE42=0 SIMDLIB_HAS_AVX=0 SIMDLIB_HAS_AVX2=0 - SIMDLIB_HAS_FMA=0 SIMDLIB_HAS_BMI1=0 SIMDLIB_HAS_BMI2=0) - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_definitions(SimdLibTestsUInt128Optimized PRIVATE - SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) - target_compile_definitions(SimdLibTestsUInt128Portable PRIVATE - SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(SimdLibTestsUInt128Optimized PRIVATE /arch:AVX2) - target_compile_options(SimdLibTestsUInt128Portable PRIVATE /arch:AVX2) - endif() - else() - target_compile_options(SimdLibTestsUInt128Optimized PRIVATE -msse4.2) - target_compile_options(SimdLibTestsUInt128Portable PRIVATE -msse4.2) - endif() - add_test(NAME SimdLib.Tests.UInt128ResultSetEquivalence - COMMAND ${CMAKE_COMMAND} - -DPORTABLE_EXECUTABLE=$ - -DOPTIMIZED_EXECUTABLE=$ - -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareUInt128ResultSets.cmake) - set_tests_properties(SimdLib.Tests.UInt128ResultSetEquivalence PROPERTIES LABELS "UINT128;EQUIVALENCE;SSE42") - - add_test(NAME SimdLib.Tests.UInt128ScalarResultSetEquivalence - COMMAND ${CMAKE_COMMAND} - -DPORTABLE_EXECUTABLE=$ - -DOPTIMIZED_EXECUTABLE=$ - -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareUInt128ResultSets.cmake) - set_tests_properties(SimdLib.Tests.UInt128ScalarResultSetEquivalence PROPERTIES LABELS "UINT128;EQUIVALENCE;SCALAR") - endif() - - if(SIMDLIB_BUILD_TESTS_256) - simdlib_add_catch_test(SimdLibTests256 tests/Api256.tests.cpp - SimdLib.Tests.AVX2 "AVX2") - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(SimdLibTests256 PRIVATE /arch:AVX2) - else() - target_compile_options(SimdLibTests256 PRIVATE -mavx2) - endif() - endif() - - if(SIMDLIB_BUILD_TESTS_FMA) - simdlib_add_catch_test(SimdLibTestsFmaEnabled tests/SimdFma.tests.cpp - SimdLib.Tests.FMA.Enabled "FMA;ENABLED") - simdlib_add_catch_test(SimdLibTestsFmaDisabled tests/SimdFma.tests.cpp - SimdLib.Tests.FMA.Disabled "FMA;DISABLED") - target_compile_definitions(SimdLibTestsFmaEnabled PRIVATE SIMDLIB_HAS_FMA=1 SIMDLIB_EXPECT_FMA=1) - target_compile_definitions(SimdLibTestsFmaDisabled PRIVATE SIMDLIB_HAS_FMA=0 SIMDLIB_EXPECT_FMA=0) - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(SimdLibTestsFmaEnabled PRIVATE /arch:AVX2) - target_compile_options(SimdLibTestsFmaDisabled PRIVATE /arch:AVX2) - else() - target_compile_options(SimdLibTestsFmaEnabled PRIVATE -mavx2 -mfma) - target_compile_options(SimdLibTestsFmaDisabled PRIVATE -mavx2 -mno-fma) - endif() - endif() - - if(SIMDLIB_BUILD_TESTS_OPTIONAL) - function(simdlib_add_bmi_profile profile_name bmi1 bmi2) - set(target SimdLibTestsBmi${profile_name}) - set(test_name SimdLib.Tests.Bmi.${profile_name}) - simdlib_add_catch_test(${target} tests/Bmi.tests.cpp ${test_name} - "BMI;${profile_name};OPTIONAL") - target_compile_definitions(${target} PRIVATE - SIMDLIB_HAS_BMI1=${bmi1} SIMDLIB_HAS_BMI2=${bmi2} - SIMDLIB_BMI_EXPECT_BMI1=${bmi1} SIMDLIB_BMI_EXPECT_BMI2=${bmi2}) - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(${target} PRIVATE /arch:AVX2) - else() - target_compile_options(${target} PRIVATE -mno-bmi -mno-bmi2) - if(bmi1) - target_compile_options(${target} PRIVATE -mbmi) - endif() - if(bmi2) - target_compile_options(${target} PRIVATE -mbmi2) - endif() - endif() - set(equivalence_name SimdLib.Tests.Bmi.${profile_name}.Equivalence) - add_test(NAME ${equivalence_name} - COMMAND ${CMAKE_COMMAND} - -DPORTABLE_EXECUTABLE=$ - -DENABLED_EXECUTABLE=$ - -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareBmiResultSets.cmake) - set_tests_properties(${equivalence_name} PROPERTIES LABELS "BMI;EQUIVALENCE;${profile_name};OPTIONAL") - endfunction() + $) - simdlib_add_bmi_profile(Bmi1Only 1 0) - simdlib_add_bmi_profile(Bmi2Only 0 1) - simdlib_add_bmi_profile(Bmi1AndBmi2 1 1) - endif() +add_library(SimdLibRegister INTERFACE) +add_library(SimdLib::Register ALIAS SimdLibRegister) +target_link_libraries(SimdLibRegister INTERFACE SimdLib::SimdLib) +target_compile_features(SimdLibRegister INTERFACE cxx_std_23) +target_compile_definitions(SimdLibRegister INTERFACE + SIMDLIB_REQUIRE_REGISTER_INTERFACE=1) +target_compile_options(SimdLibRegister INTERFACE + $<$:/std:c++latest>) - if(SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS) - add_executable(SimdLibTestsVectorAlgorithms - tests/SimdVector.tests.cpp - tests/SimdAlgo.tests.cpp - tests/PreconditionBoundary.tests.cpp - tests/SimdResample.tests.cpp) - target_link_libraries(SimdLibTestsVectorAlgorithms PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) - simdlib_enable_development_warnings(SimdLibTestsVectorAlgorithms) - simdlib_set_coverage_profile_prefix(SimdLibTestsVectorAlgorithms - "SimdLib.Tests.VectorAlgorithms") - catch_discover_tests(SimdLibTestsVectorAlgorithms - TEST_PREFIX "SimdLib.Tests.VectorAlgorithms." - TEST_LIST SimdLibTestsVectorAlgorithms_DISCOVERED_TESTS) - simdlib_label_discovered_tests(SimdLibTestsVectorAlgorithms_DISCOVERED_TESTS - "VECTOR_ALGORITHMS;AVX2") - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(SimdLibTestsVectorAlgorithms PRIVATE /arch:AVX2) - else() - target_compile_options(SimdLibTestsVectorAlgorithms PRIVATE -mavx2 -mfma) - endif() - - simdlib_add_catch_test(SimdLibTestsVectorChecks tests/SimdVectorChecks.tests.cpp - SimdLib.Tests.VectorChecks "VECTOR_ALGORITHMS;AVX2;CHECKS") - target_compile_definitions(SimdLibTestsVectorChecks PRIVATE SIMDLIB_ENABLE_CHECKS=1) - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(SimdLibTestsVectorChecks PRIVATE /arch:AVX2) - else() - target_compile_options(SimdLibTestsVectorChecks PRIVATE -mavx2 -mfma) - endif() - - add_executable(SimdLibPreconditionTests tests/PreconditionFailure.tests.cpp) - target_link_libraries(SimdLibPreconditionTests PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) - simdlib_enable_development_warnings(SimdLibPreconditionTests) - simdlib_set_coverage_profile_prefix(SimdLibPreconditionTests - "SimdLib.Tests.Preconditions") - target_compile_definitions(SimdLibPreconditionTests PRIVATE SIMDLIB_ENABLE_CHECKS=1) - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(SimdLibPreconditionTests PRIVATE /arch:AVX2) - else() - target_compile_options(SimdLibPreconditionTests PRIVATE -mavx2 -mfma) - endif() - catch_discover_tests(SimdLibPreconditionTests - TEST_PREFIX "SimdLib.Tests.Preconditions." - TEST_LIST SimdLibPreconditionTests_DISCOVERED_TESTS - PROPERTIES - PASS_REGULAR_EXPRESSION "SIMDLIB_PRECONDITION_FAILURE_EXPECTED_18A7E3" - TIMEOUT 10) - simdlib_label_discovered_tests(SimdLibPreconditionTests_DISCOVERED_TESTS - "PRECONDITIONS;CHECKS;AVX2") - - add_executable(SimdLibTestsResampleScalar tests/SimdResample.tests.cpp) - target_link_libraries(SimdLibTestsResampleScalar PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) - simdlib_enable_development_warnings(SimdLibTestsResampleScalar) - simdlib_set_coverage_profile_prefix(SimdLibTestsResampleScalar - "SimdLib.Tests.ResampleScalar") - target_compile_definitions(SimdLibTestsResampleScalar PRIVATE - SIMDLIB_HAS_SSE3=0 SIMDLIB_HAS_SSSE3=0 SIMDLIB_HAS_SSE41=0 SIMDLIB_HAS_SSE42=0 - SIMDLIB_HAS_AVX=0 SIMDLIB_HAS_AVX2=0 SIMDLIB_HAS_FMA=0) - catch_discover_tests(SimdLibTestsResampleScalar - TEST_PREFIX "SimdLib.Tests.ResampleScalar." - TEST_LIST SimdLibTestsResampleScalar_DISCOVERED_TESTS) - simdlib_label_discovered_tests(SimdLibTestsResampleScalar_DISCOVERED_TESTS - "VECTOR_ALGORITHMS;SCALAR") - endif() -endif() - -if(SIMDLIB_BUILD_BENCHMARKS) - if(NOT TARGET Catch2::Catch2WithMain) - find_package(Catch2 3 CONFIG QUIET) - endif() - if(NOT Catch2_FOUND AND NOT TARGET Catch2::Catch2WithMain AND SIMDLIB_FETCH_TEST_DEPENDENCIES) - include(FetchContent) - FetchContent_Declare(Catch2 - GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v3.8.1 - GIT_SHALLOW TRUE) - FetchContent_MakeAvailable(Catch2) - endif() - if(NOT TARGET Catch2::Catch2WithMain) - message(FATAL_ERROR "Catch2 3 is required; install it or enable SIMDLIB_FETCH_TEST_DEPENDENCIES") - endif() - add_executable(SimdLibBenchmarks benchmarks/SimdLib.benchmarks.cpp) - target_link_libraries(SimdLibBenchmarks PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) - simdlib_enable_development_warnings(SimdLibBenchmarks) - target_compile_definitions(SimdLibBenchmarks PRIVATE SIMDLIB_HAS_BMI1=1 SIMDLIB_HAS_BMI2=1) - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(SimdLibBenchmarks PRIVATE /arch:AVX2) - else() - target_compile_options(SimdLibBenchmarks PRIVATE -mavx2 -mfma -mbmi -mbmi2) - endif() -endif() - -if(SIMDLIB_BUILD_EXAMPLES) - add_executable(SimdLibApiExamples examples/ApiExamples.cpp) - target_link_libraries(SimdLibApiExamples PRIVATE SimdLib::SimdLib) - simdlib_enable_development_warnings(SimdLibApiExamples) - if(SIMDLIB_MSVC_STYLE_DRIVER) - target_compile_options(SimdLibApiExamples PRIVATE /arch:AVX2) - else() - target_compile_options(SimdLibApiExamples PRIVATE -mavx2 -mfma -mbmi -mbmi2) - endif() - add_test(NAME SimdLib.ApiExamples COMMAND SimdLibApiExamples) - set_tests_properties(SimdLib.ApiExamples PROPERTIES LABELS "EXAMPLES;AVX2;FMA;BMI") - simdlib_set_coverage_profile_prefix(SimdLibApiExamples "SimdLib.ApiExamples") -endif() - -if(SIMDLIB_ENABLE_COVERAGE) - get_filename_component(simdlib_compiler_directory "${CMAKE_CXX_COMPILER}" DIRECTORY) - find_program(SIMDLIB_LLVM_PROFDATA - NAMES llvm-profdata - HINTS "${simdlib_compiler_directory}" - REQUIRED) - find_program(SIMDLIB_LLVM_COV - NAMES llvm-cov - HINTS "${simdlib_compiler_directory}" - REQUIRED) - find_program(SIMDLIB_LLVM_READOBJ - NAMES llvm-readobj - HINTS "${simdlib_compiler_directory}" - REQUIRED) - - get_property(simdlib_coverage_targets GLOBAL PROPERTY SIMDLIB_COVERAGE_TARGETS) - list(REMOVE_DUPLICATES simdlib_coverage_targets) - if(NOT simdlib_coverage_targets) - message(FATAL_ERROR "SIMDLIB_ENABLE_COVERAGE requires at least one executable target") - endif() - - set(simdlib_coverage_manifest "") - foreach(coverage_target IN LISTS simdlib_coverage_targets) - get_target_property(coverage_profile_prefix ${coverage_target} - SIMDLIB_COVERAGE_PROFILE_PREFIX) - if(NOT coverage_profile_prefix) - message(FATAL_ERROR - "Coverage target ${coverage_target} has no CTest profile prefix") - endif() - string(APPEND simdlib_coverage_manifest - "${coverage_target}|$|${coverage_profile_prefix}\n") - endforeach() - set(simdlib_coverage_manifest_file - "${CMAKE_CURRENT_BINARY_DIR}/coverage-targets-$.txt") - file(GENERATE - OUTPUT "${simdlib_coverage_manifest_file}" - CONTENT "${simdlib_coverage_manifest}") - - add_custom_target(SimdLibCoverageReset - COMMAND ${CMAKE_COMMAND} - -DBINARY_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR} - -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ResetCoverage.cmake - COMMENT "Removing previous SimdLib coverage data" - VERBATIM) - - add_custom_target(SimdLibCoverageReport - COMMAND ${CMAKE_COMMAND} - -DBINARY_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR} - -DSOURCE_DIRECTORY=${CMAKE_CURRENT_SOURCE_DIR} - -DCOVERAGE_MANIFEST=${simdlib_coverage_manifest_file} - -DLLVM_PROFDATA=${SIMDLIB_LLVM_PROFDATA} - -DLLVM_COV=${SIMDLIB_LLVM_COV} - -DLLVM_READOBJ=${SIMDLIB_LLVM_READOBJ} - -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/GenerateCoverageReport.cmake - DEPENDS ${simdlib_coverage_targets} - COMMENT "Generating SimdLib LCOV coverage report" - VERBATIM) +set(SIMDLIB_MSVC_STYLE_DRIVER ${MSVC}) +if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" + AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") + set(SIMDLIB_MSVC_STYLE_DRIVER OFF) +endif() + +set(SIMDLIB_REGISTER_COMPILER_SUPPORTED OFF) +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" + AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 19.44) + set(SIMDLIB_REGISTER_COMPILER_SUPPORTED ON) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" + AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 20) + set(SIMDLIB_REGISTER_COMPILER_SUPPORTED ON) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" + AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 14 + AND CMAKE_SYSTEM_NAME STREQUAL "Linux" + AND CMAKE_SIZEOF_VOID_P EQUAL 8) + set(SIMDLIB_REGISTER_COMPILER_SUPPORTED ON) +endif() +set_property(TARGET SimdLibRegister PROPERTY + SIMDLIB_REGISTER_COMPILER_SUPPORTED ${SIMDLIB_REGISTER_COMPILER_SUPPORTED}) + +if(PROJECT_IS_TOP_LEVEL) + include(cmake/development/Development.cmake) endif() diff --git a/CMakePresets.json b/CMakePresets.json index de93a49..51549db 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -1,85 +1,463 @@ { - "version": 11, + "version": 10, "cmakeMinimumRequired": { - "major": 4, - "minor": 4, + "major": 3, + "minor": 31, "patch": 0 }, "configurePresets": [ { - "name": "msvc", - "displayName": "MSVC development", - "description": "Release-capable MSVC build with the complete test matrix", - "generator": "Visual Studio 17 2022", - "architecture": "x64", - "binaryDir": "${sourceDir}/build", + "name": "development-base-options", + "hidden": true, "cacheVariables": { - "SIMDLIB_BUILD_TESTS": "ON", - "SIMDLIB_BUILD_TESTS_OPTIONAL": "ON", + "BUILD_TESTING": "ON", + "SIMDLIB_BUILD_SMOKE_TESTS": "OFF", + "SIMDLIB_BUILD_RUNTIME_TESTS": "OFF", + "SIMDLIB_BUILD_API_SSE42_TESTS": "OFF", + "SIMDLIB_BUILD_API_AVX2_TESTS": "OFF", + "SIMDLIB_BUILD_FMA_TESTS": "OFF", + "SIMDLIB_BUILD_BMI_TESTS": "OFF", + "SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS": "OFF", "SIMDLIB_BUILD_BENCHMARKS": "OFF", + "SIMDLIB_BUILD_EXAMPLES": "OFF", + "SIMDLIB_BUILD_CONFIGURATION_PROBES": "OFF", + "SIMDLIB_BUILD_CONSTEXPR_PROBES": "OFF", + "SIMDLIB_BUILD_HEADER_PROBES": "OFF", + "SIMDLIB_FETCH_TEST_DEPENDENCIES": "ON", "SIMDLIB_STRICT_WARNINGS": "ON", - "SIMDLIB_ENABLE_COVERAGE": "OFF" + "SIMDLIB_ENABLE_COVERAGE": "OFF", + "SIMDLIB_DEFAULT_CHECKS_PROBE": "NONE", + "SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES": "OFF", + "SIMDLIB_BUILD_REGISTER_CODEGEN_GATES": "OFF", + "SIMDLIB_REGISTER_CODEGEN_MODE": "OFF", + "SIMDLIB_VALIDATION_PROFILE": "CUSTOM", + "SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS": "ON" + } + }, + { + "name": "release-exhaustive-options", + "hidden": true, + "inherits": "development-base-options", + "cacheVariables": { + "SIMDLIB_BUILD_SMOKE_TESTS": "ON", + "SIMDLIB_BUILD_RUNTIME_TESTS": "ON", + "SIMDLIB_BUILD_API_SSE42_TESTS": "ON", + "SIMDLIB_BUILD_API_AVX2_TESTS": "ON", + "SIMDLIB_BUILD_FMA_TESTS": "ON", + "SIMDLIB_BUILD_BMI_TESTS": "ON", + "SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS": "ON", + "SIMDLIB_BUILD_BENCHMARKS": "ON", + "SIMDLIB_BUILD_EXAMPLES": "ON", + "SIMDLIB_BUILD_CONFIGURATION_PROBES": "ON", + "SIMDLIB_BUILD_CONSTEXPR_PROBES": "ON", + "SIMDLIB_BUILD_HEADER_PROBES": "ON", + "SIMDLIB_DEFAULT_CHECKS_PROBE": "RELEASE", + "SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES": "ON", + "SIMDLIB_BUILD_REGISTER_CODEGEN_GATES": "ON", + "SIMDLIB_REGISTER_CODEGEN_MODE": "ENFORCE", + "SIMDLIB_VALIDATION_PROFILE": "RELEASE", + "SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS": "ON" + } + }, + { + "name": "debug-diagnostics-options", + "hidden": true, + "inherits": "development-base-options", + "cacheVariables": { + "SIMDLIB_BUILD_RUNTIME_TESTS": "ON", + "SIMDLIB_BUILD_API_SSE42_TESTS": "ON", + "SIMDLIB_BUILD_API_AVX2_TESTS": "ON", + "SIMDLIB_BUILD_FMA_TESTS": "ON", + "SIMDLIB_BUILD_BMI_TESTS": "OFF", + "SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS": "ON", + "SIMDLIB_BUILD_BENCHMARKS": "OFF", + "SIMDLIB_BUILD_EXAMPLES": "OFF", + "SIMDLIB_BUILD_SMOKE_TESTS": "OFF", + "SIMDLIB_BUILD_CONSTEXPR_PROBES": "OFF", + "SIMDLIB_BUILD_CONFIGURATION_PROBES": "OFF", + "SIMDLIB_BUILD_HEADER_PROBES": "OFF", + "SIMDLIB_DEFAULT_CHECKS_PROBE": "DEBUG", + "SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES": "OFF", + "SIMDLIB_BUILD_REGISTER_CODEGEN_GATES": "OFF", + "SIMDLIB_REGISTER_CODEGEN_MODE": "OFF", + "SIMDLIB_VALIDATION_PROFILE": "DEBUG", + "SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS": "ON" } }, { - "name": "clang-coverage", - "displayName": "Clang LLVM coverage", - "description": "Debug Clang build instrumented for CTest LLVM coverage", + "name": "codegen-diagnostic-options", + "hidden": true, + "inherits": "development-base-options", + "cacheVariables": { + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "SIMDLIB_BUILD_RUNTIME_TESTS": "OFF", + "SIMDLIB_BUILD_API_SSE42_TESTS": "OFF", + "SIMDLIB_BUILD_API_AVX2_TESTS": "OFF", + "SIMDLIB_BUILD_FMA_TESTS": "OFF", + "SIMDLIB_BUILD_BMI_TESTS": "OFF", + "SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS": "OFF", + "SIMDLIB_BUILD_BENCHMARKS": "OFF", + "SIMDLIB_BUILD_EXAMPLES": "OFF", + "SIMDLIB_BUILD_SMOKE_TESTS": "OFF", + "SIMDLIB_BUILD_CONSTEXPR_PROBES": "OFF", + "SIMDLIB_BUILD_CONFIGURATION_PROBES": "OFF", + "SIMDLIB_BUILD_HEADER_PROBES": "OFF", + "SIMDLIB_FETCH_TEST_DEPENDENCIES": "OFF", + "SIMDLIB_BUILD_REGISTER_CODEGEN_GATES": "ON", + "SIMDLIB_REGISTER_CODEGEN_MODE": "RECORD", + "SIMDLIB_VALIDATION_PROFILE": "CODEGEN_DIAGNOSTIC", + "SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS": "ON" + } + }, + { + "name": "asan-ubsan-codegen-diagnostic-options", + "hidden": true, + "inherits": "codegen-diagnostic-options", + "cacheVariables": { + "CMAKE_CXX_FLAGS_DEBUG": "-fsanitize=address,undefined -fno-omit-frame-pointer", + "CMAKE_EXE_LINKER_FLAGS_DEBUG": "-fsanitize=address,undefined" + } + }, + { + "name": "debug-asan-ubsan-options", + "hidden": true, + "inherits": "development-base-options", + "cacheVariables": { + "CMAKE_CXX_FLAGS_DEBUG": "-fsanitize=address,undefined -fno-omit-frame-pointer", + "CMAKE_EXE_LINKER_FLAGS_DEBUG": "-fsanitize=address,undefined", + "SIMDLIB_BUILD_RUNTIME_TESTS": "ON", + "SIMDLIB_BUILD_API_SSE42_TESTS": "ON", + "SIMDLIB_BUILD_API_AVX2_TESTS": "ON", + "SIMDLIB_BUILD_FMA_TESTS": "ON", + "SIMDLIB_BUILD_BMI_TESTS": "ON", + "SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS": "ON", + "SIMDLIB_BUILD_BENCHMARKS": "OFF", + "SIMDLIB_BUILD_EXAMPLES": "OFF", + "SIMDLIB_BUILD_SMOKE_TESTS": "OFF", + "SIMDLIB_BUILD_CONFIGURATION_PROBES": "OFF", + "SIMDLIB_BUILD_CONSTEXPR_PROBES": "OFF", + "SIMDLIB_BUILD_HEADER_PROBES": "OFF", + "SIMDLIB_DEFAULT_CHECKS_PROBE": "DEBUG", + "SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES": "OFF", + "SIMDLIB_BUILD_REGISTER_CODEGEN_GATES": "OFF", + "SIMDLIB_REGISTER_CODEGEN_MODE": "OFF", + "SIMDLIB_VALIDATION_PROFILE": "SANITIZER", + "SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS": "ON" + } + }, + { + "name": "coverage-options", + "hidden": true, + "inherits": "development-base-options", + "cacheVariables": { + "SIMDLIB_BUILD_RUNTIME_TESTS": "ON", + "SIMDLIB_BUILD_API_SSE42_TESTS": "ON", + "SIMDLIB_BUILD_API_AVX2_TESTS": "ON", + "SIMDLIB_BUILD_FMA_TESTS": "ON", + "SIMDLIB_BUILD_BMI_TESTS": "ON", + "SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS": "ON", + "SIMDLIB_BUILD_BENCHMARKS": "OFF", + "SIMDLIB_BUILD_EXAMPLES": "OFF", + "SIMDLIB_BUILD_SMOKE_TESTS": "OFF", + "SIMDLIB_BUILD_CONFIGURATION_PROBES": "OFF", + "SIMDLIB_BUILD_CONSTEXPR_PROBES": "OFF", + "SIMDLIB_BUILD_HEADER_PROBES": "OFF", + "SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES": "OFF", + "SIMDLIB_BUILD_REGISTER_CODEGEN_GATES": "OFF", + "SIMDLIB_REGISTER_CODEGEN_MODE": "OFF", + "SIMDLIB_ENABLE_COVERAGE": "ON", + "SIMDLIB_VALIDATION_PROFILE": "COVERAGE", + "SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS": "ON" + } + }, + { + "name": "compiler-contract-options", + "hidden": true, + "inherits": "development-base-options", + "cacheVariables": { + "SIMDLIB_BUILD_CONFIGURATION_PROBES": "ON", + "SIMDLIB_BUILD_HEADER_PROBES": "ON", + "SIMDLIB_FETCH_TEST_DEPENDENCIES": "OFF", + "SIMDLIB_DEFAULT_CHECKS_PROBE": "RELEASE", + "SIMDLIB_VALIDATION_PROFILE": "COMPILER_CONTRACTS", + "SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS": "ON" + } + }, + { + "name": "msvc-common", + "hidden": true, + "generator": "Visual Studio 17 2022", + "architecture": "x64", + "binaryDir": "$env{SIMDLIB_BUILD_DIRECTORY}" + }, + { + "name": "msvc-ninja-common", + "hidden": true, "generator": "Ninja", - "binaryDir": "${sourceDir}/build-coverage", + "binaryDir": "$env{SIMDLIB_BUILD_DIRECTORY}", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "cl", + "CMAKE_MAKE_PROGRAM": "$env{SIMDLIB_NINJA}" + } + }, + { + "name": "clangcl-common", + "hidden": true, + "generator": "Ninja", + "binaryDir": "$env{SIMDLIB_BUILD_DIRECTORY}", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "clang-cl", + "CMAKE_MAKE_PROGRAM": "$env{SIMDLIB_NINJA}" + } + }, + { + "name": "container-common", + "hidden": true, + "generator": "Ninja", + "binaryDir": "$env{SIMDLIB_BUILD_DIRECTORY}", + "cacheVariables": { + "CMAKE_CXX_STANDARD": "20", + "CMAKE_CXX_STANDARD_REQUIRED": "ON", + "CMAKE_CXX_EXTENSIONS": "OFF", + "CMAKE_CXX_SCAN_FOR_MODULES": "OFF" + } + }, + { + "name": "container-release-exhaustive", + "hidden": true, + "inherits": ["container-common", "release-exhaustive-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "container-debug-diagnostics", + "hidden": true, + "inherits": ["container-common", "debug-diagnostics-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "container-debug-codegen-diagnostic", + "hidden": true, + "inherits": ["container-common", "codegen-diagnostic-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "container-asan-ubsan-codegen-diagnostic", + "hidden": true, + "inherits": ["container-common", "asan-ubsan-codegen-diagnostic-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "msvc-release-exhaustive", + "displayName": "MSVC Release exhaustive", + "description": "Windows x64 optimized correctness, ABI, and enforced generated-code qualification", + "inherits": ["msvc-common", "release-exhaustive-options"], + "cacheVariables": { + "CMAKE_CONFIGURATION_TYPES": "Release" + } + }, + { + "name": "msvc-debug-diagnostics", + "displayName": "MSVC Debug diagnostics", + "description": "Windows x64 Debug runtime correctness without generated-code diagnostics", + "inherits": ["msvc-common", "debug-diagnostics-options"], + "cacheVariables": { + "CMAKE_CONFIGURATION_TYPES": "Debug" + } + }, + { + "name": "msvc-compiler-contracts", + "displayName": "MSVC compiler contracts", + "description": "Focused MSVC preprocessing, header, configuration, and language contracts", + "inherits": ["msvc-common", "compiler-contract-options"], + "cacheVariables": { + "CMAKE_CONFIGURATION_TYPES": "Release" + } + }, + { + "name": "msvc-debug-codegen-diagnostic", + "displayName": "MSVC Debug codegen diagnostic", + "description": "Optional MSVC Debug record-only Register generated-code diagnostic", + "inherits": ["msvc-ninja-common", "codegen-diagnostic-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "clangcl-release-exhaustive", + "displayName": "clang-cl Release exhaustive", + "description": "Windows x64 optimized correctness, ABI, and enforced generated-code qualification", + "inherits": ["clangcl-common", "release-exhaustive-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "clangcl-debug-diagnostics", + "displayName": "clang-cl Debug diagnostics", + "description": "Windows x64 Debug runtime correctness without generated-code diagnostics", + "inherits": ["clangcl-common", "debug-diagnostics-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "clangcl-compiler-contracts", + "displayName": "clang-cl compiler contracts", + "description": "Focused clang-cl preprocessing, header, configuration, and language contracts", + "inherits": ["clangcl-common", "compiler-contract-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "clangcl-debug-codegen-diagnostic", + "displayName": "clang-cl Debug codegen diagnostic", + "description": "Optional clang-cl Debug record-only Register generated-code diagnostic", + "inherits": ["clangcl-common", "codegen-diagnostic-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "gcc13-core-release-exhaustive", + "displayName": "GCC 13.2 core Release exhaustive", + "inherits": ["container-release-exhaustive"], + "description": "Linux x64 C++20 core-only Release qualification", + "cacheVariables": { + "SIMDLIB_BUILD_REGISTER_CODEGEN_GATES": "OFF", + "SIMDLIB_REGISTER_CODEGEN_MODE": "OFF" + } + }, + { + "name": "gcc13-core-debug-diagnostics", + "displayName": "GCC 13.2 core Debug diagnostics", + "inherits": ["container-debug-diagnostics"], + "description": "Linux x64 C++20 core-only Debug qualification" + }, + { + "name": "gcc14-release-exhaustive", + "displayName": "GCC 14 Release exhaustive", + "description": "Linux x64 optimized core and Register qualification in the pinned GCC 14 image", + "inherits": ["container-release-exhaustive"] + }, + { + "name": "gcc14-debug-diagnostics", + "displayName": "GCC 14 Debug diagnostics", + "description": "Linux x64 Debug core and Register diagnostics in the pinned GCC 14 image", + "inherits": ["container-debug-diagnostics"] + }, + { + "name": "gcc14-debug-codegen-diagnostic", + "displayName": "GCC 14 Debug codegen diagnostic", + "description": "Optional GCC 14 Debug record-only Register generated-code diagnostic", + "inherits": ["container-debug-codegen-diagnostic"] + }, + { + "name": "clang22-release-exhaustive", + "displayName": "Clang 22 Release exhaustive", + "description": "Linux x64 optimized core and Register qualification in the pinned Clang 22 image", + "inherits": ["container-release-exhaustive"] + }, + { + "name": "clang22-debug-diagnostics", + "displayName": "Clang 22 Debug diagnostics", + "description": "Linux x64 Debug core and Register diagnostics in the pinned Clang 22 image", + "inherits": ["container-debug-diagnostics"] + }, + { + "name": "clang22-debug-codegen-diagnostic", + "displayName": "Clang 22 Debug codegen diagnostic", + "description": "Optional Clang 22 Debug record-only Register generated-code diagnostic", + "inherits": ["container-debug-codegen-diagnostic"] + }, + { + "name": "clang22-asan-ubsan-codegen-diagnostic", + "displayName": "Clang 22 ASan and UBSan codegen diagnostic", + "description": "Optional Clang 22 sanitizer-instrumented record-only Register generated-code diagnostic", + "inherits": ["container-asan-ubsan-codegen-diagnostic"] + }, + { + "name": "clang22-debug-asan-ubsan", + "displayName": "Clang 22 Debug ASan and UBSan", + "description": "Independent Linux x64 Clang AddressSanitizer and UndefinedBehaviorSanitizer fingerprint", + "inherits": ["container-common", "debug-asan-ubsan-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "clang-debug-coverage", + "displayName": "Clang Debug coverage", + "description": "Independent native Clang Debug source-coverage fingerprint", + "generator": "Ninja", + "binaryDir": "$env{SIMDLIB_BUILD_DIRECTORY}", + "inherits": "coverage-options", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_CXX_COMPILER": "clang++", - "SIMDLIB_BUILD_TESTS": "ON", - "SIMDLIB_BUILD_TESTS_OPTIONAL": "ON", + "CMAKE_MAKE_PROGRAM": "$env{SIMDLIB_NINJA}" + } + }, + { + "name": "container-release-contracts", + "displayName": "Container Release contracts", + "description": "Narrow Linux Release contract preset for direct container-environment diagnostics", + "inherits": ["container-common", "compiler-contract-options"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release", + "SIMDLIB_BUILD_RUNTIME_TESTS": "OFF", + "SIMDLIB_BUILD_SMOKE_TESTS": "OFF", "SIMDLIB_BUILD_BENCHMARKS": "OFF", - "SIMDLIB_STRICT_WARNINGS": "ON", - "SIMDLIB_ENABLE_COVERAGE": "ON" + "SIMDLIB_BUILD_EXAMPLES": "OFF", + "SIMDLIB_BUILD_CONSTEXPR_PROBES": "OFF", + "SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES": "OFF", + "SIMDLIB_BUILD_REGISTER_CODEGEN_GATES": "OFF", + "SIMDLIB_REGISTER_CODEGEN_MODE": "OFF", + "SIMDLIB_DEFAULT_CHECKS_PROBE": "RELEASE", + "SIMDLIB_VALIDATION_PROFILE": "COMPILER_CONTRACTS" } } ], "buildPresets": [ - { - "name": "msvc-release", - "displayName": "MSVC Release", - "configurePreset": "msvc", - "configuration": "Release", - "jobs": 0 - }, - { - "name": "coverage", - "displayName": "Clang LLVM coverage", - "configurePreset": "clang-coverage", - "jobs": 0 - } + { "name": "msvc-release-exhaustive", "description": "Build the MSVC Release exhaustive validation artifacts", "configurePreset": "msvc-release-exhaustive", "configuration": "Release", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "msvc-release-benchmarks", "description": "Build only benchmark executables in the existing MSVC Release tree", "configurePreset": "msvc-release-exhaustive", "configuration": "Release", "targets": ["BenchmarkArtifacts"], "jobs": 0 }, + { "name": "msvc-debug-diagnostics", "description": "Build the MSVC Debug diagnostic artifacts", "configurePreset": "msvc-debug-diagnostics", "configuration": "Debug", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "msvc-compiler-contracts", "description": "Build only MSVC compiler-front-end contracts", "configurePreset": "msvc-compiler-contracts", "configuration": "Release", "targets": ["SimdLibCompilerContractArtifacts"], "jobs": 0 }, + { "name": "msvc-debug-codegen-diagnostic", "description": "Record only MSVC Debug Register generated code", "configurePreset": "msvc-debug-codegen-diagnostic", "targets": ["SimdLibDebugDiagnosticArtifacts"], "jobs": 0 }, + { "name": "clangcl-release-exhaustive", "description": "Build the clang-cl Release exhaustive validation artifacts", "configurePreset": "clangcl-release-exhaustive", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "clangcl-release-benchmarks", "description": "Build only benchmark executables in the existing clang-cl Release tree", "configurePreset": "clangcl-release-exhaustive", "targets": ["BenchmarkArtifacts"], "jobs": 0 }, + { "name": "clangcl-debug-diagnostics", "description": "Build the clang-cl Debug diagnostic artifacts", "configurePreset": "clangcl-debug-diagnostics", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "clangcl-compiler-contracts", "description": "Build only clang-cl compiler-front-end contracts", "configurePreset": "clangcl-compiler-contracts", "targets": ["SimdLibCompilerContractArtifacts"], "jobs": 0 }, + { "name": "clangcl-debug-codegen-diagnostic", "description": "Record only clang-cl Debug Register generated code", "configurePreset": "clangcl-debug-codegen-diagnostic", "targets": ["SimdLibDebugDiagnosticArtifacts"], "jobs": 0 }, + { "name": "gcc13-core-release-exhaustive", "description": "Build the GCC 13.2 core-only Release validation artifacts", "configurePreset": "gcc13-core-release-exhaustive", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "gcc13-core-release-benchmarks", "description": "Build only core benchmark executables in the existing GCC 13.2 Release tree", "configurePreset": "gcc13-core-release-exhaustive", "targets": ["BenchmarkArtifacts"], "jobs": 0 }, + { "name": "gcc13-core-debug-diagnostics", "description": "Build the GCC 13.2 core-only Debug diagnostic artifacts", "configurePreset": "gcc13-core-debug-diagnostics", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "gcc14-release-exhaustive", "description": "Build the GCC 14 Release exhaustive validation artifacts", "configurePreset": "gcc14-release-exhaustive", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "gcc14-release-benchmarks", "description": "Build only benchmark executables in the existing GCC 14 Release tree", "configurePreset": "gcc14-release-exhaustive", "targets": ["BenchmarkArtifacts"], "jobs": 0 }, + { "name": "gcc14-debug-diagnostics", "description": "Build the GCC 14 Debug diagnostic artifacts", "configurePreset": "gcc14-debug-diagnostics", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "gcc14-debug-codegen-diagnostic", "description": "Record only GCC 14 Debug Register generated code", "configurePreset": "gcc14-debug-codegen-diagnostic", "targets": ["SimdLibDebugDiagnosticArtifacts"], "jobs": 0 }, + { "name": "clang22-release-exhaustive", "description": "Build the Clang 22 Release exhaustive validation artifacts", "configurePreset": "clang22-release-exhaustive", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "clang22-release-benchmarks", "description": "Build only benchmark executables in the existing Clang 22 Release tree", "configurePreset": "clang22-release-exhaustive", "targets": ["BenchmarkArtifacts"], "jobs": 0 }, + { "name": "clang22-debug-diagnostics", "description": "Build the Clang 22 Debug diagnostic artifacts", "configurePreset": "clang22-debug-diagnostics", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "clang22-debug-codegen-diagnostic", "description": "Record only Clang 22 Debug Register generated code", "configurePreset": "clang22-debug-codegen-diagnostic", "targets": ["SimdLibDebugDiagnosticArtifacts"], "jobs": 0 }, + { "name": "clang22-asan-ubsan-codegen-diagnostic", "description": "Record only Clang 22 sanitizer-instrumented Register generated code", "configurePreset": "clang22-asan-ubsan-codegen-diagnostic", "targets": ["SimdLibDebugDiagnosticArtifacts"], "jobs": 0 }, + { "name": "clang22-debug-asan-ubsan", "description": "Build the Clang 22 ASan and UBSan validation artifacts", "configurePreset": "clang22-debug-asan-ubsan", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "clang-debug-coverage", "description": "Build the native Clang coverage validation artifacts", "configurePreset": "clang-debug-coverage", "targets": ["ExhaustiveArtifacts"], "jobs": 0 }, + { "name": "container-release-contracts", "description": "Build the narrow container Release contract artifacts", "configurePreset": "container-release-contracts", "targets": ["SimdLibCompilerContractArtifacts"], "jobs": 0 } ], "testPresets": [ - { - "name": "msvc-release", - "displayName": "MSVC Release", - "configurePreset": "msvc", - "configuration": "Release", - "output": { - "outputOnFailure": true - }, - "execution": { - "jobs": 0 - } - }, - { - "name": "coverage", - "displayName": "Clang LLVM coverage", - "configurePreset": "clang-coverage", - "inheritConfigureEnvironment": true, - "environment": { - "LLVM_PROFILE_FILE": "${sourceDir}/build-coverage/ctest-%p-%m.profraw" - }, - "output": { - "outputOnFailure": true - }, - "execution": { - "jobs": 0 - } - } + { "name": "msvc-release-exhaustive", "description": "Run the MSVC Release runtime and artifact-validation inventory", "configurePreset": "msvc-release-exhaustive", "configuration": "Release", "output": { "outputOnFailure": true }, "execution": { "jobs": 0 } }, + { "name": "msvc-debug-diagnostics", "description": "Run the MSVC Debug runtime and artifact-validation inventory", "configurePreset": "msvc-debug-diagnostics", "configuration": "Debug", "output": { "outputOnFailure": true }, "execution": { "jobs": 0 } }, + { "name": "clangcl-release-exhaustive", "description": "Run the clang-cl Release runtime and artifact-validation inventory", "configurePreset": "clangcl-release-exhaustive", "output": { "outputOnFailure": true }, "execution": { "jobs": 0 } }, + { "name": "clangcl-debug-diagnostics", "description": "Run the clang-cl Debug runtime and artifact-validation inventory", "configurePreset": "clangcl-debug-diagnostics", "output": { "outputOnFailure": true }, "execution": { "jobs": 0 } }, + { "name": "clang-debug-coverage", "description": "Run the native Clang coverage inventory with isolated profile output", "configurePreset": "clang-debug-coverage", "inheritConfigureEnvironment": true, "environment": { "LLVM_PROFILE_FILE": "$env{SIMDLIB_BUILD_DIRECTORY}/ctest-%p-%m.profraw" }, "output": { "outputOnFailure": true }, "execution": { "jobs": 0 } } ] } diff --git a/README.md b/README.md index 5a6f6e1..e329524 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,24 @@ # SimdLib -SimdLib is a small, header-only C++20 library for working with SIMD data and -bit-heavy code without scattering compiler intrinsics throughout your project. -It brings register operations, fixed-size vectors, bulk algorithms, bit -manipulation helpers, and a practical 128-bit integer under one consistent API. +SimdLib is a small, header-only library for working with SIMD data and bit-heavy +code without scattering compiler intrinsics throughout your project. Its core +surface remains C++20; supporting C++23 translation units can additionally use +the complete-register value interface. There is no library binary to build or ship. Add the headers to your project, link the CMake interface target, and use only the pieces you need. ## What is included? -- `NativeApi` provides a typed SIMD facade and automatically selects the - widest register supported by the compile target. -- `Api` remains available when an algorithm needs an explicit 128-bit or - 256-bit register width. +- `NativeRegister` is the preferred C++23 value interface for operations on + one complete target-selected SIMD register. +- `Register` selects an explicit 128-bit or 256-bit representation for + stable storage and ABI contracts. +- `RegisterMask` preserves native comparison predicates and provides + composition, reduction, observation, and selection operations. +- `NativeApi` and `Api` remain supported for C++20, compatibility, + specialized low-level access, collection helpers, and operations intentionally + excluded from `Register`. - `SimdVector` wraps a register in a fixed-size, value-like container. - `SimdAlgo` applies common operations to arrays and spans. - `SimdResample` packs and expands byte masks, with a scalar fallback when the @@ -21,8 +26,10 @@ link the CMake interface target, and use only the pieces you need. - `Bmi` collects portable and hardware-assisted bit-manipulation helpers. - `uint128_t` provides an unsigned 128-bit value type with formatting support. -SimdLib is currently aimed at x86 and x64 projects and is tested with MSVC, -clang-cl, Clang, and GCC. It requires C++20. +SimdLib targets Windows x64 with MSVC or clang-cl and Linux x64 with Clang or +GCC. The core requires C++20. GCC 13.2 qualifies the Linux core-only surface; +GCC 14 or newer qualifies both the core and `Register`. `Register` otherwise +requires a supported C++23 compiler with explicit-object member support. ## Add it to a project @@ -34,7 +41,14 @@ add_subdirectory(external/SimdLib) target_link_libraries(MyTarget PRIVATE SimdLib::SimdLib) ``` -Then include the complete public surface: +Link the opt-in target for a C++23 translation unit that uses `Register`: + +```cmake +target_link_libraries(MyRegisterTarget PRIVATE SimdLib::Register) +``` + +Then include the complete public surface. The umbrella exposes `Register` only +when `SIMDLIB_REGISTER_INTERFACE_AVAILABLE` is nonzero: ```cpp #include @@ -68,6 +82,128 @@ const Vector3 cameraPosition = position + cameraOffset; const float cameraHeight = cameraPosition.z(); ``` +### Operating on one complete register + +Use `NativeRegister` when the register width may follow the compile target: + +```cpp +#include + +using FloatRegister = SimdLib::NativeRegister; + +const FloatRegister values = FloatRegister::broadcast(3.0F); +const FloatRegister scale = FloatRegister::broadcast(2.0F); +const FloatRegister offset = FloatRegister::broadcast(1.0F); +const FloatRegister transformed = values * scale + offset; +``` + +`NativeRegister` resolves to 128 bits in an SSE4.2-only translation unit and +256 bits when AVX2 is enabled. Do not store it in an ABI or exchange it across +translation units that may use incompatible ISA or SimdLib configuration +settings. Use explicit `Register` for stable storage, interfaces, and +ABI contracts. + +On platforms where SimdLib enables a vector calling convention, a non-inlined +consumer function must declare the appropriate `SIMD_FLAGS(...)` boundary mode +itself. The annotations on Register members do not propagate to a surrounding +function: + +```cpp +using StableFloatRegister = SimdLib::Register; + +/** + * @brief Applies a consumer-defined complete-register transformation. + * @param value Input register. + * @return Transformed register. + */ +StableFloatRegister SIMD_FLAGS(InOut) add_one(StableFloatRegister value) noexcept +{ + return value + StableFloatRegister::broadcast(1.0F); +} +``` + +### Declaring SIMD function contracts + +Place `SIMD_FLAGS(...)` after the return type and immediately before the +function name. Every declaration starts with exactly one boundary mode: + +| Mode | Promise | Invocation | +|---|---|---| +| `Neither` | No native SIMD value, `Register`, or `RegisterMask` crosses the boundary by value | `SIMD_FLAGS(Neither)` | +| `In` | At least one SIMD value enters by value, and no SIMD value is returned by value | `SIMD_FLAGS(In)` | +| `Out` | A SIMD value is returned by value, and none enters by value | `SIMD_FLAGS(Out)` | +| `InOut` | SIMD values both enter and leave by value | `SIMD_FLAGS(InOut)` | + +The optional modifiers follow in the fixed order `RegisterOnly`, `ForceInline`, +then `Flatten`: + +- `SIMD_FLAGS(InOut, RegisterOnly)` promises that every runtime path performs only input reads and + register/scalar computation, with no authored write to addressable memory. +- `SIMD_FLAGS(InOut, ForceInline)` requests that the annotated function be incorporated into its + caller. +- `SIMD_FLAGS(InOut, Flatten)` requests recursive inlining of eligible calls made by the annotated + function. It does not request that the function itself be inlined into its + caller. + +For example, a reviewed header-defined register transform may use all three: + +```cpp +/** + * @brief Adds one to every lane without writing addressable memory. + * @param value Input register. + * @return Transformed register. + */ +StableFloatRegister +SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) +add_one_inline(StableFloatRegister value) noexcept +{ + return value + StableFloatRegister::broadcast(1.0F); +} +``` + +The flags are developer promises, not inferred properties. Do not apply +`RegisterOnly` to stores, writable spans, output pointers or references, +addressable-buffer algorithms, or functions with unreviewed transitive calls. +Declaration and definition flag lists must be ABI-compatible, and translation +units that exchange flagged functions must agree on the vectorcall +configuration. `Out` requests the configured calling convention but cannot +override a platform ABI that uses hidden return storage. + +See the [SIMD method-flag contract](docs/MethodFlagsContract.md) for declaration +forms, compiler mappings, unsupported categories, custom-toolchain adapters, +and the qualification required when adding another flag. + +### Runtime controls for immediate-mode operations + +Unsuffixed operations use compile-time controls or genuinely native runtime controls such as selector and mask registers. A method ending in `_slow` is the explicit runtime-scalar substitute for an immediate-controlled instruction and may require dispatch, branching, or a longer synthesized sequence. See [Runtime controls for immediate-mode operations](docs/ImmediateControlRuntimeNaming.md) for the complete naming and availability inventory. + +### Working with RegisterMask + +Comparisons create `RegisterMask` values. Masks can be combined with +`&`, `|`, `^`, and `~`; reduced with `any()`, `all()`, or `none()`; observed as +compact lane bits with `bits()` or as a by-value native predicate through the +public `native` member; and applied with `select()`: + +```cpp +#include + +using IntRegister = SimdLib::Register; + +const IntRegister values = IntRegister::from_lanes(-2, 0, 4, 9); +const auto positive = values.compare_greater(IntRegister::zero()); +const auto not_nine = ~values.compare_equal(IntRegister::broadcast(9)); +const auto selected_lanes = positive & not_nine; +const auto compact_bits = selected_lanes.bits(); +const auto observed_native = selected_lanes.native; +const IntRegister selected = + selected_lanes.select(values, IntRegister::zero()); +``` + +Floating comparisons use the selected hardware intrinsic's ordered semantics. +A NaN lane is false for the five named comparisons, including +`compare_equal`; positive and negative zero compare equal. Predicate lanes +retain their native all-zero or all-one bit patterns. + ### Transforming a collection `Api::transform` applies a register operation across an entire span, including @@ -112,9 +248,56 @@ FloatApi::transform( // Each value is now clamp(localHeight * 0.02F + 64.0F, -500.0F, 8'000.0F). ``` -The executable [API example](examples/ApiExamples.cpp) shows the register -facade, vectors, algorithms, bit helpers, `uint128_t`, resampling, and -formatting together in one short program. +The executable [Register example](examples/RegisterExamples.cpp) demonstrates +the preferred C++23 complete-register and mask workflows. The separate +[API example](examples/ApiExamples.cpp) demonstrates the supported C++20 +facade, vectors, collection algorithms, bit helpers, `uint128_t`, resampling, +and formatting. + +## MSVC stack-cookie behavior + +> [!WARNING] +> [MSVC's default `/GS` heuristic](https://learn.microsoft.com/en-us/cpp/build/reference/gs-buffer-security-check?view=msvc-170) +> treats any pointer-free data structure larger than eight bytes as a +> security-sensitive buffer. Consequently, a non-inlined +> function that creates or accepts `Register` by value may receive a +> security-cookie prologue and epilogue even when `__vectorcall` transports the +> value entirely in SIMD registers. This is compiler-generated overhead, not a +> spill required by the `Register` representation. + +SimdLib marks narrowly audited functions with the `RegisterOnly` modifier when +their runtime path cannot write through pointers, references, spans, arrays, +or addressable local buffers. On MSVC, `SIMD_FLAGS(..., RegisterOnly, ...)` +expands to +[`__declspec(safebuffers)`](https://learn.microsoft.com/en-us/cpp/cpp/safebuffers?view=msvc-170) +and the attribute mapping is empty on other compilers. `RegisterOnly` remains +independent from the `In`, `Out`, and `InOut` boundary modes: stores, +transforms, dynamic array-backed fallbacks, and other memory-writing functions +retain normal `/GS` protection. This is a strong developer promise used to +justify suppressing `/GS` for that function, not a compiler-verified guarantee +that the function cannot spill or otherwise use the stack. + +The operational methods in the `Api`, `Register`, `RegisterMask`, and legacy +`SimdVector` facades use the `Flatten` modifier to make their transitive-inlining +intent explicit. The mapping facades do the same for paths inherited directly +by `Api`. Flattening is an optimization request rather than proof of generated +code, so the mandatory codegen gates still compare wrapper and raw-intrinsic +objects. + +Consumer-defined, non-inlined functions can therefore still encounter this +MSVC behavior. Keep `/GS` enabled globally. Only after reviewing an individual +hot function and its generated code should a consumer consider applying +`__declspec(safebuffers)` to that function; the annotation disables `/GS` +protection for the entire annotated function. + +The mandatory MSVC generated-code gates compare SSE4.2 and AVX2 wrapper objects +with raw-intrinsic mirrors. SSE4.2 is an optimized diagnostic profile; AVX2 is +the strict zero-overhead profile. The pure register-only AVX2 subset permits no +cookie exception. Its sole optimized exception is the exact 128-bit +`Register::from_array` `/GS` sequence. The SSE4.2 diagnostic recognizes +the corresponding legacy-instruction cookie sequence so the remainder stays +comparable. Store, transfer, mutating-reference, opaque-call, and array-return +fixtures retain normal `/GS` protection and paired disassembly for review. ## Learn more @@ -123,8 +306,8 @@ formatting together in one short program. configuration details, formatting, and development commands. - [Public namespace and compatibility](docs/PublicNamespace.md) describes the supported API boundary. -- [Validation record](docs/Validation.md) documents the compiler, sanitizer, - consumer, and test evidence. +- [Build and validation](docs/BuildPipeline.md) documents the supported build, + test, compiler-matrix, and reporting commands. ## License diff --git a/benchmarks/SimdLib.benchmarks.cpp b/benchmarks/Core.benchmarks.cpp similarity index 100% rename from benchmarks/SimdLib.benchmarks.cpp rename to benchmarks/Core.benchmarks.cpp diff --git a/benchmarks/Register.benchmarks.cpp b/benchmarks/Register.benchmarks.cpp new file mode 100644 index 0000000..fad5655 --- /dev/null +++ b/benchmarks/Register.benchmarks.cpp @@ -0,0 +1,143 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace +{ + +/** @brief Returns a process-local runtime seed that prevents compile-time operand folding. */ +[[nodiscard]] std::uint64_t runtime_seed() noexcept +{ + return static_cast(std::chrono::steady_clock::now().time_since_epoch().count()) | std::uint64_t{1}; +} + +/** + * @brief Generates runtime-derived floating operands for one complete register. + * @tparam count Number of generated lanes. + * @param state Mutable pseudo-random state. + * @return Complete floating lane array whose values are finite and nonzero. + */ +template [[nodiscard]] std::array make_float_lanes(std::uint64_t &state) noexcept +{ + std::array result{}; + for (std::size_t lane = 0; lane < count; ++lane) + { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + result[lane] = static_cast((state & 0x3ffU) + 1U) / 37.0F; + } + return result; +} + +/** + * @brief Generates runtime-derived nonzero unsigned divisors for one complete register. + * @tparam count Number of generated lanes. + * @param state Mutable pseudo-random state. + * @return Complete unsigned lane array containing values in the range one through 31. + */ +template [[nodiscard]] std::array make_unsigned_lanes(std::uint64_t &state) noexcept +{ + std::array result{}; + for (std::size_t lane = 0; lane < count; ++lane) + { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + result[lane] = static_cast((state & 0x1fU) + 1U); + } + return result; +} + +} // namespace + +TEST_CASE("Register runtime-derived wrapper and raw benchmarks", "[simdlib][benchmark][register]") +{ + using register128 = SimdLib::Register; + using register256 = SimdLib::Register; + using api128 = typename register128::api_type; + using api256 = typename register256::api_type; + using integer_register128 = SimdLib::Register; + using integer_register256 = SimdLib::Register; + using integer_api128 = typename integer_register128::api_type; + using integer_api256 = typename integer_register256::api_type; + + auto seed = runtime_seed(); + const auto lhs128_lanes = make_float_lanes(seed); + const auto rhs128_lanes = make_float_lanes(seed); + const auto lhs256_lanes = make_float_lanes(seed); + const auto rhs256_lanes = make_float_lanes(seed); + const auto integer_lhs128_lanes = make_unsigned_lanes(seed); + const auto integer_rhs128_lanes = make_unsigned_lanes(seed); + const auto integer_lhs256_lanes = make_unsigned_lanes(seed); + const auto integer_rhs256_lanes = make_unsigned_lanes(seed); + + const auto lhs128 = register128::load(std::span{lhs128_lanes}); + const auto rhs128 = register128::load(std::span{rhs128_lanes}); + const auto lhs256 = register256::load(std::span{lhs256_lanes}); + const auto rhs256 = register256::load(std::span{rhs256_lanes}); + const auto integer_lhs128 = integer_register128::load(std::span{integer_lhs128_lanes}); + const auto integer_rhs128 = integer_register128::load(std::span{integer_rhs128_lanes}); + const auto integer_lhs256 = integer_register256::load(std::span{integer_lhs256_lanes}); + const auto integer_rhs256 = integer_register256::load(std::span{integer_rhs256_lanes}); + const auto mask128 = lhs128.compare_greater(rhs128); + const auto mask256 = lhs256.compare_greater(rhs256); + const auto raw_mask128 = api128::compare_greater(lhs128.native, rhs128.native); + const auto raw_mask256 = api256::compare_greater(lhs256.native, rhs256.native); + + BENCHMARK("Register 128-bit float add") + { + return (lhs128 + rhs128).native; + }; + BENCHMARK("Raw Api 128-bit float add") + { + return api128::add(lhs128.native, rhs128.native); + }; + BENCHMARK("Register 256-bit float add") + { + return (lhs256 + rhs256).native; + }; + BENCHMARK("Raw Api 256-bit float add") + { + return api256::add(lhs256.native, rhs256.native); + }; + BENCHMARK("Register 128-bit mask select") + { + return mask128.select(lhs128, rhs128).native; + }; + BENCHMARK("Raw Api 128-bit mask select") + { + return api128::select(raw_mask128, lhs128.native, rhs128.native); + }; + BENCHMARK("Register 256-bit mask select") + { + return mask256.select(lhs256, rhs256).native; + }; + BENCHMARK("Raw Api 256-bit mask select") + { + return api256::select(raw_mask256, lhs256.native, rhs256.native); + }; + BENCHMARK("Register 128-bit unsigned division") + { + return (integer_lhs128 / integer_rhs128).native; + }; + BENCHMARK("Raw Api 128-bit unsigned division") + { + return integer_api128::divide(integer_lhs128.native, integer_rhs128.native); + }; + BENCHMARK("Register 256-bit unsigned division") + { + return (integer_lhs256 / integer_rhs256).native; + }; + BENCHMARK("Raw Api 256-bit unsigned division") + { + return integer_api256::divide(integer_lhs256.native, integer_rhs256.native); + }; +} diff --git a/cmake/AuditPublicHeaderAssertions.cmake b/cmake/AuditPublicHeaderAssertions.cmake deleted file mode 100644 index 4cf588c..0000000 --- a/cmake/AuditPublicHeaderAssertions.cmake +++ /dev/null @@ -1,62 +0,0 @@ -cmake_policy(VERSION 3.20) - -if(NOT DEFINED SOURCE_DIRECTORY) - message(FATAL_ERROR "SOURCE_DIRECTORY is required") -endif() - -set(allowlist_file "${SOURCE_DIRECTORY}/cmake/PublicHeaderStaticAssertAllowlist.txt") -file(STRINGS "${allowlist_file}" allowlist_entries) -list(FILTER allowlist_entries EXCLUDE REGEX "^[ \\t]*(#|$)") -list(LENGTH allowlist_entries allowlist_count) -if(allowlist_count EQUAL 0) - message(FATAL_ERROR "The public-header static_assert allowlist is empty") -endif() - -math(EXPR allowlist_last "${allowlist_count} - 1") -foreach(index RANGE 0 ${allowlist_last}) - set(allowlist_used_${index} FALSE) -endforeach() - -file(GLOB_RECURSE public_headers "${SOURCE_DIRECTORY}/include/SimdLib/*.h") -set(assertion_count 0) -foreach(header IN LISTS public_headers) - get_filename_component(header_name "${header}" NAME) - file(READ "${header}" header_text) - string(REPLACE ";" "" header_text "${header_text}") - string(REPLACE "\r\n" "\n" header_text "${header_text}") - string(REGEX MATCHALL "static_assert[^\n]*" assertion_contexts "${header_text}") - foreach(context IN LISTS assertion_contexts) - string(REPLACE "" ";" context "${context}") - math(EXPR assertion_count "${assertion_count} + 1") - set(matched FALSE) - foreach(allowlist_index RANGE 0 ${allowlist_last}) - list(GET allowlist_entries ${allowlist_index} entry) - string(REPLACE "|" ";" fields "${entry}") - list(LENGTH fields field_count) - if(NOT field_count EQUAL 3) - message(FATAL_ERROR "Malformed static_assert allowlist entry: ${entry}") - endif() - list(GET fields 0 allowed_header) - list(GET fields 1 allowed_substring) - if(header_name STREQUAL allowed_header) - string(FIND "${context}" "${allowed_substring}" match_position) - if(NOT match_position EQUAL -1) - set(matched TRUE) - set(allowlist_used_${allowlist_index} TRUE) - break() - endif() - endif() - endforeach() - if(NOT matched) - message(FATAL_ERROR "Unallowlisted static_assert in ${header}: ${context}") - endif() - endforeach() -endforeach() -foreach(index RANGE 0 ${allowlist_last}) - if(NOT allowlist_used_${index}) - list(GET allowlist_entries ${index} unused_entry) - message(FATAL_ERROR "Stale static_assert allowlist entry: ${unused_entry}") - endif() -endforeach() - -message(STATUS "Validated ${assertion_count} production-header static_assert occurrences against ${allowlist_count} justified allowlist entries") \ No newline at end of file diff --git a/cmake/AuditValidationInventory.cmake b/cmake/AuditValidationInventory.cmake new file mode 100644 index 0000000..5a0bc0d --- /dev/null +++ b/cmake/AuditValidationInventory.cmake @@ -0,0 +1,179 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + MATRIX_FILE CELL_ID BUILD_DIRECTORY CMAKE_CTEST_COMMAND RESULT_FILE) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR + "AuditValidationInventory requires ${required_variable}") + endif() +endforeach() + +# @brief Reads one JSON array into a CMake list. +# @param output_variable Variable that receives the array values. +# @param json_document JSON document. +# @param path Remaining arguments that identify the array. +function(simdlib_read_json_array output_variable json_document) + string(JSON item_count LENGTH "${json_document}" ${ARGN}) + set(items "") + if(item_count GREATER 0) + math(EXPR last_index "${item_count} - 1") + foreach(item_index RANGE 0 ${last_index}) + string(JSON item GET "${json_document}" ${ARGN} ${item_index}) + list(APPEND items "${item}") + endforeach() + endif() + set(${output_variable} "${items}" PARENT_SCOPE) +endfunction() + +file(READ "${MATRIX_FILE}" matrix_json) +string(JSON matrix_schema GET "${matrix_json}" schema) +if(NOT matrix_schema STREQUAL "simdlib.validation-matrix.v1") + message(FATAL_ERROR "Validation matrix has an unsupported schema") +endif() +string(JSON cell_type ERROR_VARIABLE cell_error + TYPE "${matrix_json}" cells "${CELL_ID}") +if(cell_error OR NOT cell_type STREQUAL "OBJECT") + message(FATAL_ERROR "Validation matrix does not define cell ${CELL_ID}") +endif() +string(JSON profile GET "${matrix_json}" cells "${CELL_ID}" profile) +simdlib_read_json_array(allowed_target_categories "${matrix_json}" + profiles "${profile}" allowedTargetCategories) +simdlib_read_json_array(selected_target_categories "${matrix_json}" + profiles "${profile}" selectedTargetCategories) +simdlib_read_json_array(allowed_test_owners "${matrix_json}" + profiles "${profile}" allowedTestOwners) + +set(target_ownership_file + "${BUILD_DIRECTORY}/development-target-ownership.tsv") +if(NOT EXISTS "${target_ownership_file}") + message(FATAL_ERROR + "Generated target ownership inventory is missing: " + "${target_ownership_file}") +endif() +file(STRINGS "${target_ownership_file}" target_rows) +list(POP_FRONT target_rows target_header) +if(NOT target_header STREQUAL + "target\tcategory\towning_aggregate\tselected") + message(FATAL_ERROR + "Generated target ownership inventory has an invalid header") +endif() + +set(target_names "") +set(selected_target_count 0) +foreach(target_row IN LISTS target_rows) + if(NOT target_row MATCHES "^([^\t]+)\t([^\t]+)\t([^\t]+)\t(YES|NO)$") + message(FATAL_ERROR + "Unowned or malformed target inventory row: ${target_row}") + endif() + set(target_name "${CMAKE_MATCH_1}") + set(target_category "${CMAKE_MATCH_2}") + set(target_selected "${CMAKE_MATCH_4}") + if(target_name IN_LIST target_names) + message(FATAL_ERROR + "Duplicate target ownership entry: ${target_name}") + endif() + list(APPEND target_names "${target_name}") + if(NOT target_category IN_LIST allowed_target_categories) + message(FATAL_ERROR + "Target ${target_name} has unexpected profile membership " + "${target_category} in ${profile}") + endif() + if(target_category IN_LIST selected_target_categories) + if(NOT target_selected STREQUAL "YES") + message(FATAL_ERROR + "Target ${target_name} is omitted from its owning profile") + endif() + math(EXPR selected_target_count "${selected_target_count} + 1") + elseif(NOT target_selected STREQUAL "NO") + message(FATAL_ERROR + "Target ${target_name} is selected from non-default category " + "${target_category}") + endif() +endforeach() +list(LENGTH target_names target_count) + +if(DEFINED TEST_JSON_FILE AND NOT "${TEST_JSON_FILE}" STREQUAL "") + file(READ "${TEST_JSON_FILE}" ctest_json) +else() + set(ctest_arguments + --test-dir "${BUILD_DIRECTORY}" --show-only=json-v1) + if(DEFINED CONFIGURATION AND NOT "${CONFIGURATION}" STREQUAL "") + list(APPEND ctest_arguments -C "${CONFIGURATION}") + endif() + execute_process( + COMMAND "${CMAKE_CTEST_COMMAND}" ${ctest_arguments} + RESULT_VARIABLE ctest_result + OUTPUT_VARIABLE ctest_json + ERROR_VARIABLE ctest_error) + if(NOT ctest_result EQUAL 0) + message(FATAL_ERROR + "Unable to enumerate CTest ownership in ${BUILD_DIRECTORY}: " + "${ctest_error}") + endif() +endif() + +string(JSON ctest_schema_major GET "${ctest_json}" version major) +if(NOT ctest_schema_major EQUAL 1) + message(FATAL_ERROR "CTest inventory has an unsupported JSON schema") +endif() +string(JSON test_count LENGTH "${ctest_json}" tests) +set(test_names "") +set(test_index 0) +while(test_index LESS test_count) + string(JSON test_name GET "${ctest_json}" tests ${test_index} name) + if(test_name IN_LIST test_names) + message(FATAL_ERROR "Duplicate CTest identity: ${test_name}") + endif() + list(APPEND test_names "${test_name}") + + set(test_owner_labels "") + string(JSON property_count LENGTH + "${ctest_json}" tests ${test_index} properties) + set(property_index 0) + while(property_index LESS property_count) + string(JSON property_name GET + "${ctest_json}" tests ${test_index} properties + ${property_index} name) + if(property_name STREQUAL "LABELS") + simdlib_read_json_array(test_labels "${ctest_json}" + tests ${test_index} properties ${property_index} value) + foreach(test_label IN LISTS test_labels) + if(test_label MATCHES "^SIMDLIB_OWNER_(.+)$") + list(APPEND test_owner_labels "${CMAKE_MATCH_1}") + endif() + endforeach() + endif() + math(EXPR property_index "${property_index} + 1") + endwhile() + list(REMOVE_DUPLICATES test_owner_labels) + list(LENGTH test_owner_labels test_owner_count) + if(NOT test_owner_count EQUAL 1) + message(FATAL_ERROR + "CTest ${test_name} has ${test_owner_count} validation owners") + endif() + list(GET test_owner_labels 0 test_owner) + if(NOT test_owner IN_LIST allowed_test_owners) + message(FATAL_ERROR + "CTest ${test_name} has unexpected profile membership " + "${test_owner} in ${profile}") + endif() + math(EXPR test_index "${test_index} + 1") +endwhile() + +get_filename_component(result_directory "${RESULT_FILE}" DIRECTORY) +file(MAKE_DIRECTORY "${result_directory}") +file(WRITE "${RESULT_FILE}" + "{\n" + " \"schema\": \"simdlib.validation-inventory-audit.v1\",\n" + " \"status\": \"complete\",\n" + " \"cell\": \"${CELL_ID}\",\n" + " \"profile\": \"${profile}\",\n" + " \"targets\": ${target_count},\n" + " \"selectedTargets\": ${selected_target_count},\n" + " \"tests\": ${test_count}\n" + "}\n") + +message(STATUS + "Validation inventory audit passed for ${CELL_ID}: " + "${target_count} owned targets, ${selected_target_count} selected targets, " + "${test_count} owned tests") diff --git a/cmake/CheckPublicConsumerBoundary.cmake b/cmake/CheckPublicConsumerBoundary.cmake new file mode 100644 index 0000000..0d49f5b --- /dev/null +++ b/cmake/CheckPublicConsumerBoundary.cmake @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.31) + +if(NOT DEFINED SOURCE_DIRECTORY OR "${SOURCE_DIRECTORY}" STREQUAL "") + message(FATAL_ERROR "SOURCE_DIRECTORY is required") +endif() + +file(GLOB_RECURSE public_consumer_sources + "${SOURCE_DIRECTORY}/examples/*.cpp" + "${SOURCE_DIRECTORY}/tests/consumer/*.cpp" + "${SOURCE_DIRECTORY}/tests/format_odr/*.cpp" + "${SOURCE_DIRECTORY}/tests/headers/*.cpp" + "${SOURCE_DIRECTORY}/tests/register_odr/*.cpp" + "${SOURCE_DIRECTORY}/tests/smoke/*.cpp") +list(SORT public_consumer_sources) +foreach(consumer_source IN LISTS public_consumer_sources) + file(READ "${consumer_source}" consumer_source_text) + if(consumer_source_text MATCHES "SimdLib::Detail|: +subq $0x28, %rsp +movq (%rip), %rax # 0x +xorq %rsp, %rax +movq %rax, 0x10(%rsp) +movq (%rcx), %rax +movq %rax, (%rsp) +movq 0x8(%rcx), %rax +movq %rax, 0x8(%rsp) +vmovdqu (%rsp), %vreg +movq 0x10(%rsp), %rcx +xorq %rsp, %rcx +callq 0x +addq $0x28, %rsp +retq]=]) + set(raw_profile [=[: +subq $0x18, %rsp +movq (%rcx), %rax +movq %rax, (%rsp) +movq 0x8(%rcx), %rax +movq %rax, 0x8(%rsp) +vmovdqu (%rsp), %vreg +addq $0x18, %rsp +retq]=]) + if(ISA_PROFILE STREQUAL "SSE42") + string(REPLACE "vmovdqu" "movdqu" cookie_profile "${cookie_profile}") + string(REPLACE "vmovdqu" "movdqu" raw_profile "${raw_profile}") + endif() + string(FIND "${input_text}" "${cookie_profile}" cookie_index) + if(cookie_index LESS 0) + return() + endif() + string(LENGTH "${cookie_profile}" cookie_length) + math(EXPR cookie_tail_index "${cookie_index} + ${cookie_length}") + string(SUBSTRING "${input_text}" ${cookie_tail_index} -1 cookie_tail) + string(FIND "${cookie_tail}" "${cookie_profile}" second_cookie_relative_index) + if(second_cookie_relative_index LESS 0) + return() + endif() + math(EXPR second_cookie_index "${cookie_tail_index} + ${second_cookie_relative_index}") + math(EXPR second_cookie_tail_index "${second_cookie_index} + ${cookie_length}") + string(SUBSTRING "${input_text}" 0 ${second_cookie_index} comparable_prefix) + string(SUBSTRING "${input_text}" ${second_cookie_tail_index} -1 comparable_suffix) + set(comparable_profile "${comparable_prefix}${raw_profile}${comparable_suffix}") + set(${output_variable} "${comparable_profile}" PARENT_SCOPE) + set(${accepted_variable} ON PARENT_SCOPE) +endfunction() + +# @brief Removes object identity, instruction addresses, and encoded bytes while retaining instructions. +# @param input_text Raw object disassembly. +# @param output_variable Variable that receives normalized disassembly. +function(simdlib_normalize_disassembly input_text output_variable) + set(normalized "${input_text}") + string(REPLACE "\r\n" "\n" normalized "${normalized}") + string(REPLACE "\n" ";" disassembly_lines "${normalized}") + set(fixture_only "") + set(in_fixture OFF) + foreach(disassembly_line IN LISTS disassembly_lines) + if(disassembly_line MATCHES "<[^>]*${SYMBOL_PATTERN}[^>]*>:") + if(EXCLUDE_SYMBOL_PATTERN STREQUAL "" OR NOT disassembly_line MATCHES "<[^>]*${EXCLUDE_SYMBOL_PATTERN}[^>]*>:") + set(in_fixture ON) + string(APPEND fixture_only ":\n") + else() + set(in_fixture OFF) + endif() + elseif(disassembly_line MATCHES "^[ \t]*[0-9A-Fa-f]+[ \t]+<[^>]+>:") + set(in_fixture OFF) + elseif(in_fixture AND NOT disassembly_line MATCHES "^Disassembly of section") + string(APPEND fixture_only "${disassembly_line}\n") + if(disassembly_line MATCHES "[ \t]ret[qwl]?([ \t]|$)") + set(in_fixture OFF) + endif() + endif() + endforeach() + set(normalized "${fixture_only}") + string(REGEX REPLACE "[^\n]*file format[^\n]*\n" "" normalized "${normalized}") + string(REGEX REPLACE "(^|\n)[ \t]*[0-9A-Fa-f]+[ \t]+<" "\\1<" normalized "${normalized}") + string(REGEX REPLACE "(^|\n)[ \t]*[0-9A-Fa-f]+:[ \t]+([0-9A-Fa-f][0-9A-Fa-f][ \t]+)+" "\\1" normalized "${normalized}") + string(REGEX REPLACE "<[^>]+>" "" normalized "${normalized}") + string(REGEX REPLACE "[0-9A-Fa-f]+[ \t]+" "" normalized "${normalized}") + string(REGEX REPLACE "[ \t]+\n" "\n" normalized "${normalized}") + string(REGEX REPLACE "\n+" "\n" normalized "${normalized}") + string(STRIP "${normalized}" normalized) + set(${output_variable} "${normalized}" PARENT_SCOPE) +endfunction() + +# @brief Removes allocator-selected vector-register identities, including names repeated in disassembler comments. +# @param input_text Normalized fixture disassembly. +# @param output_variable Variable that receives the allocation-independent instruction profile. +function(simdlib_profile_disassembly input_text output_variable) + set(profile "${input_text}") + string(REGEX REPLACE "%[xyz]mm[0-9]+" "%vreg" profile "${profile}") + string(REGEX REPLACE "[xyz]mm[0-9]+" "vreg" profile "${profile}") + set(${output_variable} "${profile}" PARENT_SCOPE) +endfunction() + +# @brief Removes the one accepted MSVC scalar-result security-cookie sequence. +# @param input_text Allocation-independent wrapper instruction profile. +# @param output_variable Variable that receives the comparable wrapper profile. +# @param accepted_variable Variable that reports whether the exact exception was found. +function(simdlib_accept_msvc_scalar_cookie input_text output_variable accepted_variable) + set(${output_variable} "${input_text}" PARENT_SCOPE) + set(${accepted_variable} OFF PARENT_SCOPE) + if(NOT COMPILER_ID STREQUAL "MSVC" OR + NOT SYSTEM_NAME STREQUAL "Windows" OR + NOT VECTORCALL_ENABLED STREQUAL "1" OR + NOT SYMBOL_PATTERN STREQUAL "simdlib_codegen_") + return() + endif() + + if(REGISTER_WIDTH STREQUAL "128") + set(cookie_profile [=[: +subq $0x18, %rsp +movq (%rip), %rax # 0x +xorq %rsp, %rax +movq %rax, (%rsp) +vpmovmskb %vreg, %eax +movq (%rsp), %rcx +xorq %rsp, %rcx +callq 0x +addq $0x18, %rsp +retq]=]) + set(raw_scalar_profile [=[: +vpmovmskb %vreg, %eax +retq]=]) + elseif(REGISTER_WIDTH STREQUAL "256") + set(cookie_profile [=[: +subq $0x18, %rsp +movq (%rip), %rax # 0x +xorq %rsp, %rax +movq %rax, (%rsp) +vpmovmskb %vreg, %eax +vzeroupper +movq (%rsp), %rcx +xorq %rsp, %rcx +callq 0x +addq $0x18, %rsp +retq]=]) + set(raw_scalar_profile [=[: +vpmovmskb %vreg, %eax +vzeroupper +retq]=]) + else() + return() + endif() + + string(FIND "${input_text}" "${cookie_profile}" cookie_index) + if(cookie_index LESS 0) + return() + endif() + string(LENGTH "${cookie_profile}" cookie_length) + math(EXPR cookie_tail_index "${cookie_index} + ${cookie_length}") + string(SUBSTRING "${input_text}" ${cookie_tail_index} -1 cookie_tail) + string(FIND "${cookie_tail}" "${cookie_profile}" duplicate_cookie_index) + if(NOT duplicate_cookie_index LESS 0) + return() + endif() + + string(REPLACE "${cookie_profile}" "${raw_scalar_profile}" comparable_profile "${input_text}") + set(${output_variable} "${comparable_profile}" PARENT_SCOPE) + set(${accepted_variable} ON PARENT_SCOPE) +endfunction() + +#[[ +The MSVC compound-assignment exception is disabled with the compound-assignment +API. Reassignment avoids the mutable wrapper reference that triggers the +redundant security-cookie and 32-byte stack-alignment frame, so its codegen gate +requires exact parity. The former exception remains here for diagnostic history. +# @brief Removes the one accepted MSVC compound-assignment security-cookie sequence. +# @param input_text Allocation-independent wrapper instruction profile. +# @param output_variable Variable that receives the comparable wrapper profile. +# @param accepted_variable Variable that reports whether the exact exception was found. +function(simdlib_accept_msvc_compound_cookie input_text output_variable accepted_variable) + set(${output_variable} "${input_text}" PARENT_SCOPE) + set(${accepted_variable} OFF PARENT_SCOPE) + if(NOT COMPILER_ID STREQUAL "MSVC" OR + NOT SYSTEM_NAME STREQUAL "Windows" OR + NOT VECTORCALL_ENABLED STREQUAL "1" OR + NOT SYMBOL_PATTERN STREQUAL "simdlib_codegen_compound_arithmetic") + return() + endif() + + if(REGISTER_WIDTH STREQUAL "128") + set(cookie_profile [=[: +subq $0x18, %rsp +movq (%rip), %rax # 0x +xorq %rsp, %rax +movq %rax, (%rsp) +vaddps %vreg, %vreg, %vreg +vmulps %vreg, %vreg, %vreg +movq (%rsp), %rcx +xorq %rsp, %rcx +callq 0x +addq $0x18, %rsp +retq]=]) + elseif(REGISTER_WIDTH STREQUAL "256") + set(cookie_profile [=[: +pushq %rbp +subq $0x30, %rsp +leaq 0x20(%rsp), %rbp +andq $-0x20, %rbp +movq (%rip), %rax # 0x +xorq %rsp, %rax +movq %rax, (%rbp) +vaddps %vreg, %vreg, %vreg +vmulps %vreg, %vreg, %vreg +movq (%rbp), %rcx +xorq %rsp, %rcx +callq 0x +addq $0x30, %rsp +popq %rbp +retq]=]) + else() + return() + endif() + set(raw_profile [=[: +vaddps %vreg, %vreg, %vreg +vmulps %vreg, %vreg, %vreg +retq]=]) + if(input_text STREQUAL cookie_profile) + set(${output_variable} "${raw_profile}" PARENT_SCOPE) + set(${accepted_variable} ON PARENT_SCOPE) + endif() +endfunction() +]] + +simdlib_disassemble("${WRAPPER_OBJECT}" wrapper_disassembly) +simdlib_disassemble("${RAW_OBJECT}" raw_disassembly) +simdlib_normalize_disassembly("${wrapper_disassembly}" wrapper_normalized) +simdlib_normalize_disassembly("${raw_disassembly}" raw_normalized) +simdlib_profile_disassembly("${wrapper_normalized}" wrapper_profile) +simdlib_profile_disassembly("${raw_normalized}" raw_profile) + +string(FIND "${wrapper_profile}" "vfmadd" wrapper_fma_index) +string(FIND "${raw_profile}" "vfmadd" raw_fma_index) +if(NOT RECORD_ONLY AND FMA_EXPECTATION STREQUAL "enabled" AND (wrapper_fma_index LESS 0 OR raw_fma_index LESS 0)) + message(FATAL_ERROR "The FMA-enabled generated-code profile does not contain fused multiply-add instructions") +elseif(NOT RECORD_ONLY AND FMA_EXPECTATION STREQUAL "disabled" AND (NOT wrapper_fma_index LESS 0 OR NOT raw_fma_index LESS 0)) + message(FATAL_ERROR "The FMA-disabled generated-code profile unexpectedly contains fused multiply-add instructions") +endif() + +set(comparable_wrapper_profile "${wrapper_profile}") +set(comparison_result "exact-parity") +set(accepted_exception "none") +if(NOT wrapper_profile STREQUAL raw_profile) + simdlib_accept_msvc_scalar_cookie( + "${wrapper_profile}" comparable_wrapper_profile accepted_msvc_scalar_cookie) + if(accepted_msvc_scalar_cookie AND comparable_wrapper_profile STREQUAL raw_profile) + set(comparison_result "accepted-compiler-exception") + set(accepted_exception "msvc-gs-scalar-cookie") + else() + simdlib_accept_msvc_from_array_cookie( + "${wrapper_profile}" comparable_wrapper_profile accepted_msvc_from_array_cookie) + if(accepted_msvc_from_array_cookie AND comparable_wrapper_profile STREQUAL raw_profile) + set(comparison_result "accepted-compiler-exception") + set(accepted_exception "msvc-gs-from-array-cookie") + else() + set(comparison_result "failed") + endif() + #[[ + The compound-assignment exception branch is disabled with the public + compound-assignment API. Reassignment must satisfy exact parity. + simdlib_accept_msvc_compound_cookie( + "${wrapper_profile}" comparable_wrapper_profile accepted_msvc_compound_cookie) + if(accepted_msvc_compound_cookie AND comparable_wrapper_profile STREQUAL raw_profile) + set(comparison_result "accepted-compiler-exception") + set(accepted_exception "msvc-gs-compound-cookie") + else() + set(comparison_result "failed") + endif() + ]] + endif() +endif() + +if(RECORD_ONLY AND comparison_result STREQUAL "failed") + set(comparison_result "recorded-difference") + set(accepted_exception "${RECORDED_DIFFERENCE_REASON}") +endif() + +file(WRITE "${ARTIFACT_DIRECTORY}/wrapper.disassembly.txt" "${wrapper_disassembly}") +file(WRITE "${ARTIFACT_DIRECTORY}/raw.disassembly.txt" "${raw_disassembly}") +file(WRITE "${ARTIFACT_DIRECTORY}/wrapper.normalized.txt" "${wrapper_normalized}\n") +file(WRITE "${ARTIFACT_DIRECTORY}/raw.normalized.txt" "${raw_normalized}\n") +file(WRITE "${ARTIFACT_DIRECTORY}/wrapper.profile.txt" "${wrapper_profile}\n") +file(WRITE "${ARTIFACT_DIRECTORY}/raw.profile.txt" "${raw_profile}\n") +file(WRITE "${ARTIFACT_DIRECTORY}/wrapper.comparable.profile.txt" "${comparable_wrapper_profile}\n") +file(WRITE "${ARTIFACT_DIRECTORY}/comparison.txt" + "result=${comparison_result}\n" + "accepted_exception=${accepted_exception}\n") +file(WRITE "${ARTIFACT_DIRECTORY}/provenance.txt" + "compiler_id=${COMPILER_ID}\n" + "compiler_version=${COMPILER_VERSION}\n" + "compiler_path=${COMPILER_PATH}\n" + "system_name=${SYSTEM_NAME}\n" + "system_processor=${SYSTEM_PROCESSOR}\n" + "configuration=${CONFIGURATION}\n" + "register_width=${REGISTER_WIDTH}\n" + "isa_profile=${ISA_PROFILE}\n" + "vectorcall_enabled=${VECTORCALL_ENABLED}\n" + "stack_protector_mode=${STACK_PROTECTOR_MODE}\n" + "codegen_profile=${CODEGEN_PROFILE}\n" + "fma_expectation=${FMA_EXPECTATION}\n" + "exclude_symbol_pattern=${EXCLUDE_SYMBOL_PATTERN}\n" + "record_only=${RECORD_ONLY}\n" + "comparison_result=${comparison_result}\n" + "accepted_exception=${accepted_exception}\n" + "wrapper_object=${WRAPPER_OBJECT}\n" + "raw_object=${RAW_OBJECT}\n") + +if(comparison_result STREQUAL "failed") + message(FATAL_ERROR + "Register wrapper generated code differs from the raw fixture; inspect ${ARTIFACT_DIRECTORY}") +endif() + +file(SHA256 "${WRAPPER_OBJECT}" wrapper_hash) +file(SHA256 "${RAW_OBJECT}" raw_hash) +file(SHA256 "${OBJDUMP}" tool_hash) +execute_process( + COMMAND "${OBJDUMP}" --version + RESULT_VARIABLE tool_version_result + OUTPUT_VARIABLE tool_version_output + ERROR_VARIABLE tool_version_error) +if(NOT tool_version_result EQUAL 0) + message(FATAL_ERROR "Unable to identify generated-code comparison tool: ${tool_version_error}") +endif() +string(REGEX REPLACE "\r?\n.*" "" tool_version "${tool_version_output}") +if(RECORD_ONLY) + set(policy_mode "RECORD") +else() + set(policy_mode "ENFORCE") +endif() +string(TIMESTAMP codegen_end_epoch "%s" UTC) +math(EXPR codegen_total_seconds + "${codegen_end_epoch} - ${codegen_start_epoch}") +foreach(json_value IN ITEMS + WRAPPER_OBJECT RAW_OBJECT OBJDUMP tool_version COMPILER_ID COMPILER_VERSION + COMPILER_PATH SYSTEM_NAME SYSTEM_PROCESSOR CONFIGURATION ISA_PROFILE + STACK_PROTECTOR_MODE CODEGEN_PROFILE FMA_EXPECTATION SYMBOL_PATTERN EXCLUDE_SYMBOL_PATTERN + comparison_result accepted_exception policy_mode) + simdlib_escape_json("${${json_value}}" "${json_value}_json") +endforeach() +file(WRITE "${record_temporary_file}" + "{\n" + " \"schema\": \"simdlib.codegen-record.v1\",\n" + " \"kind\": \"comparison\",\n" + " \"result\": \"${comparison_result_json}\",\n" + " \"accepted_exception\": \"${accepted_exception_json}\",\n" + " \"inputs\": {\n" + " \"wrapper\": {\"path\": \"${WRAPPER_OBJECT_json}\", \"sha256\": \"${wrapper_hash}\"},\n" + " \"raw\": {\"path\": \"${RAW_OBJECT_json}\", \"sha256\": \"${raw_hash}\"}\n" + " },\n" + " \"tool\": {\"path\": \"${OBJDUMP_json}\", \"version\": \"${tool_version_json}\", \"sha256\": \"${tool_hash}\"},\n" + " \"policy\": {\"id\": \"register-codegen-comparison-v1\", \"mode\": \"${policy_mode_json}\", " + "\"codegen_profile\": \"${CODEGEN_PROFILE_json}\", \"fma_expectation\": \"${FMA_EXPECTATION_json}\", " + "\"symbol_pattern\": \"${SYMBOL_PATTERN_json}\", \"exclude_symbol_pattern\": \"${EXCLUDE_SYMBOL_PATTERN_json}\"},\n" + " \"compiler\": {\"id\": \"${COMPILER_ID_json}\", \"version\": \"${COMPILER_VERSION_json}\", " + "\"path\": \"${COMPILER_PATH_json}\"},\n" + " \"platform\": {\"system\": \"${SYSTEM_NAME_json}\", \"processor\": \"${SYSTEM_PROCESSOR_json}\"},\n" + " \"configuration\": \"${CONFIGURATION_json}\",\n" + " \"timing\": {\"total_seconds\": ${codegen_total_seconds}},\n" + " \"register_width\": ${REGISTER_WIDTH},\n" + " \"isa_profile\": \"${ISA_PROFILE_json}\",\n" + " \"vectorcall_enabled\": ${VECTORCALL_ENABLED},\n" + " \"stack_protector_mode\": \"${STACK_PROTECTOR_MODE_json}\"\n" + "}\n") +file(RENAME "${record_temporary_file}" "${RECORD_FILE}") + +if(comparison_result STREQUAL "recorded-difference") + message(STATUS + "Recorded Register wrapper/raw diagnostic ${accepted_exception}; artifacts: ${ARTIFACT_DIRECTORY}") +elseif(comparison_result STREQUAL "accepted-compiler-exception") + message(STATUS + "Accepted the exact MSVC /GS security-cookie exception ${accepted_exception}; artifacts: ${ARTIFACT_DIRECTORY}") +endif() diff --git a/cmake/CompareUInt128ResultSets.cmake b/cmake/CompareUInt128ResultSets.cmake index 5cee3e7..4eab010 100644 --- a/cmake/CompareUInt128ResultSets.cmake +++ b/cmake/CompareUInt128ResultSets.cmake @@ -1,3 +1,5 @@ +cmake_minimum_required(VERSION 3.31) + if(NOT DEFINED PORTABLE_EXECUTABLE OR NOT DEFINED OPTIMIZED_EXECUTABLE) message(FATAL_ERROR "Both uint128 test executable paths are required") endif() diff --git a/cmake/CompilerConfiguration.md b/cmake/CompilerConfiguration.md index f7c62b7..46aa958 100644 --- a/cmake/CompilerConfiguration.md +++ b/cmake/CompilerConfiguration.md @@ -1,19 +1,30 @@ # Compiler configuration probes -`Config.h` owns the standalone compiler configuration surface. Every -library-controlled macro uses `#ifndef`, so a downstream project may override -it before including any SimdLib header. +`Config.h` owns the standalone compiler configuration surface. Public function +declarations use `SIMD_FLAGS(...)`; downstream toolchains customize its +placement-safe compiler adapters before including any SimdLib header. -- `VECTORCALL` is ABI-affecting. It defaults to the shared `__vectorcall` - keyword for MSVC and Clang x86/x64 targets and is empty elsewhere. A caller - that supplies an empty `VECTORCALL` also sets - `SIMDLIB_VECTORCALL_ENABLED=0`. The shared keyword preserves one declaration - shape for free functions, members, templates, and function pointers. - An empty fallback changes only the calling convention; it does not affect - `SIMDLIB_HAS_*` instruction availability. Every linked translation unit must - use the same definition to avoid an ABI mismatch. -- `SIMDLIB_FORCE_INLINE` defaults to the supported C++11 vendor attribute plus - `inline`; callers may set it to ordinary `inline`. +- `Neither`, `In`, `Out`, and `InOut` describe whether native or SimdLib SIMD + values cross the function boundary by value. `In`, `Out`, and `InOut` emit + the configured vector calling convention exactly once when the selected + compiler supports it. +- `RegisterOnly`, `ForceInline`, and `Flatten` are independent modifiers. + `RegisterOnly` maps to safe-buffer suppression only on supported Microsoft + configurations. `ForceInline` requests that the annotated function be + inlined into its caller; `Flatten` requests recursive inlining of eligible + calls made by the annotated function. +- `SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL`, + `SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS`, + `SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE`, and + `SIMDLIB_METHOD_FLAGS_HAS_FLATTEN` report adapter capabilities. The matching + `SIMDLIB_METHOD_FLAGS_VECTORCALL`, `SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS`, + `SIMDLIB_METHOD_FLAGS_FORCE_INLINE`, and `SIMDLIB_METHOD_FLAGS_FLATTEN` + adapters may be defined by a custom toolchain before the first SimdLib + include. +- Vector calling-convention configuration is ABI-affecting. Every linked + translation unit that exchanges flagged functions must use compatible + capability and adapter definitions. Empty compiler mappings do not affect + `SIMDLIB_HAS_*` instruction availability or erase the source-level promise. - `SIMDLIB_PRECONDITION(condition, message)` defaults to `assert` and is the sole standalone replacement point for runtime preconditions. - `SIMDLIB_TARGET_X86` and `SIMDLIB_TARGET_X64` report the selected compiler @@ -30,12 +41,14 @@ only when `SIMDLIB_HAS_FMA` is enabled and otherwise retain multiply-plus-add behavior. `SimdLib::is_api_available_v` exposes this compile-time availability without instantiating an unavailable backend. -Standalone tests are split and labelled `SSE42`, `AVX2`, `FMA`, and -`OPTIONAL`. Their matching `SIMDLIB_BUILD_TESTS_*` switches let CI omit runtime -families that the host CPU cannot execute. +Standalone tests are split and labelled `SSE42`, `AVX2`, `FMA`, `BMI`, and +`SCALAR`. The matching `SIMDLIB_BUILD_API_SSE42_TESTS`, +`SIMDLIB_BUILD_API_AVX2_TESTS`, `SIMDLIB_BUILD_FMA_TESTS`, and +`SIMDLIB_BUILD_BMI_TESTS` controls describe the owned artifact families. `SIMDLIB_STRICT_WARNINGS=ON` selects `/W4 /WX /permissive-` for MSVC and -clang-cl, and `-Wall -Wextra -Wpedantic -Werror` for native Clang/GCC. The policy intentionally +clang-cl on Windows, and `-Wall -Wextra -Wpedantic -Werror` for GNU-like Clang +and GCC on Linux. The policy intentionally suppresses Clang `-Wunknown-attributes` and `-Wc2y-extensions`, GCC `-Wattributes`, plus `-Wignored-attributes` on both, because public headers retain vendor attributes such as `[[msvc::flatten]]` and compiler SIMD register types can trigger @@ -49,17 +62,18 @@ macros remain the source of truth even on MSVC, where `/arch:AVX2` is used to make the intrinsic declarations available to the independently forced probes. The configuration OBJECT probes cover default declaration placement for -ordinary/static/template functions and a function pointer; caller overrides; -disabled instruction families; vendor attributes; and an explicitly forced -Clang non-x86 configuration. The default probe compiles the same -`__vectorcall` declaration shapes with MSVC and Clang. See the +ordinary, static, and template functions; callback types derived with +`decltype`; caller overrides; disabled instruction families; vendor +attributes; and an explicitly forced Clang non-x86 configuration. The method +flags probes compile the same `SIMD_FLAGS(...)` declaration shapes with MSVC +and Clang. See the [MSVC `__vectorcall` reference](https://learn.microsoft.com/en-us/cpp/cpp/vectorcall?view=msvc-170) and [Clang vectorcall reference](https://clang.llvm.org/docs/AttributeReference.html#vectorcall). The compile-only constexpr matrix builds BMI under all four feature-macro profiles, UInt128 under compiler-carry, portable-carry, and scalar profiles, and the API/vector contracts under SSE4.2, AVX2, and fully disabled profiles. -`SimdLibConstexprProbes` aggregates these targets. The production-header +`ConstexprProbes` aggregates these targets. The production-header assertion audit is a build dependency and a CTest entry; any unallowlisted assertion or stale justification fails with its header and assertion text. -See [`docs/ConstexprCompilerEvidence.md`](../docs/ConstexprCompilerEvidence.md) -for compiler-specific runtime-path evidence and measurement results. \ No newline at end of file +The durable target/profile ownership and compiler-specific runtime-path +assignments are recorded in [`docs/TestCoverage.md`](../docs/TestCoverage.md). diff --git a/cmake/GenerateCoverageReport.cmake b/cmake/GenerateCoverageReport.cmake index ece5077..ac41101 100644 --- a/cmake/GenerateCoverageReport.cmake +++ b/cmake/GenerateCoverageReport.cmake @@ -1,3 +1,5 @@ +cmake_minimum_required(VERSION 3.31) + foreach(required_variable IN ITEMS BINARY_DIRECTORY SOURCE_DIRECTORY @@ -36,6 +38,7 @@ if(NOT coverage_manifest) endif() set(target_keys "") +set(seen_target_names "") foreach(manifest_entry IN LISTS coverage_manifest) if(NOT manifest_entry MATCHES "^([^|]+)[|]([^|]+)[|](.+)$") message(FATAL_ERROR "Invalid coverage manifest entry: ${manifest_entry}") @@ -47,6 +50,11 @@ foreach(manifest_entry IN LISTS coverage_manifest) message(FATAL_ERROR "Coverage object for ${target_name} does not exist: ${target_object}") endif() + if(target_name IN_LIST seen_target_names) + message(FATAL_ERROR + "Coverage manifest contains duplicate target ${target_name}") + endif() + list(APPEND seen_target_names "${target_name}") string(SHA256 target_key "${target_name}") list(APPEND target_keys "${target_key}") set(target_name_${target_key} "${target_name}") @@ -79,6 +87,16 @@ foreach(manifest_entry IN LISTS coverage_manifest) endif() endforeach() +set(seen_binary_ids "") +foreach(target_key IN LISTS target_keys) + set(target_binary_id "${target_binary_id_${target_key}}") + if(target_binary_id IN_LIST seen_binary_ids) + message(FATAL_ERROR + "Coverage manifest maps more than one executable to binary identity ${target_binary_id}") + endif() + list(APPEND seen_binary_ids "${target_binary_id}") +endforeach() + file(GLOB_RECURSE coverage_profiles LIST_DIRECTORIES FALSE "${BINARY_DIRECTORY}/*.profraw" "${BINARY_DIRECTORY}/*.profdata") @@ -166,6 +184,11 @@ set(coverage_work_directory "${BINARY_DIRECTORY}/coverage-work") file(REMOVE_RECURSE "${coverage_work_directory}") file(MAKE_DIRECTORY "${coverage_work_directory}") set(object_trace_files "") +string(CONCAT coverage_provenance + "schema\tsimdlib-coverage-provenance-v1\n" + "merge_scope\tper-executable\n" + "constexpr_evidence\texcluded\n" + "target\tbinary_id\tprofile_count\tprofile_prefix\texecutable\n") foreach(target_key IN LISTS target_keys) set(target_name "${target_name_${target_key}}") set(target_profiles "${target_profiles_${target_key}}") @@ -216,6 +239,8 @@ foreach(target_key IN LISTS target_keys) list(APPEND object_trace_files "${target_trace_file}") list(LENGTH target_profiles target_profile_count) + string(APPEND coverage_provenance + "${target_name}\t${target_binary_id_${target_key}}\t${target_profile_count}\t${target_prefix_${target_key}}\t${target_object_${target_key}}\n") message(STATUS "Mapped ${target_profile_count} profiles to ${target_name}") endforeach() @@ -224,6 +249,14 @@ set(TRACE_FILES "${object_trace_files}") set(OUTPUT_FILE "${BINARY_DIRECTORY}/coverage.info") include("${CMAKE_CURRENT_LIST_DIR}/MergeLcov.cmake") +set(coverage_provenance_file + "${BINARY_DIRECTORY}/coverage-provenance.tsv") +set(coverage_provenance_temporary_file + "${coverage_provenance_file}.tmp") +file(WRITE "${coverage_provenance_temporary_file}" "${coverage_provenance}") +file(RENAME "${coverage_provenance_temporary_file}" + "${coverage_provenance_file}") + list(LENGTH target_keys target_count) message(STATUS - "Generated ${OUTPUT_FILE} from ${assigned_profile_count} profiles mapped to ${target_count} executables; excluded ${excluded_profile_count} multi-executable/tool profiles") + "Generated ${OUTPUT_FILE} and ${coverage_provenance_file} from ${assigned_profile_count} profiles mapped to ${target_count} executables; excluded ${excluded_profile_count} multi-executable/tool profiles") diff --git a/cmake/MergeLcov.cmake b/cmake/MergeLcov.cmake index c3442e8..e2fc931 100644 --- a/cmake/MergeLcov.cmake +++ b/cmake/MergeLcov.cmake @@ -1,3 +1,5 @@ +cmake_minimum_required(VERSION 3.31) + foreach(required_variable IN ITEMS TRACE_FILES OUTPUT_FILE) if(NOT DEFINED ${required_variable}) message(FATAL_ERROR "${required_variable} is required") diff --git a/cmake/PublicHeaderStaticAssertAllowlist.txt b/cmake/PublicHeaderStaticAssertAllowlist.txt deleted file mode 100644 index 8bb1ebe..0000000 --- a/cmake/PublicHeaderStaticAssertAllowlist.txt +++ /dev/null @@ -1,31 +0,0 @@ -# Header|assertion-line substring|classification and justification -UInt128.h|width >= 0 && width <= 128|template constraint: rejects masks wider than uint128_t -UInt128.h|sizeof(uint128_t) == 16|ABI invariant: uint128_t must occupy one 128-bit register -UInt128.h|alignof(uint128_t) == 16|ABI invariant: uint128_t must retain SIMD-compatible alignment -UInt128.h|std::is_standard_layout_v|ABI invariant: object representation must remain standard-layout -UInt128.h|std::is_trivially_copyable_v|ABI invariant: register conversion requires trivial copying -Bmi.h|sizeof(unsigned_type) == sizeof(std::uint64_t)|implementation safety invariant: the 64-bit split product branch requires 64-bit words -Bmi.h|start <= 255 && len <= 255|template constraint: BMI bit-extract controls must fit their fields -Bmi.h|BMI bit-extract length must fit the intrinsic control field|template constraint: BMI bit-extract length must fit its control field -SimdAlgo.h|WriteWidth == 1|template constraint: packed comparisons support one-bit output or the documented legacy shape -SimdAlgo.h|count % write_data_size == 0|template constraint: packed output must contain whole destination elements -Api.h|is_widen_target_v|template constraint: widening requires the destination SIMD shape -Api.h|static_assert(using_int|template constraint: widening accepts integral source lanes only -Api.h|std::is_integral_v|template constraint: widening accepts integral destination lanes only -Api.h|sizeof(element_t) < sizeof(typename target_simd::element_type)|template constraint: widening must increase lane width -Api.h|target_simd::register_width == 128|template constraint: widening supports documented register widths only -Api.h|requires(vector_t value) { impl::template widen|unsupported-instantiation diagnostic: reports missing backend widening mappings -Api.h|shift >= 0|template constraint: immediate whole-register shift counts cannot be negative -Api.h|element_width == 32|template constraint: public integer-float conversions require 32-bit lanes -Api.h|std::unsigned_integral|template constraint: packed transforms require unsigned result storage -Api.h|element_count * result_bit_width <= 64|template constraint: one packed register result cannot exceed 64 bits -Api.h|element_count * result_bit_width <= std::numeric_limits::digits|template constraint: packed result type must hold every produced bit -Api.h|std::endian::native == std::endian::little|implementation safety invariant: packed lane order assumes little-endian storage -Api.h|remaining_storage_byte_count <= sizeof(native_word_t)|implementation safety invariant: the final packed store fits one native word -Api.h|std::is_invocable_r_v|template constraint: unary transforms must preserve the register type -Api.h|std::is_invocable_r_v|template constraint: binary transforms must preserve the register type -Implementations.h|dependent_false_v|unsupported-instantiation diagnostic: unavailable widening shapes must fail dependently -Implementations.h|SimdMappings<128>::extract index out of range|template constraint: 128-bit extraction index must name an existing lane -Implementations.h|SimdMappings<256>::extract index out of range|template constraint: 256-bit extraction index must name an existing lane -Implementations.h|Unsupported element size|implementation safety invariant: scalar register transforms support 1, 2, 4, or 8-byte lanes -Extensions.h|shift >= 0|template constraint: immediate whole-register extension shifts cannot be negative \ No newline at end of file diff --git a/cmake/RecordArtifactHashes.cmake b/cmake/RecordArtifactHashes.cmake new file mode 100644 index 0000000..96994ad --- /dev/null +++ b/cmake/RecordArtifactHashes.cmake @@ -0,0 +1,58 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS MODE RECORD_FILE) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR "RecordArtifactHashes requires ${required_variable}") + endif() +endforeach() +if(NOT MODE MATCHES "^(RECORD|VALIDATE)$") + message(FATAL_ERROR "RecordArtifactHashes MODE must be RECORD or VALIDATE") +endif() + +# @brief Hashes a sorted set of required build artifacts. +# @param artifact_paths Paths to hash. +# @param output_variable Variable that receives the machine-readable record body. +function(simdlib_hash_artifacts artifact_paths output_variable) + set(paths "${artifact_paths}") + list(REMOVE_DUPLICATES paths) + list(SORT paths) + set(record "schema=simdlib.artifact-record.v1\n") + foreach(artifact_path IN LISTS paths) + if(NOT EXISTS "${artifact_path}" OR IS_DIRECTORY "${artifact_path}") + message(FATAL_ERROR "Required build artifact is missing: ${artifact_path}") + endif() + file(SHA256 "${artifact_path}" artifact_hash) + string(APPEND record "${artifact_path}|${artifact_hash}\n") + endforeach() + set(${output_variable} "${record}" PARENT_SCOPE) +endfunction() + +if(MODE STREQUAL "RECORD") + if(NOT DEFINED ARTIFACTS OR "${ARTIFACTS}" STREQUAL "") + message(FATAL_ERROR "RecordArtifactHashes RECORD mode requires ARTIFACTS") + endif() + string(REPLACE "|" ";" artifact_paths "${ARTIFACTS}") + simdlib_hash_artifacts("${artifact_paths}" current_record) + cmake_path(GET RECORD_FILE PARENT_PATH record_directory) + file(MAKE_DIRECTORY "${record_directory}") + file(WRITE "${RECORD_FILE}.tmp" "${current_record}") + file(RENAME "${RECORD_FILE}.tmp" "${RECORD_FILE}") +elseif(NOT EXISTS "${RECORD_FILE}") + message(FATAL_ERROR "Required build artifact record is missing: ${RECORD_FILE}") +else() + file(STRINGS "${RECORD_FILE}" recorded_lines) + list(POP_FRONT recorded_lines schema_line) + if(NOT schema_line STREQUAL "schema=simdlib.artifact-record.v1") + message(FATAL_ERROR "Build artifact record has an unsupported schema: ${RECORD_FILE}") + endif() + set(artifact_paths "") + foreach(recorded_line IN LISTS recorded_lines) + string(REGEX REPLACE "\\|[0-9a-fA-F]+$" "" artifact_path "${recorded_line}") + list(APPEND artifact_paths "${artifact_path}") + endforeach() + simdlib_hash_artifacts("${artifact_paths}" current_record) + file(READ "${RECORD_FILE}" recorded_record) + if(NOT current_record STREQUAL recorded_record) + message(FATAL_ERROR "Build artifact record is stale: ${RECORD_FILE}") + endif() +endif() diff --git a/cmake/RecordRegisterDefaultAbi.cmake b/cmake/RecordRegisterDefaultAbi.cmake new file mode 100644 index 0000000..ebabe88 --- /dev/null +++ b/cmake/RecordRegisterDefaultAbi.cmake @@ -0,0 +1,108 @@ +cmake_minimum_required(VERSION 3.31) + +string(TIMESTAMP codegen_start_epoch "%s" UTC) + +foreach(required_variable IN ITEMS + WRAPPER_OBJECT RAW_OBJECT OBJDUMP ARTIFACT_DIRECTORY COMPILER_ID + COMPILER_VERSION COMPILER_PATH SYSTEM_NAME SYSTEM_PROCESSOR CONFIGURATION REGISTER_WIDTH + ISA_PROFILE VECTORCALL_ENABLED STACK_PROTECTOR_MODE) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR "RecordRegisterDefaultAbi requires ${required_variable}") + endif() +endforeach() +if(NOT DEFINED RECORD_FILE OR "${RECORD_FILE}" STREQUAL "") + set(RECORD_FILE "${ARTIFACT_DIRECTORY}/default-abi.record.json") +endif() +file(REMOVE "${RECORD_FILE}") +string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef record_temporary_suffix) +set(record_temporary_file "${RECORD_FILE}.${record_temporary_suffix}.tmp") + +# @brief Disassembles one default-convention ABI fixture and writes the artifact. +# @param object_file Compiled fixture object. +# @param output_file Destination disassembly file. +function(simdlib_record_default_abi object_file output_file) + execute_process( + COMMAND "${OBJDUMP}" -d "${object_file}" + RESULT_VARIABLE disassembly_result + OUTPUT_VARIABLE disassembly + ERROR_VARIABLE disassembly_error) + if(NOT disassembly_result EQUAL 0) + message(FATAL_ERROR "Unable to disassemble ${object_file}: ${disassembly_error}") + endif() + file(WRITE "${output_file}" "${disassembly}") +endfunction() + +# @brief Escapes a string for inclusion as a JSON string value. +# @param input_text Unescaped text. +# @param output_variable Variable that receives escaped text. +function(simdlib_escape_json input_text output_variable) + set(escaped "${input_text}") + string(REPLACE "\\" "\\\\" escaped "${escaped}") + string(REPLACE "\"" "\\\"" escaped "${escaped}") + string(REPLACE "\r" "\\r" escaped "${escaped}") + string(REPLACE "\n" "\\n" escaped "${escaped}") + string(REPLACE "\t" "\\t" escaped "${escaped}") + set(${output_variable} "${escaped}" PARENT_SCOPE) +endfunction() + +simdlib_record_default_abi("${WRAPPER_OBJECT}" "${ARTIFACT_DIRECTORY}/default-wrapper.disassembly.txt") +simdlib_record_default_abi("${RAW_OBJECT}" "${ARTIFACT_DIRECTORY}/default-raw.disassembly.txt") +file(WRITE "${ARTIFACT_DIRECTORY}/default-abi.provenance.txt" + "compiler_id=${COMPILER_ID}\n" + "compiler_version=${COMPILER_VERSION}\n" + "compiler_path=${COMPILER_PATH}\n" + "system_name=${SYSTEM_NAME}\n" + "system_processor=${SYSTEM_PROCESSOR}\n" + "configuration=${CONFIGURATION}\n" + "register_width=${REGISTER_WIDTH}\n" + "isa_profile=${ISA_PROFILE}\n" + "calling_convention=platform-default\n" + "vectorcall_enabled=${VECTORCALL_ENABLED}\n" + "stack_protector_mode=${STACK_PROTECTOR_MODE}\n" + "wrapper_object=${WRAPPER_OBJECT}\n" + "raw_object=${RAW_OBJECT}\n") + +file(SHA256 "${WRAPPER_OBJECT}" wrapper_hash) +file(SHA256 "${RAW_OBJECT}" raw_hash) +file(SHA256 "${OBJDUMP}" tool_hash) +execute_process( + COMMAND "${OBJDUMP}" --version + RESULT_VARIABLE tool_version_result + OUTPUT_VARIABLE tool_version_output + ERROR_VARIABLE tool_version_error) +if(NOT tool_version_result EQUAL 0) + message(FATAL_ERROR "Unable to identify default-ABI recording tool: ${tool_version_error}") +endif() +string(REGEX REPLACE "\r?\n.*" "" tool_version "${tool_version_output}") +string(TIMESTAMP codegen_end_epoch "%s" UTC) +math(EXPR codegen_total_seconds + "${codegen_end_epoch} - ${codegen_start_epoch}") +foreach(json_value IN ITEMS + WRAPPER_OBJECT RAW_OBJECT OBJDUMP tool_version COMPILER_ID COMPILER_VERSION + COMPILER_PATH SYSTEM_NAME SYSTEM_PROCESSOR CONFIGURATION ISA_PROFILE STACK_PROTECTOR_MODE) + simdlib_escape_json("${${json_value}}" "${json_value}_json") +endforeach() +file(WRITE "${record_temporary_file}" + "{\n" + " \"schema\": \"simdlib.codegen-record.v1\",\n" + " \"kind\": \"diagnostic\",\n" + " \"result\": \"recorded-diagnostic\",\n" + " \"accepted_exception\": \"none\",\n" + " \"inputs\": {\n" + " \"wrapper\": {\"path\": \"${WRAPPER_OBJECT_json}\", \"sha256\": \"${wrapper_hash}\"},\n" + " \"raw\": {\"path\": \"${RAW_OBJECT_json}\", \"sha256\": \"${raw_hash}\"}\n" + " },\n" + " \"tool\": {\"path\": \"${OBJDUMP_json}\", \"version\": \"${tool_version_json}\", \"sha256\": \"${tool_hash}\"},\n" + " \"policy\": {\"id\": \"register-default-abi-diagnostic-v1\", \"mode\": \"RECORD\", " + "\"calling_convention\": \"platform-default\"},\n" + " \"compiler\": {\"id\": \"${COMPILER_ID_json}\", \"version\": \"${COMPILER_VERSION_json}\", " + "\"path\": \"${COMPILER_PATH_json}\"},\n" + " \"platform\": {\"system\": \"${SYSTEM_NAME_json}\", \"processor\": \"${SYSTEM_PROCESSOR_json}\"},\n" + " \"configuration\": \"${CONFIGURATION_json}\",\n" + " \"timing\": {\"total_seconds\": ${codegen_total_seconds}},\n" + " \"register_width\": ${REGISTER_WIDTH},\n" + " \"isa_profile\": \"${ISA_PROFILE_json}\",\n" + " \"vectorcall_enabled\": ${VECTORCALL_ENABLED},\n" + " \"stack_protector_mode\": \"${STACK_PROTECTOR_MODE_json}\"\n" + "}\n") +file(RENAME "${record_temporary_file}" "${RECORD_FILE}") diff --git a/cmake/RecordTestInventory.cmake b/cmake/RecordTestInventory.cmake new file mode 100644 index 0000000..81ea497 --- /dev/null +++ b/cmake/RecordTestInventory.cmake @@ -0,0 +1,88 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS MODE TEST_DIRECTORY INVENTORY_FILE) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR "RecordTestInventory requires ${required_variable}") + endif() +endforeach() +if(NOT MODE MATCHES "^(RECORD|VALIDATE)$") + message(FATAL_ERROR "RecordTestInventory MODE must be RECORD or VALIDATE") +endif() + +# @brief Produces a deterministic inventory of executables owned by one CTest tree. +# @param output_variable Variable that receives newline-delimited path and SHA-256 pairs. +function(simdlib_collect_test_inventory output_variable) + set(ctest_arguments --test-dir "${TEST_DIRECTORY}" -N -V) + if(DEFINED CONFIGURATION AND NOT "${CONFIGURATION}" STREQUAL "") + list(APPEND ctest_arguments -C "${CONFIGURATION}") + endif() + execute_process( + COMMAND "${CMAKE_CTEST_COMMAND}" ${ctest_arguments} + RESULT_VARIABLE ctest_result + OUTPUT_VARIABLE ctest_output + ERROR_VARIABLE ctest_error) + if(NOT ctest_result EQUAL 0) + message(FATAL_ERROR "Unable to enumerate tests in ${TEST_DIRECTORY}: ${ctest_error}") + endif() + + set(executables "") + string(REPLACE "\r\n" "\n" ctest_output "${ctest_output}") + string(REGEX MATCHALL "Test command: [^\n\r]+" command_lines "${ctest_output}") + foreach(command_line IN LISTS command_lines) + if(NOT command_line MATCHES "^Test command: \"?([^\" ]+)") + continue() + endif() + set(executable "${CMAKE_MATCH_1}") + cmake_path(ABSOLUTE_PATH executable NORMALIZE OUTPUT_VARIABLE absolute_executable) + cmake_path(IS_PREFIX TEST_DIRECTORY "${absolute_executable}" NORMALIZE executable_is_owned) + if(NOT executable_is_owned OR NOT EXISTS "${absolute_executable}" OR IS_DIRECTORY "${absolute_executable}") + continue() + endif() + list(APPEND executables "${absolute_executable}") + endforeach() + list(REMOVE_DUPLICATES executables) + list(SORT executables) + + set(entries "") + foreach(executable IN LISTS executables) + file(SHA256 "${executable}" executable_hash) + list(APPEND entries "${executable}|${executable_hash}") + endforeach() + string(REPLACE ";" "\n" artifact_inventory "${entries}") + set(inventory "schema=simdlib.test-artifact-inventory.v1\n") + if(NOT artifact_inventory STREQUAL "") + string(APPEND inventory "${artifact_inventory}\n") + endif() + set(${output_variable} "${inventory}" PARENT_SCOPE) +endfunction() + +if(MODE STREQUAL "RECORD") + simdlib_collect_test_inventory(current_inventory) + cmake_path(GET INVENTORY_FILE PARENT_PATH inventory_directory) + file(MAKE_DIRECTORY "${inventory_directory}") + set(temporary_file "${INVENTORY_FILE}.tmp") + file(WRITE "${temporary_file}" "${current_inventory}") + file(RENAME "${temporary_file}" "${INVENTORY_FILE}") +elseif(NOT EXISTS "${INVENTORY_FILE}") + message(FATAL_ERROR "Required test artifact inventory is missing: ${INVENTORY_FILE}") +else() + file(STRINGS "${INVENTORY_FILE}" inventory_lines) + list(POP_FRONT inventory_lines schema_line) + if(NOT schema_line STREQUAL "schema=simdlib.test-artifact-inventory.v1") + message(FATAL_ERROR "Test artifact inventory has an unsupported schema: ${INVENTORY_FILE}") + endif() + foreach(inventory_line IN LISTS inventory_lines) + if(NOT inventory_line MATCHES "^(.+)\\|([0-9a-fA-F]+)$") + message(FATAL_ERROR "Test artifact inventory entry is malformed: ${inventory_line}") + endif() + set(executable "${CMAKE_MATCH_1}") + set(recorded_hash "${CMAKE_MATCH_2}") + if(NOT EXISTS "${executable}" OR IS_DIRECTORY "${executable}") + message(FATAL_ERROR "Required test artifact is missing: ${executable}") + endif() + file(SHA256 "${executable}" current_hash) + if(NOT current_hash STREQUAL recorded_hash) + message(FATAL_ERROR "Required test artifact is stale: ${executable}") + endif() + endforeach() +endif() diff --git a/cmake/ResetCoverage.cmake b/cmake/ResetCoverage.cmake index e78688c..9ea7c84 100644 --- a/cmake/ResetCoverage.cmake +++ b/cmake/ResetCoverage.cmake @@ -1,9 +1,11 @@ +cmake_minimum_required(VERSION 3.31) + if(NOT DEFINED BINARY_DIRECTORY) message(FATAL_ERROR "BINARY_DIRECTORY is required") endif() -# CTest 4.4 clears profiles for tests selected in its current invocation. Clear -# the entire build tree as well so a partial run cannot inherit unrelated data. +# Clear the entire build tree so a partial run cannot inherit profiles from +# unrelated tests or a previous coverage invocation. file(GLOB_RECURSE coverage_profiles LIST_DIRECTORIES FALSE "${BINARY_DIRECTORY}/*.profraw" "${BINARY_DIRECTORY}/*.profdata") diff --git a/cmake/SummarizeCodegenDiagnostic.cmake b/cmake/SummarizeCodegenDiagnostic.cmake new file mode 100644 index 0000000..78d7cb0 --- /dev/null +++ b/cmake/SummarizeCodegenDiagnostic.cmake @@ -0,0 +1,168 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + RECORD_INDEX OUTPUT_FILE COMPILE_COMMANDS SOURCE_REVISION SOURCE_DIGEST + FINGERPRINT COMPILER_ID PRESET CONFIGURATION SANITIZER + COMPILATION_SECONDS COMPARISON_SECONDS) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR + "SummarizeCodegenDiagnostic requires ${required_variable}") + endif() +endforeach() +if(NOT EXISTS "${RECORD_INDEX}") + message(FATAL_ERROR "Codegen record index is missing: ${RECORD_INDEX}") +endif() +if(NOT EXISTS "${COMPILE_COMMANDS}") + message(FATAL_ERROR "Compiler-flag inventory is missing: ${COMPILE_COMMANDS}") +endif() + +# @brief Escapes a string for inclusion as a JSON string value. +# @param input_text Unescaped text. +# @param output_variable Variable that receives escaped text. +function(simdlib_escape_json input_text output_variable) + set(escaped "${input_text}") + string(REPLACE "\\" "\\\\" escaped "${escaped}") + string(REPLACE "\"" "\\\"" escaped "${escaped}") + string(REPLACE "\r" "\\r" escaped "${escaped}") + string(REPLACE "\n" "\\n" escaped "${escaped}") + string(REPLACE "\t" "\\t" escaped "${escaped}") + set(${output_variable} "${escaped}" PARENT_SCOPE) +endfunction() + +file(STRINGS "${RECORD_INDEX}" record_files) +set(record_count 0) +set(record_total_seconds 0) +set(slowest_record_seconds -1) +set(slowest_record "") +set(slowest_record_profile "") +set(stack_protector_modes "") +set(disassembly_tool_keys "") +set(disassembly_tool_entries "") +foreach(record_file IN LISTS record_files) + if(record_file STREQUAL "") + continue() + endif() + file(READ "${record_file}" record_json) + string(JSON policy_mode GET "${record_json}" policy mode) + if(NOT policy_mode STREQUAL "RECORD") + message(FATAL_ERROR + "Diagnostic record does not use RECORD policy: ${record_file}") + endif() + string(JSON record_configuration GET "${record_json}" configuration) + if(NOT record_configuration STREQUAL CONFIGURATION) + message(FATAL_ERROR + "Diagnostic record configuration is ${record_configuration}, expected " + "${CONFIGURATION}: ${record_file}") + endif() + string(JSON record_seconds GET "${record_json}" timing total_seconds) + string(JSON stack_protector_mode GET + "${record_json}" stack_protector_mode) + list(APPEND stack_protector_modes "${stack_protector_mode}") + string(JSON disassembly_tool_path GET "${record_json}" tool path) + string(JSON disassembly_tool_version GET "${record_json}" tool version) + string(JSON disassembly_tool_hash GET "${record_json}" tool sha256) + set(disassembly_tool_key + "${disassembly_tool_path}|${disassembly_tool_version}|${disassembly_tool_hash}") + if(NOT disassembly_tool_key IN_LIST disassembly_tool_keys) + list(APPEND disassembly_tool_keys "${disassembly_tool_key}") + simdlib_escape_json("${disassembly_tool_path}" + disassembly_tool_path_json) + simdlib_escape_json("${disassembly_tool_version}" + disassembly_tool_version_json) + list(APPEND disassembly_tool_entries + "{\"path\": \"${disassembly_tool_path_json}\", \"version\": \"${disassembly_tool_version_json}\", \"sha256\": \"${disassembly_tool_hash}\"}") + endif() + string(JSON record_profile ERROR_VARIABLE record_profile_error + GET "${record_json}" policy codegen_profile) + if(record_profile_error) + set(record_profile "default-abi") + endif() + math(EXPR record_count "${record_count} + 1") + math(EXPR record_total_seconds "${record_total_seconds} + ${record_seconds}") + if(record_seconds GREATER slowest_record_seconds) + set(slowest_record_seconds ${record_seconds}) + set(slowest_record "${record_file}") + set(slowest_record_profile "${record_profile}") + endif() +endforeach() +if(record_count EQUAL 0) + message(FATAL_ERROR "Diagnostic record index contains no records") +endif() + +list(REMOVE_DUPLICATES stack_protector_modes) +list(SORT stack_protector_modes) +set(stack_protector_modes_json "") +foreach(stack_protector_mode IN LISTS stack_protector_modes) + simdlib_escape_json("${stack_protector_mode}" stack_protector_mode_json) + if(NOT stack_protector_modes_json STREQUAL "") + string(APPEND stack_protector_modes_json ", ") + endif() + string(APPEND stack_protector_modes_json + "\"${stack_protector_mode_json}\"") +endforeach() +string(JOIN ", " disassembly_tools_json ${disassembly_tool_entries}) + +file(SHA256 "${RECORD_INDEX}" record_index_hash) +file(SHA256 "${COMPILE_COMMANDS}" compile_commands_hash) +math(EXPR invocation_total_seconds + "${COMPILATION_SECONDS} + ${COMPARISON_SECONDS}") +set(measured_compilation_seconds ${COMPILATION_SECONDS}) +set(measured_comparison_seconds ${COMPARISON_SECONDS}) +if(EXISTS "${OUTPUT_FILE}") + file(READ "${OUTPUT_FILE}" prior_provenance_json) + string(JSON prior_compile_commands_hash ERROR_VARIABLE prior_compile_hash_error + GET "${prior_provenance_json}" compiler_flags sha256) + string(JSON prior_record_index_hash ERROR_VARIABLE prior_record_hash_error + GET "${prior_provenance_json}" records sha256) + if(NOT prior_compile_hash_error AND NOT prior_record_hash_error AND + prior_compile_commands_hash STREQUAL compile_commands_hash AND + prior_record_index_hash STREQUAL record_index_hash) + string(JSON prior_compilation_seconds ERROR_VARIABLE prior_measured_error + GET "${prior_provenance_json}" timing measured compilation_seconds) + string(JSON prior_comparison_seconds ERROR_VARIABLE prior_comparison_error + GET "${prior_provenance_json}" timing measured comparison_seconds) + if(prior_measured_error OR prior_comparison_error) + string(JSON prior_compilation_seconds ERROR_VARIABLE prior_legacy_error + GET "${prior_provenance_json}" timing compilation_seconds) + string(JSON prior_comparison_seconds ERROR_VARIABLE prior_legacy_comparison_error + GET "${prior_provenance_json}" timing comparison_seconds) + if(prior_legacy_error OR prior_legacy_comparison_error) + set(prior_compilation_seconds 0) + set(prior_comparison_seconds 0) + endif() + endif() + if(prior_compilation_seconds GREATER measured_compilation_seconds) + set(measured_compilation_seconds ${prior_compilation_seconds}) + endif() + if(prior_comparison_seconds GREATER measured_comparison_seconds) + set(measured_comparison_seconds ${prior_comparison_seconds}) + endif() + endif() +endif() +math(EXPR measured_total_seconds + "${measured_compilation_seconds} + ${measured_comparison_seconds}") +foreach(json_value IN ITEMS + RECORD_INDEX COMPILE_COMMANDS SOURCE_REVISION SOURCE_DIGEST FINGERPRINT + COMPILER_ID PRESET CONFIGURATION SANITIZER slowest_record slowest_record_profile) + simdlib_escape_json("${${json_value}}" "${json_value}_json") +endforeach() +file(WRITE "${OUTPUT_FILE}" + "{\n" + " \"schema\": \"simdlib.codegen-diagnostic-provenance.v1\",\n" + " \"operation\": \"record-codegen\",\n" + " \"status\": \"complete\",\n" + " \"source_revision\": \"${SOURCE_REVISION_json}\",\n" + " \"source_digest\": \"${SOURCE_DIGEST_json}\",\n" + " \"fingerprint\": \"${FINGERPRINT_json}\",\n" + " \"compiler_id\": \"${COMPILER_ID_json}\",\n" + " \"configuration\": {\"preset\": \"${PRESET_json}\", \"build_profile\": \"${CONFIGURATION_json}\", \"sanitizer\": \"${SANITIZER_json}\", \"codegen_mode\": \"RECORD\"},\n" + " \"compiler_flags\": {\"path\": \"${COMPILE_COMMANDS_json}\", \"sha256\": \"${compile_commands_hash}\"},\n" + " \"stack_protector_modes\": [${stack_protector_modes_json}],\n" + " \"disassembly_tools\": [${disassembly_tools_json}],\n" + " \"records\": {\"index\": \"${RECORD_INDEX_json}\", \"sha256\": \"${record_index_hash}\", \"count\": ${record_count}, \"reported_seconds\": ${record_total_seconds}},\n" + " \"slowest_record\": {\"path\": \"${slowest_record_json}\", \"profile\": \"${slowest_record_profile_json}\", \"seconds\": ${slowest_record_seconds}},\n" + " \"timing\": {\n" + " \"invocation\": {\"compilation_seconds\": ${COMPILATION_SECONDS}, \"comparison_seconds\": ${COMPARISON_SECONDS}, \"total_seconds\": ${invocation_total_seconds}},\n" + " \"measured\": {\"compilation_seconds\": ${measured_compilation_seconds}, \"comparison_seconds\": ${measured_comparison_seconds}, \"total_seconds\": ${measured_total_seconds}}\n" + " }\n" + "}\n") diff --git a/cmake/ValidateCodegenRecords.cmake b/cmake/ValidateCodegenRecords.cmake new file mode 100644 index 0000000..4daa240 --- /dev/null +++ b/cmake/ValidateCodegenRecords.cmake @@ -0,0 +1,78 @@ +cmake_minimum_required(VERSION 3.31) + +if(NOT DEFINED RECORD_INDEX OR "${RECORD_INDEX}" STREQUAL "") + message(FATAL_ERROR "ValidateCodegenRecords requires RECORD_INDEX") +endif() +if(NOT EXISTS "${RECORD_INDEX}") + message(FATAL_ERROR "Required generated-code record index is missing: ${RECORD_INDEX}") +endif() + +# @brief Validates one generated-code record and its hashed inputs. +# @param record_file Machine-readable comparison or diagnostic record. +function(simdlib_validate_codegen_record record_file) + if(NOT EXISTS "${record_file}") + message(FATAL_ERROR "Required generated-code record is missing: ${record_file}") + endif() + file(READ "${record_file}" record_json) + string(JSON schema ERROR_VARIABLE schema_error GET "${record_json}" schema) + if(schema_error OR NOT schema STREQUAL "simdlib.codegen-record.v1") + message(FATAL_ERROR "Generated-code record has an unsupported schema: ${record_file}") + endif() + string(JSON result ERROR_VARIABLE result_error GET "${record_json}" result) + if(result_error OR NOT result MATCHES "^(exact-parity|accepted-compiler-exception|recorded-difference|recorded-diagnostic)$") + message(FATAL_ERROR "Generated-code record has an invalid result: ${record_file}") + endif() + if(DEFINED EXPECTED_POLICY_MODE AND NOT "${EXPECTED_POLICY_MODE}" STREQUAL "") + string(JSON policy_mode ERROR_VARIABLE policy_mode_error + GET "${record_json}" policy mode) + if(policy_mode_error OR NOT policy_mode STREQUAL EXPECTED_POLICY_MODE) + message(FATAL_ERROR + "Generated-code record policy is ${policy_mode}, expected " + "${EXPECTED_POLICY_MODE}: ${record_file}") + endif() + endif() + if(DEFINED EXPECTED_CONFIGURATION AND NOT "${EXPECTED_CONFIGURATION}" STREQUAL "") + string(JSON configuration ERROR_VARIABLE configuration_error + GET "${record_json}" configuration) + if(configuration_error OR NOT configuration STREQUAL EXPECTED_CONFIGURATION) + message(FATAL_ERROR + "Generated-code record configuration is ${configuration}, expected " + "${EXPECTED_CONFIGURATION}: ${record_file}") + endif() + endif() + + foreach(input_name IN ITEMS wrapper raw) + string(JSON input_path ERROR_VARIABLE path_error GET "${record_json}" inputs ${input_name} path) + string(JSON input_hash ERROR_VARIABLE hash_error GET "${record_json}" inputs ${input_name} sha256) + if(path_error OR hash_error OR NOT EXISTS "${input_path}") + message(FATAL_ERROR "Generated-code record input is missing: ${record_file} (${input_name})") + endif() + file(SHA256 "${input_path}" current_hash) + if(NOT current_hash STREQUAL input_hash) + message(FATAL_ERROR "Generated-code record input is stale: ${record_file} (${input_name})") + endif() + endforeach() + + string(JSON tool_path ERROR_VARIABLE tool_path_error GET "${record_json}" tool path) + string(JSON tool_hash ERROR_VARIABLE tool_hash_error GET "${record_json}" tool sha256) + if(tool_path_error OR tool_hash_error OR NOT EXISTS "${tool_path}") + message(FATAL_ERROR "Generated-code record tool is missing: ${record_file}") + endif() + file(SHA256 "${tool_path}" current_tool_hash) + if(NOT current_tool_hash STREQUAL tool_hash) + message(FATAL_ERROR "Generated-code record tool identity is stale: ${record_file}") + endif() +endfunction() + +file(STRINGS "${RECORD_INDEX}" record_files) +set(validated_record_count 0) +foreach(record_file IN LISTS record_files) + if(NOT record_file STREQUAL "") + simdlib_validate_codegen_record("${record_file}") + math(EXPR validated_record_count "${validated_record_count} + 1") + endif() +endforeach() +if(DEFINED REQUIRE_RECORDS AND REQUIRE_RECORDS AND validated_record_count EQUAL 0) + message(FATAL_ERROR + "Generated-code record index contains no records: ${RECORD_INDEX}") +endif() diff --git a/cmake/ValidateRegisterCodegenProfile.cmake b/cmake/ValidateRegisterCodegenProfile.cmake new file mode 100644 index 0000000..26474fb --- /dev/null +++ b/cmake/ValidateRegisterCodegenProfile.cmake @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + ENFORCED_RECORD_INDEX DIAGNOSTIC_RECORD_INDEX CODEGEN_MODE CONFIGURATION) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR + "ValidateRegisterCodegenProfile requires ${required_variable}") + endif() +endforeach() +if(NOT CODEGEN_MODE MATCHES "^(ENFORCE|RECORD)$") + message(FATAL_ERROR "Unsupported Register codegen mode: ${CODEGEN_MODE}") +endif() + +if(CODEGEN_MODE STREQUAL "ENFORCE") + set(RECORD_INDEX "${ENFORCED_RECORD_INDEX}") + set(EXPECTED_POLICY_MODE ENFORCE) + set(EXPECTED_CONFIGURATION "${CONFIGURATION}") + set(REQUIRE_RECORDS "${REQUIRE_ENFORCED_RECORDS}") + include("${CMAKE_CURRENT_LIST_DIR}/ValidateCodegenRecords.cmake") +endif() + +set(RECORD_INDEX "${DIAGNOSTIC_RECORD_INDEX}") +set(EXPECTED_POLICY_MODE RECORD) +set(EXPECTED_CONFIGURATION "${CONFIGURATION}") +set(REQUIRE_RECORDS ON) +include("${CMAKE_CURRENT_LIST_DIR}/ValidateCodegenRecords.cmake") diff --git a/cmake/VerifyArtifactAggregateFailure.cmake b/cmake/VerifyArtifactAggregateFailure.cmake new file mode 100644 index 0000000..eaadd7a --- /dev/null +++ b/cmake/VerifyArtifactAggregateFailure.cmake @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + CASE SOURCE_DIRECTORY BINARY_DIRECTORY GENERATOR MAKE_PROGRAM) + if(NOT DEFINED ${required_variable}) + message(FATAL_ERROR "Missing required variable ${required_variable}") + endif() +endforeach() + +if(CASE STREQUAL "UNOWNED") + set(expected_diagnostic "has no validation-category owner") +elseif(CASE STREQUAL "MULTIPLE") + set(expected_diagnostic "has multiple validation owners") +elseif(CASE STREQUAL "EXCLUDED") + set(expected_diagnostic "excluded by validation") +else() + message(FATAL_ERROR "Unsupported artifact-aggregate failure case ${CASE}") +endif() + +execute_process( + COMMAND "${CMAKE_COMMAND}" + --fresh + -G "${GENERATOR}" + -S "${SOURCE_DIRECTORY}/tests/cmake/artifact_aggregates" + -B "${BINARY_DIRECTORY}" + "-DCMAKE_MAKE_PROGRAM=${MAKE_PROGRAM}" + "-DSIMDLIB_SOURCE_DIRECTORY=${SOURCE_DIRECTORY}" + "-DSIMDLIB_ARTIFACT_FAILURE_CASE=${CASE}" + RESULT_VARIABLE configure_result + OUTPUT_VARIABLE configure_stdout + ERROR_VARIABLE configure_stderr) +set(configure_output "${configure_stdout}${configure_stderr}") +if(configure_result EQUAL 0) + message(FATAL_ERROR + "Artifact-aggregate case ${CASE} unexpectedly configured successfully") +endif() +if(NOT configure_output MATCHES "${expected_diagnostic}") + message(FATAL_ERROR + "Artifact-aggregate case ${CASE} did not emit ${expected_diagnostic}:\n" + "${configure_output}") +endif() + +message(STATUS "Artifact-aggregate case ${CASE} failed as required") diff --git a/cmake/VerifyArtifactAggregateInventory.cmake b/cmake/VerifyArtifactAggregateInventory.cmake new file mode 100644 index 0000000..ed712bc --- /dev/null +++ b/cmake/VerifyArtifactAggregateInventory.cmake @@ -0,0 +1,129 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + OWNERSHIP_FILE AGGREGATE_FILE MEMBERSHIP_FILE PROFILE SELECTED_CATEGORIES) + if(NOT DEFINED ${required_variable}) + message(FATAL_ERROR "Missing required variable ${required_variable}") + endif() +endforeach() + +if(NOT EXISTS "${OWNERSHIP_FILE}") + message(FATAL_ERROR "Ownership inventory does not exist: ${OWNERSHIP_FILE}") +endif() +if(NOT EXISTS "${AGGREGATE_FILE}") + message(FATAL_ERROR "Aggregate inventory does not exist: ${AGGREGATE_FILE}") +endif() +if(NOT EXISTS "${MEMBERSHIP_FILE}") + message(FATAL_ERROR "Aggregate membership does not exist: ${MEMBERSHIP_FILE}") +endif() + +file(STRINGS "${OWNERSHIP_FILE}" ownership_rows) +list(POP_FRONT ownership_rows ownership_header) +if(NOT ownership_header STREQUAL + "target\tcategory\towning_aggregate\tselected") + message(FATAL_ERROR "Ownership inventory has an invalid header") +endif() + +set(previous_target "") +set(seen_targets "") +foreach(ownership_row IN LISTS ownership_rows) + if(NOT ownership_row MATCHES + "^([^\t]+)\t([^\t]+)\t([^\t]+)\t(YES|NO)$") + message(FATAL_ERROR "Malformed ownership row: ${ownership_row}") + endif() + set(target "${CMAKE_MATCH_1}") + set(category "${CMAKE_MATCH_2}") + set(aggregate "${CMAKE_MATCH_3}") + set(selected "${CMAKE_MATCH_4}") + + if(target IN_LIST seen_targets) + message(FATAL_ERROR "Target ${target} occurs more than once") + endif() + if(previous_target AND target STRLESS previous_target) + message(FATAL_ERROR "Ownership rows are not sorted deterministically") + endif() + if(NOT aggregate MATCHES "^(SimdLib.+Artifacts|BenchmarkArtifacts)$") + message(FATAL_ERROR "Target ${target} has invalid aggregate ${aggregate}") + endif() + if(category IN_LIST SELECTED_CATEGORIES) + if(NOT selected STREQUAL "YES") + message(FATAL_ERROR + "Profile ${PROFILE} failed to select ${target} from ${category}") + endif() + elseif(NOT selected STREQUAL "NO") + message(FATAL_ERROR + "Profile ${PROFILE} selected forbidden target ${target} from ${category}") + endif() + if(category STREQUAL "BENCHMARK" AND selected STREQUAL "YES") + message(FATAL_ERROR "Benchmarks entered the default validation aggregate") + endif() + + list(APPEND seen_targets "${target}") + set(previous_target "${target}") +endforeach() + +file(STRINGS "${AGGREGATE_FILE}" aggregate_rows) +list(POP_FRONT aggregate_rows aggregate_header) +if(NOT aggregate_header STREQUAL "aggregate\tcategory") + message(FATAL_ERROR "Aggregate inventory has an invalid header") +endif() +set(required_aggregates + ExhaustiveArtifacts + SimdLibCompilerContractArtifacts + SimdLibConstexprContractArtifacts + SimdLibRuntimeValidationArtifacts + SimdLibChecksValidationArtifacts + SimdLibSmokeValidationArtifacts + SimdLibOptimizedCodegenArtifacts + SimdLibDebugDiagnosticArtifacts + SimdLibSanitizerValidationArtifacts + SimdLibCoverageValidationArtifacts + SimdLibCoverageSupportArtifacts + BenchmarkArtifacts) +foreach(required_aggregate IN LISTS required_aggregates) + set(aggregate_matches ${aggregate_rows}) + list(FILTER aggregate_matches INCLUDE REGEX "^${required_aggregate}\t") + list(LENGTH aggregate_matches aggregate_match_count) + if(NOT aggregate_match_count EQUAL 1) + message(FATAL_ERROR + "Aggregate inventory does not contain exactly one ${required_aggregate} row") + endif() +endforeach() + +file(STRINGS "${MEMBERSHIP_FILE}" membership_rows) +list(POP_FRONT membership_rows membership_header) +if(NOT membership_header STREQUAL "aggregate\tdependency") + message(FATAL_ERROR "Aggregate membership has an invalid header") +endif() +foreach(ownership_row IN LISTS ownership_rows) + if(NOT ownership_row MATCHES + "^([^\t]+)\t([^\t]+)\t([^\t]+)\t(YES|NO)$") + message(FATAL_ERROR "Malformed ownership row: ${ownership_row}") + endif() + set(expected_membership "${CMAKE_MATCH_3}\t${CMAKE_MATCH_1}") + list(FIND membership_rows "${expected_membership}" membership_index) + if(membership_index EQUAL -1) + message(FATAL_ERROR + "Owning aggregate membership is missing: ${expected_membership}") + endif() +endforeach() + +foreach(forbidden_membership IN ITEMS + "ExhaustiveArtifacts\tBenchmarkArtifacts" + "SimdLibSanitizerValidationArtifacts\tSimdLibCompilerContractArtifacts" + "SimdLibSanitizerValidationArtifacts\tSimdLibConstexprContractArtifacts" + "SimdLibSanitizerValidationArtifacts\tSimdLibOptimizedCodegenArtifacts" + "SimdLibSanitizerValidationArtifacts\tSimdLibDebugDiagnosticArtifacts" + "SimdLibCoverageValidationArtifacts\tSimdLibCompilerContractArtifacts" + "SimdLibCoverageValidationArtifacts\tSimdLibConstexprContractArtifacts" + "SimdLibCoverageValidationArtifacts\tSimdLibSmokeValidationArtifacts" + "SimdLibCoverageValidationArtifacts\tSimdLibOptimizedCodegenArtifacts" + "SimdLibCoverageValidationArtifacts\tSimdLibDebugDiagnosticArtifacts") + if(forbidden_membership IN_LIST membership_rows) + message(FATAL_ERROR + "Forbidden aggregate membership exists: ${forbidden_membership}") + endif() +endforeach() + +message(STATUS + "Validated scoped artifact ownership for profile ${PROFILE}: ${OWNERSHIP_FILE}") diff --git a/cmake/VerifyChecksConfiguration.cmake b/cmake/VerifyChecksConfiguration.cmake new file mode 100644 index 0000000..60a3a96 --- /dev/null +++ b/cmake/VerifyChecksConfiguration.cmake @@ -0,0 +1,91 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS PROPERTY_FILE DEFAULT_CHECKS_PROBE) + if(NOT DEFINED ${required_variable}) + message(FATAL_ERROR "Missing required variable ${required_variable}") + endif() +endforeach() +if(NOT EXISTS "${PROPERTY_FILE}") + message(FATAL_ERROR + "Checks-contract property inventory does not exist: ${PROPERTY_FILE}") +endif() + +file(STRINGS "${PROPERTY_FILE}" property_rows) +list(POP_FRONT property_rows property_header) +if(NOT property_header STREQUAL "target\tcompile_definitions\tsources") + message(FATAL_ERROR "Checks-contract property inventory has an invalid header") +endif() + +set(default_checks_target_count 0) +foreach(property_row IN LISTS property_rows) + if(NOT property_row MATCHES "^([^\t]+)\t([^\t]*)\t(.+)$") + message(FATAL_ERROR "Malformed checks-contract property row: ${property_row}") + endif() + set(target "${CMAKE_MATCH_1}") + set(compile_definitions "${CMAKE_MATCH_2}") + set(sources "${CMAKE_MATCH_3}") + + if(target STREQUAL "ConfigDefaultChecksDebugProbe") + math(EXPR default_checks_target_count "${default_checks_target_count} + 1") + if(NOT compile_definitions MATCHES + "(^|,)SIMDLIB_EXPECT_DEFAULT_CHECKS=1(,|$)") + message(FATAL_ERROR + "Debug default-checks probe has an invalid contract: ${property_row}") + endif() + string(REPLACE "," ";" source_list "${sources}") + list(GET source_list 0 source) + file(READ "${source}" source_text) + if(NOT source_text MATCHES + "SIMDLIB_EXPECT_DEFAULT_CHECKS && defined\\(NDEBUG\\)") + message(FATAL_ERROR + "Debug default-checks probe does not reject NDEBUG: ${source}") + endif() + elseif(target MATCHES "^(VectorChecksTests|PreconditionTests)$") + if(NOT compile_definitions MATCHES + "(^|,)SIMDLIB_ENABLE_CHECKS=1(,|$)") + message(FATAL_ERROR + "Checks target ${target} does not explicitly enable checks") + endif() + if(target STREQUAL "PreconditionTests") + string(REPLACE "," ";" source_list "${sources}") + list(GET source_list 0 source) + file(READ "${source}" source_text) + string(FIND "${source_text}" + "#define SIMDLIB_PRECONDITION" precondition_definition_position) + string(FIND "${source_text}" + "#include " api_include_position) + if(precondition_definition_position LESS 0 OR + api_include_position LESS 0 OR + NOT precondition_definition_position LESS api_include_position) + message(FATAL_ERROR + "PreconditionTests does not install its explicit failure hook before Api.h") + endif() + endif() + elseif(target STREQUAL "RegisterPreconditionTests") + string(REPLACE "," ";" source_list "${sources}") + list(GET source_list 0 source) + file(READ "${source}" source_text) + string(FIND "${source_text}" + "#define SIMDLIB_PRECONDITION" precondition_definition_position) + string(FIND "${source_text}" + "#include " register_include_position) + if(precondition_definition_position LESS 0 OR + register_include_position LESS 0 OR + NOT precondition_definition_position LESS register_include_position) + message(FATAL_ERROR + "RegisterPreconditionTests does not install its explicit failure hook before Register.h") + endif() + endif() +endforeach() + +if(DEFAULT_CHECKS_PROBE STREQUAL "DEBUG") + if(NOT default_checks_target_count EQUAL 1) + message(FATAL_ERROR + "The checks-enabled Debug profile requires exactly one default-checks probe") + endif() +elseif(NOT default_checks_target_count EQUAL 0) + message(FATAL_ERROR + "A Debug default-checks target exists while its profile is ${DEFAULT_CHECKS_PROBE}") +endif() + +message(STATUS "Validated explicit checks and precondition configuration") diff --git a/cmake/VerifyCodegenPolicySeparation.cmake b/cmake/VerifyCodegenPolicySeparation.cmake new file mode 100644 index 0000000..7f5514a --- /dev/null +++ b/cmake/VerifyCodegenPolicySeparation.cmake @@ -0,0 +1,73 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS SOURCE_DIRECTORY BINARY_DIRECTORY) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR + "VerifyCodegenPolicySeparation requires ${required_variable}") + endif() +endforeach() + +file(MAKE_DIRECTORY "${BINARY_DIRECTORY}") +set(wrapper_input "${BINARY_DIRECTORY}/wrapper.obj") +set(raw_input "${BINARY_DIRECTORY}/raw.obj") +set(tool_input "${BINARY_DIRECTORY}/objdump.exe") +set(record_file "${BINARY_DIRECTORY}/record.record.json") +set(record_index "${BINARY_DIRECTORY}/records.txt") +file(WRITE "${wrapper_input}" "wrapper\n") +file(WRITE "${raw_input}" "raw\n") +file(WRITE "${tool_input}" "tool\n") +file(SHA256 "${wrapper_input}" wrapper_hash) +file(SHA256 "${raw_input}" raw_hash) +file(SHA256 "${tool_input}" tool_hash) +foreach(path_variable IN ITEMS wrapper_input raw_input tool_input) + file(TO_CMAKE_PATH "${${path_variable}}" ${path_variable}_json) +endforeach() +file(WRITE "${record_file}" + "{\n" + " \"schema\": \"simdlib.codegen-record.v1\",\n" + " \"result\": \"recorded-diagnostic\",\n" + " \"policy\": {\"mode\": \"RECORD\"},\n" + " \"inputs\": {\n" + " \"wrapper\": {\"path\": \"${wrapper_input_json}\", \"sha256\": \"${wrapper_hash}\"},\n" + " \"raw\": {\"path\": \"${raw_input_json}\", \"sha256\": \"${raw_hash}\"}\n" + " },\n" + " \"tool\": {\"path\": \"${tool_input_json}\", \"sha256\": \"${tool_hash}\"}\n" + "}\n") +file(WRITE "${record_index}" "${record_file}\n") + +execute_process( + COMMAND "${CMAKE_COMMAND}" + "-DRECORD_INDEX=${record_index}" + -DEXPECTED_POLICY_MODE=RECORD + -DREQUIRE_RECORDS=ON + -P "${SOURCE_DIRECTORY}/cmake/ValidateCodegenRecords.cmake" + RESULT_VARIABLE record_result + OUTPUT_VARIABLE record_output + ERROR_VARIABLE record_error) +if(NOT record_result EQUAL 0) + message(FATAL_ERROR + "The record-only control validation failed unexpectedly:\n" + "${record_output}${record_error}") +endif() + +execute_process( + COMMAND "${CMAKE_COMMAND}" + "-DRECORD_INDEX=${record_index}" + -DEXPECTED_POLICY_MODE=ENFORCE + -DREQUIRE_RECORDS=ON + -P "${SOURCE_DIRECTORY}/cmake/ValidateCodegenRecords.cmake" + RESULT_VARIABLE enforce_result + OUTPUT_VARIABLE enforce_output + ERROR_VARIABLE enforce_error) +if(enforce_result EQUAL 0) + message(FATAL_ERROR + "A record-only result incorrectly satisfied ENFORCE validation") +endif() +if(NOT "${enforce_output}${enforce_error}" MATCHES + "policy is RECORD, expected ENFORCE") + message(FATAL_ERROR + "ENFORCE validation failed for an unexpected reason:\n" + "${enforce_output}${enforce_error}") +endif() + +message(STATUS "Validated RECORD and ENFORCE policy separation") diff --git a/cmake/VerifyCodegenProfileIsolation.cmake b/cmake/VerifyCodegenProfileIsolation.cmake new file mode 100644 index 0000000..a80fb3e --- /dev/null +++ b/cmake/VerifyCodegenProfileIsolation.cmake @@ -0,0 +1,68 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + BINARY_DIRECTORY OWNERSHIP_FILE PROFILE CODEGEN_MODE) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR + "VerifyCodegenProfileIsolation requires ${required_variable}") + endif() +endforeach() +if(NOT CODEGEN_MODE MATCHES "^(OFF|RECORD)$") + message(FATAL_ERROR "Unsupported isolation mode: ${CODEGEN_MODE}") +endif() +if(NOT EXISTS "${OWNERSHIP_FILE}") + message(FATAL_ERROR "Ownership inventory is missing: ${OWNERSHIP_FILE}") +endif() + +file(STRINGS "${OWNERSHIP_FILE}" ownership_rows) +list(POP_FRONT ownership_rows ownership_header) +if(NOT ownership_header STREQUAL + "target\tcategory\towning_aggregate\tselected") + message(FATAL_ERROR "Ownership inventory has an invalid header") +endif() + +set(codegen_target_count 0) +foreach(ownership_row IN LISTS ownership_rows) + if(NOT ownership_row MATCHES + "^([^\t]+)\t([^\t]+)\t([^\t]+)\t(YES|NO)$") + message(FATAL_ERROR "Malformed ownership row: ${ownership_row}") + endif() + set(target "${CMAKE_MATCH_1}") + set(category "${CMAKE_MATCH_2}") + if(category MATCHES "^(OPTIMIZED_CODEGEN|DEBUG_DIAGNOSTIC)$") + math(EXPR codegen_target_count "${codegen_target_count} + 1") + if(CODEGEN_MODE STREQUAL "OFF") + message(FATAL_ERROR + "Profile ${PROFILE} unexpectedly configures codegen target ${target}") + endif() + elseif(CODEGEN_MODE STREQUAL "RECORD") + message(FATAL_ERROR + "Diagnostic profile ${PROFILE} configures unrelated target ${target} " + "from ${category}") + endif() +endforeach() + +file(GLOB_RECURSE generated_codegen_files LIST_DIRECTORIES FALSE + "${BINARY_DIRECTORY}/register-codegen/*" + "${BINARY_DIRECTORY}/method-flags-codegen/*") +if(CODEGEN_MODE STREQUAL "OFF" AND generated_codegen_files) + list(GET generated_codegen_files 0 unexpected_codegen_file) + message(FATAL_ERROR + "Profile ${PROFILE} produced a generated-code artifact: " + "${unexpected_codegen_file}") +endif() +if(CODEGEN_MODE STREQUAL "RECORD") + if(codegen_target_count EQUAL 0) + message(FATAL_ERROR + "Diagnostic profile ${PROFILE} configures no codegen targets") + endif() + set(record_files ${generated_codegen_files}) + list(FILTER record_files INCLUDE REGEX "\\.record\\.json$") + if(NOT record_files) + message(FATAL_ERROR + "Diagnostic profile ${PROFILE} produced no codegen records") + endif() +endif() + +message(STATUS + "Validated codegen isolation for profile ${PROFILE} in mode ${CODEGEN_MODE}") diff --git a/cmake/VerifyCompilerContractIndependence.cmake b/cmake/VerifyCompilerContractIndependence.cmake new file mode 100644 index 0000000..e80a277 --- /dev/null +++ b/cmake/VerifyCompilerContractIndependence.cmake @@ -0,0 +1,94 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS PROPERTY_FILE SOURCE_FILE DEFAULT_CHECKS_PROBE) + if(NOT DEFINED ${required_variable}) + message(FATAL_ERROR "Missing required variable ${required_variable}") + endif() +endforeach() +if(NOT EXISTS "${PROPERTY_FILE}") + message(FATAL_ERROR + "Compiler-contract property inventory does not exist: ${PROPERTY_FILE}") +endif() +if(NOT EXISTS "${SOURCE_FILE}") + message(FATAL_ERROR + "Compiler-contract source inventory does not exist: ${SOURCE_FILE}") +endif() + +file(STRINGS "${PROPERTY_FILE}" property_rows) +list(POP_FRONT property_rows property_header) +if(NOT property_header STREQUAL + "target\tcompile_definitions\tcompile_options\tlink_options\tcxx_standard") + message(FATAL_ERROR "Compiler-contract property inventory has an invalid header") +endif() + +set(forbidden_property_pattern + "NDEBUG|SIMDLIB_ENABLE_CHECKS|fsanitize|sanitize=|fprofile|coverage|/RTC|\\$]*simdlib_codegen_shift_bytes_(left|right)_([0-9]+)[^>]*>:") + set(active_body_variable "body_${CMAKE_MATCH_1}_${CMAKE_MATCH_2}") + set(${active_body_variable} "") + elseif(NOT active_body_variable STREQUAL "") + string(APPEND ${active_body_variable} "${disassembly_line}\n") + if(disassembly_line MATCHES "[ \t]ret[qwl]?([ \t]|$)") + set(active_body_variable "") + endif() + endif() +endforeach() + +# @brief Requires one fixture body to contain an instruction fragment. +# @param body_variable Variable holding the disassembled function body. +# @param pattern Required regular expression. +function(simdlib_require_instruction body_variable pattern) + if(NOT DEFINED ${body_variable} OR NOT "${${body_variable}}" MATCHES "${pattern}") + message(FATAL_ERROR "${body_variable} does not contain required instruction pattern '${pattern}':\n${${body_variable}}") + endif() +endfunction() + +# @brief Rejects one instruction fragment from a fixture body. +# @param body_variable Variable holding the disassembled function body. +# @param pattern Forbidden regular expression. +function(simdlib_forbid_instruction body_variable pattern) + if(DEFINED ${body_variable} AND "${${body_variable}}" MATCHES "${pattern}") + message(FATAL_ERROR "${body_variable} contains forbidden instruction pattern '${pattern}':\n${${body_variable}}") + endif() +endfunction() + +set(common_counts 0 1 7 15 16 17) +if(REGISTER_WIDTH EQUAL 256) + list(APPEND common_counts 31 32) +endif() +foreach(count IN LISTS common_counts) + foreach(direction IN ITEMS left right) + set(body_variable "body_${direction}_${count}") + if(NOT DEFINED ${body_variable}) + message(FATAL_ERROR "Missing generated-code fixture ${body_variable} in ${OBJECT_FILE}") + endif() + simdlib_forbid_instruction(${body_variable} "(^|[ \t])(call|push|pop)[a-z]*[ \t]") + simdlib_forbid_instruction(${body_variable} "[%]?r(sp|bp)([^a-z0-9]|$)") + simdlib_forbid_instruction(${body_variable} "v?pshufb") + simdlib_forbid_instruction(${body_variable} "(^|[ \t])v?por[ \t]") + endforeach() +endforeach() + +foreach(direction IN ITEMS left right) + simdlib_forbid_instruction(body_${direction}_0 "v?p(sll|srl)dq|v?palignr|v?perm2i128|v?pxor|xorps|xorpd") +endforeach() + +if(REGISTER_WIDTH EQUAL 128) + foreach(count IN ITEMS 1 7 15) + simdlib_require_instruction(body_left_${count} "v?pslldq") + simdlib_require_instruction(body_right_${count} "v?psrldq") + endforeach() + foreach(count IN ITEMS 16 17) + foreach(direction IN ITEMS left right) + simdlib_require_instruction(body_${direction}_${count} "v?pxor|xorps|xorpd") + simdlib_forbid_instruction(body_${direction}_${count} "v?p(sll|srl)dq|v?palignr|v?perm2i128") + endforeach() + endforeach() +elseif(REGISTER_WIDTH EQUAL 256) + foreach(count IN ITEMS 1 7 15) + simdlib_require_instruction(body_left_${count} "vperm2(i|f)128") + simdlib_require_instruction(body_right_${count} "vperm2(i|f)128|vextract(i|f)128") + simdlib_require_instruction(body_left_${count} "vpalignr") + simdlib_require_instruction(body_right_${count} "vpalignr") + endforeach() + simdlib_require_instruction(body_left_16 "vperm2(i|f)128") + simdlib_require_instruction(body_right_16 "vperm2(i|f)128|vextract(i|f)128") + foreach(direction IN ITEMS left right) + simdlib_forbid_instruction(body_${direction}_16 "vpalignr|vpslldq|vpsrldq") + endforeach() + foreach(count IN ITEMS 17 31) + simdlib_require_instruction(body_left_${count} "vperm2(i|f)128") + simdlib_require_instruction(body_left_${count} "vpslldq") + simdlib_require_instruction(body_right_${count} "vperm2(i|f)128|vextract(i|f)128") + simdlib_require_instruction(body_right_${count} "vpsrldq") + endforeach() + foreach(direction IN ITEMS left right) + simdlib_require_instruction(body_${direction}_32 "vpxor|vxorps|vxorpd") + simdlib_forbid_instruction(body_${direction}_32 "vpalignr|vperm2(i|f)128|vpslldq|vpsrldq") + endforeach() +else() + message(FATAL_ERROR "Unsupported register width ${REGISTER_WIDTH}") +endif() + +file(WRITE "${OUTPUT_FILE}" "verified\n") diff --git a/cmake/VerifyMethodFlagsCodegen.cmake b/cmake/VerifyMethodFlagsCodegen.cmake new file mode 100644 index 0000000..9f39d89 --- /dev/null +++ b/cmake/VerifyMethodFlagsCodegen.cmake @@ -0,0 +1,81 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + FLAGGED_OBJECT RAW_OBJECT OBJDUMP COMPILER_ID STACK_PROTECTOR_MODE OUTPUT_FILE) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR "VerifyMethodFlagsCodegen requires ${required_variable}") + endif() +endforeach() + +# @brief Extracts one function and its relocation lines from an object disassembly. +# @param disassembly Complete disassembly text. +# @param symbol_fragment Stable fragment of the function name. +# @param output_variable Variable that receives the selected function body. +function(simdlib_extract_method_flags_symbol disassembly symbol_fragment output_variable) + string(REPLACE "\r\n" "\n" normalized "${disassembly}") + string(REPLACE "\n" ";" disassembly_lines "${normalized}") + set(selected "") + set(in_symbol OFF) + foreach(disassembly_line IN LISTS disassembly_lines) + if(disassembly_line MATCHES "<[^>]*${symbol_fragment}[^>]*>:") + set(in_symbol ON) + string(APPEND selected "${disassembly_line}\n") + elseif(in_symbol AND disassembly_line MATCHES "^[ \t]*[0-9A-Fa-f]+[ \t]+<[^>]+>:") + set(in_symbol OFF) + elseif(in_symbol) + string(APPEND selected "${disassembly_line}\n") + endif() + endforeach() + if(selected STREQUAL "") + message(FATAL_ERROR "Unable to find generated-code symbol ${symbol_fragment}") + endif() + set(${output_variable} "${selected}" PARENT_SCOPE) +endfunction() + +set(register_only_symbols + simdlib_method_flags_codegen_unary + simdlib_method_flags_codegen_binary + simdlib_method_flags_codegen_ternary + simdlib_method_flags_codegen_scalar_result + simdlib_method_flags_codegen_register_result + simdlib_method_flags_codegen_load + simdlib_method_flags_codegen_forceinline + simdlib_method_flags_codegen_flatten) + +foreach(object_file IN ITEMS "${FLAGGED_OBJECT}" "${RAW_OBJECT}") + execute_process( + COMMAND "${OBJDUMP}" -dr "${object_file}" + RESULT_VARIABLE disassembly_result + OUTPUT_VARIABLE disassembly + ERROR_VARIABLE disassembly_error) + if(NOT disassembly_result EQUAL 0) + message(FATAL_ERROR "Unable to disassemble ${object_file}: ${disassembly_error}") + endif() + + foreach(symbol_name IN LISTS register_only_symbols) + simdlib_extract_method_flags_symbol("${disassembly}" "${symbol_name}" symbol_body) + if(symbol_body MATCHES "security_(cookie|check_cookie)|stack_chk_(fail|guard)") + message(FATAL_ERROR "${symbol_name} acquired stack-cookie code in ${object_file}") + endif() + endforeach() + + if(disassembly MATCHES "call[^\n]*(\n[^\n]*)?simdlib_method_flags_force_leaf") + message(FATAL_ERROR "ForceInline did not inline its dedicated leaf in ${object_file}") + endif() + if(disassembly MATCHES "call[^\n]*(\n[^\n]*)?simdlib_method_flags_flatten_leaf") + message(FATAL_ERROR "Flatten did not inline its dedicated leaf in ${object_file}") + endif() + +endforeach() + +if(COMPILER_ID STREQUAL "MSVC" AND NOT STACK_PROTECTOR_MODE STREQUAL "msvc-gs") + message(FATAL_ERROR "MSVC method-flags codegen requires /GS stack protection") +endif() +if(NOT STACK_PROTECTOR_MODE MATCHES "^(strong|msvc-gs)$") + message(FATAL_ERROR "Method-flags codegen requires an explicit stack-protection mode") +endif() + +file(WRITE "${OUTPUT_FILE}" + "method_flags_codegen=verified\n" + "compiler_id=${COMPILER_ID}\n" + "stack_protector_mode=${STACK_PROTECTOR_MODE}\n") diff --git a/cmake/VerifyMethodFlagsCodegenRecords.cmake b/cmake/VerifyMethodFlagsCodegenRecords.cmake new file mode 100644 index 0000000..20a68dc --- /dev/null +++ b/cmake/VerifyMethodFlagsCodegenRecords.cmake @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.31) + +if(NOT DEFINED VERIFICATION_FILE OR "${VERIFICATION_FILE}" STREQUAL "") + message(FATAL_ERROR "VerifyMethodFlagsCodegenRecords requires VERIFICATION_FILE") +endif() +if(NOT EXISTS "${VERIFICATION_FILE}") + message(FATAL_ERROR "Method-flags generated-code verification is missing: ${VERIFICATION_FILE}") +endif() +include("${CMAKE_CURRENT_LIST_DIR}/ValidateCodegenRecords.cmake") diff --git a/cmake/VerifyMethodFlagsConfiguration.cmake b/cmake/VerifyMethodFlagsConfiguration.cmake new file mode 100644 index 0000000..ea7cfc4 --- /dev/null +++ b/cmake/VerifyMethodFlagsConfiguration.cmake @@ -0,0 +1,121 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + SIMDLIB_METHOD_FLAGS_COMPILER + SIMDLIB_METHOD_FLAGS_COMPILER_ID + SIMDLIB_METHOD_FLAGS_MSVC_STYLE + SIMDLIB_METHOD_FLAGS_SOURCE_DIR + SIMDLIB_METHOD_FLAGS_BINARY_DIR) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR "${required_variable} is required") + endif() +endforeach() + +set(probe_directory "${SIMDLIB_METHOD_FLAGS_BINARY_DIR}/method-flags-configuration") +file(MAKE_DIRECTORY "${probe_directory}") +set(probe_source "${probe_directory}/MethodFlagsConfigurationProbe.cpp") +set(actual_output "${probe_directory}/MethodFlagsConfigurationActual.txt") +set(method_flags_compiler_options) +if(DEFINED SIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS + AND NOT "${SIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS}" STREQUAL "") + separate_arguments(method_flags_compiler_options NATIVE_COMMAND + "${SIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS}") +endif() + +file(WRITE "${probe_source}" + "#define SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL 1\n" + "#define SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS 1\n" + "#define SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE 1\n" + "#define SIMDLIB_METHOD_FLAGS_HAS_FLATTEN 1\n" + "#define SIMDLIB_METHOD_FLAGS_VECTORCALL SIMDLIB_CONFIG_VECTORCALL\n" + "#define SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS SIMDLIB_CONFIG_SAFE_BUFFERS\n" + "#define SIMDLIB_METHOD_FLAGS_FORCE_INLINE SIMDLIB_CONFIG_FORCE_INLINE\n" + "#define SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_CONFIG_FLATTEN\n" + "#define SIMDLIB_PRECONDITION(condition, message)\n" + "#include \n" + "SIMDLIB_CONFIG_CAP_VECTORCALL SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL\n" + "SIMDLIB_CONFIG_CAP_SAFE_BUFFERS SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS\n" + "SIMDLIB_CONFIG_CAP_FORCE_INLINE SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE\n" + "SIMDLIB_CONFIG_CAP_FLATTEN SIMDLIB_METHOD_FLAGS_HAS_FLATTEN\n" + "SIMDLIB_CONFIG_CASE_NEITHER SIMD_FLAGS(Neither)\n" + "SIMDLIB_CONFIG_CASE_IN SIMD_FLAGS(In)\n" + "SIMDLIB_CONFIG_CASE_OUT SIMD_FLAGS(Out)\n" + "SIMDLIB_CONFIG_CASE_INOUT SIMD_FLAGS(InOut)\n" + "SIMDLIB_CONFIG_CASE_REGISTER_ONLY SIMD_FLAGS(Neither, RegisterOnly)\n" + "SIMDLIB_CONFIG_CASE_FORCE_INLINE SIMD_FLAGS(Neither, ForceInline)\n" + "SIMDLIB_CONFIG_CASE_FLATTEN SIMD_FLAGS(Neither, Flatten)\n" + "SIMDLIB_CONFIG_CASE_ALL SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten)\n") + +if(SIMDLIB_METHOD_FLAGS_MSVC_STYLE) + set(preprocess_arguments + /nologo + /std:c++20 + ${method_flags_compiler_options} + /EP + /TP + "/I${SIMDLIB_METHOD_FLAGS_SOURCE_DIR}/include" + "${probe_source}") +else() + set(preprocess_arguments + -std=c++20 + ${method_flags_compiler_options} + -E + -P + -x c++ + "-I${SIMDLIB_METHOD_FLAGS_SOURCE_DIR}/include" + "${probe_source}") +endif() + +execute_process( + COMMAND "${SIMDLIB_METHOD_FLAGS_COMPILER}" ${preprocess_arguments} + RESULT_VARIABLE preprocess_result + OUTPUT_VARIABLE preprocess_stdout + ERROR_VARIABLE preprocess_stderr) +file(WRITE "${actual_output}" "${preprocess_stdout}") +if(NOT preprocess_result EQUAL 0) + message(FATAL_ERROR + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID} configuration preprocessing failed:\n${preprocess_stderr}") +endif() + +set(expected_lines + "SIMDLIB_CONFIG_CAP_VECTORCALL 1" + "SIMDLIB_CONFIG_CAP_SAFE_BUFFERS 1" + "SIMDLIB_CONFIG_CAP_FORCE_INLINE 1" + "SIMDLIB_CONFIG_CAP_FLATTEN 1" + "SIMDLIB_CONFIG_CASE_NEITHER" + "SIMDLIB_CONFIG_CASE_IN SIMDLIB_CONFIG_VECTORCALL" + "SIMDLIB_CONFIG_CASE_OUT SIMDLIB_CONFIG_VECTORCALL" + "SIMDLIB_CONFIG_CASE_INOUT SIMDLIB_CONFIG_VECTORCALL" + "SIMDLIB_CONFIG_CASE_REGISTER_ONLY SIMDLIB_CONFIG_SAFE_BUFFERS" + "SIMDLIB_CONFIG_CASE_FORCE_INLINE SIMDLIB_CONFIG_FORCE_INLINE" + "SIMDLIB_CONFIG_CASE_FLATTEN SIMDLIB_CONFIG_FLATTEN" + "SIMDLIB_CONFIG_CASE_ALL SIMDLIB_CONFIG_FLATTEN SIMDLIB_CONFIG_FORCE_INLINE SIMDLIB_CONFIG_SAFE_BUFFERS SIMDLIB_CONFIG_VECTORCALL") + +string(REPLACE "\r\n" "\n" preprocess_stdout "${preprocess_stdout}") +string(REPLACE "\r" "\n" preprocess_stdout "${preprocess_stdout}") +string(REGEX MATCHALL "SIMDLIB_CONFIG_(CAP|CASE)_[A-Za-z0-9_]+[^\n]*" actual_lines + "${preprocess_stdout}") +list(LENGTH expected_lines expected_count) +list(LENGTH actual_lines actual_count) +if(NOT actual_count EQUAL expected_count) + message(FATAL_ERROR + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID} produced ${actual_count} configuration markers; expected ${expected_count}. See ${actual_output}") +endif() + +math(EXPR final_index "${expected_count} - 1") +foreach(index RANGE 0 ${final_index}) + list(GET expected_lines ${index} expected_line) + list(GET actual_lines ${index} actual_line) + string(STRIP "${actual_line}" actual_line) + string(REGEX REPLACE "[ \t]+" " " actual_line "${actual_line}") + if(NOT actual_line STREQUAL expected_line) + message(FATAL_ERROR + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID} configuration mismatch at marker ${index}:\n" + " expected: ${expected_line}\n" + " actual: ${actual_line}\n" + "See ${actual_output}") + endif() +endforeach() + +message(STATUS + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID}: verified ${expected_count} public method-flags adapter markers") diff --git a/cmake/VerifyMethodFlagsPreprocessor.cmake b/cmake/VerifyMethodFlagsPreprocessor.cmake new file mode 100644 index 0000000..2472632 --- /dev/null +++ b/cmake/VerifyMethodFlagsPreprocessor.cmake @@ -0,0 +1,258 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + SIMDLIB_METHOD_FLAGS_COMPILER + SIMDLIB_METHOD_FLAGS_COMPILER_ID + SIMDLIB_METHOD_FLAGS_MSVC_STYLE + SIMDLIB_METHOD_FLAGS_SOURCE_DIR + SIMDLIB_METHOD_FLAGS_BINARY_DIR) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR "${required_variable} is required") + endif() +endforeach() + +set(probe_directory "${SIMDLIB_METHOD_FLAGS_BINARY_DIR}/method-flags-preprocessor") +file(MAKE_DIRECTORY "${probe_directory}") +set(probe_source "${probe_directory}/MethodFlagsPreprocessorProbe.cpp") +set(expected_output "${probe_directory}/MethodFlagsPreprocessorExpected.txt") +set(actual_output "${probe_directory}/MethodFlagsPreprocessorActual.txt") + +set_property(GLOBAL PROPERTY SIMDLIB_METHOD_FLAGS_CASE_COUNT 0) +set_property(GLOBAL PROPERTY SIMDLIB_METHOD_FLAGS_PROBE_LINES "") +set_property(GLOBAL PROPERTY SIMDLIB_METHOD_FLAGS_EXPECTED_LINES "") + +# Adds one canonical boundary-and-modifier expansion to the generated fixture. +function(simdlib_add_method_flags_case boundary) + set(case_modifiers ${ARGN}) + set(case_flags ${boundary} ${case_modifiers}) + get_property(case_count GLOBAL PROPERTY SIMDLIB_METHOD_FLAGS_CASE_COUNT) + math(EXPR case_count "${case_count} + 1") + set_property(GLOBAL PROPERTY SIMDLIB_METHOD_FLAGS_CASE_COUNT "${case_count}") + + list(JOIN case_flags ", " invocation) + set(case_name "SIMDLIB_PP_CASE_${case_count}") + set(probe_line "${case_name} SIMD_FLAGS(${invocation})") + set(expected_line "${case_name}") + + list(FIND case_modifiers Flatten flatten_index) + if(NOT flatten_index EQUAL -1) + string(APPEND expected_line " SIMDLIB_PP_FLATTEN") + endif() + list(FIND case_modifiers ForceInline force_inline_index) + if(NOT force_inline_index EQUAL -1) + string(APPEND expected_line " SIMDLIB_PP_FORCE_INLINE") + endif() + list(FIND case_modifiers RegisterOnly register_only_index) + if(NOT register_only_index EQUAL -1) + string(APPEND expected_line " SIMDLIB_PP_REGISTER_ONLY") + endif() + if(NOT boundary STREQUAL "Neither") + string(APPEND expected_line " SIMDLIB_PP_VECTORCALL") + endif() + + set_property(GLOBAL APPEND PROPERTY SIMDLIB_METHOD_FLAGS_PROBE_LINES "${probe_line}") + set_property(GLOBAL APPEND PROPERTY SIMDLIB_METHOD_FLAGS_EXPECTED_LINES "${expected_line}") +endfunction() + +set(boundary_modes Neither In Out InOut) +foreach(boundary IN LISTS boundary_modes) + simdlib_add_method_flags_case(${boundary}) + simdlib_add_method_flags_case(${boundary} RegisterOnly) + simdlib_add_method_flags_case(${boundary} ForceInline) + simdlib_add_method_flags_case(${boundary} Flatten) + simdlib_add_method_flags_case(${boundary} RegisterOnly ForceInline) + simdlib_add_method_flags_case(${boundary} RegisterOnly Flatten) + simdlib_add_method_flags_case(${boundary} ForceInline Flatten) + simdlib_add_method_flags_case(${boundary} RegisterOnly ForceInline Flatten) +endforeach() + +# A function-like macro is not expanded when its name is passed as a bare flag. +# This case proves that only object-like collisions impose a caller restriction. +set_property(GLOBAL APPEND PROPERTY SIMDLIB_METHOD_FLAGS_PROBE_LINES + "#define InOut(...) downstream_function_macro" + "SIMDLIB_PP_CASE_FUNCTION_MACRO SIMD_FLAGS(InOut, Flatten)" + "#undef InOut") +set_property(GLOBAL APPEND PROPERTY SIMDLIB_METHOD_FLAGS_EXPECTED_LINES + "SIMDLIB_PP_CASE_FUNCTION_MACRO SIMDLIB_PP_FLATTEN SIMDLIB_PP_VECTORCALL") + +get_property(case_count GLOBAL PROPERTY SIMDLIB_METHOD_FLAGS_CASE_COUNT) +if(NOT case_count EQUAL 32) + message(FATAL_ERROR + "Expected 32 canonical boundary-and-modifier cases, generated ${case_count}") +endif() + +get_property(probe_lines GLOBAL PROPERTY SIMDLIB_METHOD_FLAGS_PROBE_LINES) +get_property(expected_lines GLOBAL PROPERTY SIMDLIB_METHOD_FLAGS_EXPECTED_LINES) +list(JOIN probe_lines "\n" probe_body) +list(JOIN expected_lines "\n" expected_body) + +file(WRITE "${probe_source}" + "#define SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL 1\n" + "#define SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS 1\n" + "#define SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE 1\n" + "#define SIMDLIB_METHOD_FLAGS_HAS_FLATTEN 1\n" + "#define SIMDLIB_METHOD_FLAGS_VECTORCALL SIMDLIB_PP_VECTORCALL\n" + "#define SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS SIMDLIB_PP_REGISTER_ONLY\n" + "#define SIMDLIB_METHOD_FLAGS_FORCE_INLINE SIMDLIB_PP_FORCE_INLINE\n" + "#define SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_PP_FLATTEN\n" + "#define SIMDLIB_PRECONDITION(condition, message)\n" + "#include \n" + "#if defined(Neither) || defined(In) || defined(Out) || defined(InOut) || defined(RegisterOnly) || defined(ForceInline) || defined(Flatten)\n" + "#error SIMDLIB_FLAGS_SHORT_MACRO_LEAK\n" + "#endif\n" + "${probe_body}\n") +file(WRITE "${expected_output}" "${expected_body}\n") + +if(SIMDLIB_METHOD_FLAGS_MSVC_STYLE) + set(preprocess_arguments + /nologo + /std:c++20 + ${SIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS} + /EP + /TP + "/I${SIMDLIB_METHOD_FLAGS_SOURCE_DIR}/include" + "${probe_source}") +else() + set(preprocess_arguments + -std=c++20 + ${SIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS} + -E + -P + -x c++ + "-I${SIMDLIB_METHOD_FLAGS_SOURCE_DIR}/include" + "${probe_source}") +endif() + +execute_process( + COMMAND "${SIMDLIB_METHOD_FLAGS_COMPILER}" ${preprocess_arguments} + RESULT_VARIABLE preprocess_result + OUTPUT_VARIABLE preprocess_stdout + ERROR_VARIABLE preprocess_stderr) +file(WRITE "${actual_output}" "${preprocess_stdout}") +if(NOT preprocess_result EQUAL 0) + message(FATAL_ERROR + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID} preprocessing failed:\n${preprocess_stderr}") +endif() + +string(REPLACE "\r\n" "\n" preprocess_stdout "${preprocess_stdout}") +string(REPLACE "\r" "\n" preprocess_stdout "${preprocess_stdout}") +string(REGEX MATCHALL "SIMDLIB_PP_CASE_[A-Za-z0-9_]+[^\n]*" actual_lines + "${preprocess_stdout}") +list(LENGTH expected_lines expected_count) +list(LENGTH actual_lines actual_count) +if(NOT actual_count EQUAL expected_count) + message(FATAL_ERROR + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID} produced ${actual_count} marker lines; " + "expected ${expected_count}. See ${actual_output}") +endif() + +math(EXPR final_index "${expected_count} - 1") +foreach(index RANGE 0 ${final_index}) + list(GET expected_lines ${index} expected_line) + list(GET actual_lines ${index} actual_line) + string(STRIP "${actual_line}" actual_line) + string(REGEX REPLACE "[ \t]+" " " actual_line "${actual_line}") + if(NOT actual_line STREQUAL expected_line) + message(FATAL_ERROR + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID} expansion mismatch at case ${index}:\n" + " expected: ${expected_line}\n" + " actual: ${actual_line}\n" + "See ${actual_output}") + endif() +endforeach() + +set(negative_sources + InvalidEmpty.cpp + InvalidUnknown.cpp + InvalidDuplicate.cpp + InvalidTooMany.cpp + InvalidObjectMacroCollision.cpp + InvalidMissingBoundary.cpp + InvalidModifierOrder.cpp) +set(negative_expansions + SIMDLIB_FLAGS_ERROR_EMPTY + SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_Unknown + SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_RegisterOnly_RegisterOnly + SIMDLIB_FLAGS_ERROR_TOO_MANY + SIMDLIB_DETAIL_FLAGS_BOUNDARY_downstream_object_macro + SIMDLIB_DETAIL_FLAGS_BOUNDARY_RegisterOnly + SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_Flatten_ForceInline) + +list(LENGTH negative_sources negative_count) +math(EXPR negative_final_index "${negative_count} - 1") +foreach(index RANGE 0 ${negative_final_index}) + list(GET negative_sources ${index} negative_source_name) + list(GET negative_expansions ${index} expected_expansion) + set(negative_source + "${SIMDLIB_METHOD_FLAGS_SOURCE_DIR}/tests/method_flags/${negative_source_name}") + set(negative_log "${probe_directory}/${negative_source_name}.log") + set(negative_object "${probe_directory}/${negative_source_name}.obj") + + if(SIMDLIB_METHOD_FLAGS_MSVC_STYLE) + set(negative_preprocess_arguments + /nologo + /std:c++20 + ${SIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS} + /EP + /TP + "/I${SIMDLIB_METHOD_FLAGS_SOURCE_DIR}/include" + "${negative_source}") + set(negative_arguments + /nologo + /std:c++20 + ${SIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS} + /TP + /c + "/I${SIMDLIB_METHOD_FLAGS_SOURCE_DIR}/include" + "/Fo${negative_object}" + "${negative_source}") + else() + set(negative_preprocess_arguments + -std=c++20 + ${SIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS} + -E + -P + -x c++ + "-I${SIMDLIB_METHOD_FLAGS_SOURCE_DIR}/include" + "${negative_source}") + set(negative_arguments + -std=c++20 + ${SIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS} + -fsyntax-only + -x c++ + "-I${SIMDLIB_METHOD_FLAGS_SOURCE_DIR}/include" + "${negative_source}") + endif() + + execute_process( + COMMAND "${SIMDLIB_METHOD_FLAGS_COMPILER}" ${negative_preprocess_arguments} + RESULT_VARIABLE negative_preprocess_result + OUTPUT_VARIABLE negative_preprocess_stdout + ERROR_VARIABLE negative_preprocess_stderr) + if(NOT negative_preprocess_result EQUAL 0) + message(FATAL_ERROR + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID} could not preprocess " + "${negative_source_name}:\n${negative_preprocess_stderr}") + endif() + if(NOT negative_preprocess_stdout MATCHES "${expected_expansion}") + message(FATAL_ERROR + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID} did not preserve " + "${expected_expansion} in ${negative_source_name}") + endif() + + execute_process( + COMMAND "${SIMDLIB_METHOD_FLAGS_COMPILER}" ${negative_arguments} + RESULT_VARIABLE negative_result + OUTPUT_VARIABLE negative_stdout + ERROR_VARIABLE negative_stderr) + set(negative_output "${negative_stdout}\n${negative_stderr}") + file(WRITE "${negative_log}" "${negative_output}") + if(negative_result EQUAL 0) + message(FATAL_ERROR + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID} unexpectedly accepted ${negative_source_name}") + endif() +endforeach() + +message(STATUS + "${SIMDLIB_METHOD_FLAGS_COMPILER_ID}: verified ${expected_count} canonical " + "expansions and ${negative_count} focused failures") diff --git a/cmake/VerifyPublicConsumptionProfile.cmake b/cmake/VerifyPublicConsumptionProfile.cmake new file mode 100644 index 0000000..21ae460 --- /dev/null +++ b/cmake/VerifyPublicConsumptionProfile.cmake @@ -0,0 +1,82 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS + OWNERSHIP_FILE CONSUMER_TARGET_FILE PROFILE REGISTER_SUPPORTED) + if(NOT DEFINED ${required_variable}) + message(FATAL_ERROR "Missing required variable ${required_variable}") + endif() +endforeach() + +foreach(required_file IN ITEMS OWNERSHIP_FILE CONSUMER_TARGET_FILE) + if(NOT EXISTS "${${required_file}}") + message(FATAL_ERROR + "Public-consumption inventory does not exist: ${${required_file}}") + endif() +endforeach() + +set(expected_smoke_targets + ApiExamples + FormatOdr + HeaderOnlySmoke) +set(expected_consumer_targets CoreConsumerSmoke) +if(REGISTER_SUPPORTED) + list(APPEND expected_smoke_targets + RegisterExamples + RegisterOdr) + list(APPEND expected_consumer_targets RegisterConsumerSmoke) +endif() +list(SORT expected_smoke_targets) +list(SORT expected_consumer_targets) + +file(STRINGS "${OWNERSHIP_FILE}" ownership_rows) +list(POP_FRONT ownership_rows ownership_header) +if(NOT ownership_header STREQUAL + "target\tcategory\towning_aggregate\tselected") + message(FATAL_ERROR "Ownership inventory has an invalid header") +endif() + +set(actual_smoke_targets "") +set(selected_smoke_targets "") +foreach(ownership_row IN LISTS ownership_rows) + if(NOT ownership_row MATCHES + "^([^\t]+)\t([^\t]+)\t([^\t]+)\t(YES|NO)$") + message(FATAL_ERROR "Malformed ownership row: ${ownership_row}") + endif() + if(CMAKE_MATCH_2 STREQUAL "SMOKE_VALIDATION") + list(APPEND actual_smoke_targets "${CMAKE_MATCH_1}") + if(CMAKE_MATCH_4 STREQUAL "YES") + list(APPEND selected_smoke_targets "${CMAKE_MATCH_1}") + endif() + endif() +endforeach() +list(SORT actual_smoke_targets) +list(SORT selected_smoke_targets) + +if(PROFILE STREQUAL "RELEASE") + if(NOT actual_smoke_targets STREQUAL expected_smoke_targets) + message(FATAL_ERROR + "Release public-surface targets differ from their compiler contract: " + "expected '${expected_smoke_targets}', received '${actual_smoke_targets}'") + endif() + if(NOT selected_smoke_targets STREQUAL expected_smoke_targets) + message(FATAL_ERROR + "Release did not select every public-surface target: " + "${selected_smoke_targets}") + endif() +elseif(NOT PROFILE STREQUAL "CUSTOM" AND (actual_smoke_targets OR selected_smoke_targets)) + message(FATAL_ERROR + "Profile ${PROFILE} configured Release-owned public-surface targets: " + "${actual_smoke_targets}") +endif() + +file(STRINGS "${CONSUMER_TARGET_FILE}" actual_consumer_targets) +list(SORT actual_consumer_targets) +if(NOT actual_consumer_targets STREQUAL expected_consumer_targets) + message(FATAL_ERROR + "External-consumer capability inventory differs from compiler support: " + "expected '${expected_consumer_targets}', received " + "'${actual_consumer_targets}'") +endif() + +message(STATUS + "Validated public-consumption ownership for profile ${PROFILE}") diff --git a/cmake/VerifyRuntimeTestInventory.cmake b/cmake/VerifyRuntimeTestInventory.cmake new file mode 100644 index 0000000..f1895de --- /dev/null +++ b/cmake/VerifyRuntimeTestInventory.cmake @@ -0,0 +1,83 @@ +cmake_minimum_required(VERSION 3.31) + +foreach(required_variable IN ITEMS TEST_DIRECTORY CMAKE_CTEST_COMMAND AUDIT_FILE REGISTER_REQUIRED) + if(NOT DEFINED ${required_variable} OR "${${required_variable}}" STREQUAL "") + message(FATAL_ERROR "VerifyRuntimeTestInventory requires ${required_variable}") + endif() +endforeach() +if(NOT REGISTER_REQUIRED MATCHES "^(ON|OFF)$") + message(FATAL_ERROR "REGISTER_REQUIRED must be ON or OFF") +endif() + +cmake_path(GET AUDIT_FILE PARENT_PATH audit_directory) +file(MAKE_DIRECTORY "${audit_directory}") +string(RANDOM LENGTH 16 ALPHABET 0123456789abcdef audit_temporary_suffix) +set(audit_temporary_file "${AUDIT_FILE}.${audit_temporary_suffix}.tmp") +file(WRITE "${audit_temporary_file}" + "schema=simdlib.runtime-test-inventory-audit.v1\n" + "test_directory=${TEST_DIRECTORY}\n" + "register_required=${REGISTER_REQUIRED}\n") + +# @brief Requires one CTest selection to contain at least one registered test. +# @param selection Stable audit name written to the evidence record. +# @param remaining_arguments CTest selection arguments such as --label-regex or --tests-regex. +function(simdlib_require_test_selection selection) + set(ctest_arguments --test-dir "${TEST_DIRECTORY}" -N) + if(DEFINED CONFIGURATION AND NOT "${CONFIGURATION}" STREQUAL "") + list(APPEND ctest_arguments -C "${CONFIGURATION}") + endif() + list(APPEND ctest_arguments ${ARGN}) + execute_process( + COMMAND "${CMAKE_CTEST_COMMAND}" ${ctest_arguments} + RESULT_VARIABLE ctest_result + OUTPUT_VARIABLE ctest_output + ERROR_VARIABLE ctest_error) + if(NOT ctest_result EQUAL 0) + message(FATAL_ERROR + "Unable to enumerate mandatory test selection ${selection}: ${ctest_error}") + endif() + + string(REPLACE "\r\n" "\n" ctest_output "${ctest_output}") + string(REGEX MATCH "Total Tests: ([0-9]+)" total_match "${ctest_output}") + if(NOT total_match OR CMAKE_MATCH_1 LESS 1) + message(FATAL_ERROR + "Mandatory runtime-test selection ${selection} is absent from ${TEST_DIRECTORY}") + endif() + file(APPEND "${audit_temporary_file}" "${selection}=${CMAKE_MATCH_1}\n") +endfunction() + +simdlib_require_test_selection(total) +foreach(required_label IN ITEMS AVX2 FMA BMI SCALAR) + simdlib_require_test_selection( + "label.${required_label}" --label-regex "^${required_label}$") +endforeach() + +set(required_test_families + "Api.SSE42|^Api\\.SSE42\\." + "Api.AVX2|^Api\\.AVX2\\." + "FMA.Enabled|^FMA\\.Enabled\\." + "FMA.Disabled|^FMA\\.Disabled\\." + "BmiPortable|^BmiPortable\\." + "UInt128Scalar|^UInt128Scalar\\.") +file(STRINGS "${TEST_DIRECTORY}/CMakeCache.txt" bmi_test_setting + REGEX "^SIMDLIB_BUILD_BMI_TESTS:BOOL=ON$") +if(bmi_test_setting) + list(APPEND required_test_families + "Bmi.Bmi1|^Bmi\\.Bmi1\\." + "Bmi.Bmi2|^Bmi\\.Bmi2\\." + "Bmi.Bmi1Bmi2|^Bmi\\.Bmi1Bmi2\\.") +endif() +if(REGISTER_REQUIRED) + list(APPEND required_test_families + "Register.SSE42|^Register\\.SSE42\\." + "Register.AVX2|^Register\\.AVX2\\.") +endif() +foreach(required_test_family IN LISTS required_test_families) + string(REPLACE "|" ";" family_fields "${required_test_family}") + list(GET family_fields 0 family_name) + list(GET family_fields 1 family_regex) + simdlib_require_test_selection( + "family.${family_name}" --tests-regex "${family_regex}") +endforeach() + +file(RENAME "${audit_temporary_file}" "${AUDIT_FILE}") diff --git a/cmake/development/ArtifactAggregates.cmake b/cmake/development/ArtifactAggregates.cmake new file mode 100644 index 0000000..a78c99f --- /dev/null +++ b/cmake/development/ArtifactAggregates.cmake @@ -0,0 +1,595 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "ArtifactAggregates.cmake is available only to top-level SimdLib builds") +endif() + +block(SCOPE_FOR VARIABLES) + +# @brief Collects every build-system target declared by project-owned directories. +# @param directory Configured directory whose targets and children are inspected. +# @param output_variable Variable that receives the recursively collected targets. +function(simdlib_collect_project_targets directory output_variable) + get_property(directory_targets DIRECTORY "${directory}" PROPERTY BUILDSYSTEM_TARGETS) + set(collected_targets ${directory_targets}) + + get_property(child_directories DIRECTORY "${directory}" PROPERTY SUBDIRECTORIES) + foreach(child_directory IN LISTS child_directories) + get_property(child_source_directory DIRECTORY "${child_directory}" PROPERTY SOURCE_DIR) + cmake_path(IS_PREFIX CMAKE_SOURCE_DIR "${child_source_directory}" + NORMALIZE child_is_project_owned) + cmake_path(RELATIVE_PATH child_source_directory + BASE_DIRECTORY "${CMAKE_SOURCE_DIR}" + OUTPUT_VARIABLE child_source_relative) + if(child_source_relative MATCHES + "^(out|_deps|\\.git)(/|$)|^build($|[-_/])") + set(child_is_project_owned FALSE) + endif() + if(child_is_project_owned) + simdlib_collect_project_targets("${child_directory}" child_targets) + list(APPEND collected_targets ${child_targets}) + endif() + endforeach() + + set(${output_variable} ${collected_targets} PARENT_SCOPE) +endfunction() + +# @brief Adds a globally unique aggregate for one validation category. +# @param aggregate Target name used by build profiles. +# @param category Sole target category owned by the aggregate. +function(simdlib_add_category_aggregate aggregate category) + add_custom_target(${aggregate}) + set(category_targets ${simdlib_targets_${category}}) + if(category_targets) + add_dependencies(${aggregate} ${category_targets}) + endif() + set_property(TARGET ${aggregate} PROPERTY + SIMDLIB_AGGREGATE_CATEGORY ${category}) +endfunction() + +set(simdlib_category_aggregate_COMPILER_CONTRACT + SimdLibCompilerContractArtifacts) +set(simdlib_category_aggregate_CONSTEXPR_CONTRACT + SimdLibConstexprContractArtifacts) +set(simdlib_category_aggregate_RUNTIME_VALIDATION + SimdLibRuntimeValidationArtifacts) +set(simdlib_category_aggregate_CHECKS_VALIDATION + SimdLibChecksValidationArtifacts) +set(simdlib_category_aggregate_SMOKE_VALIDATION + SimdLibSmokeValidationArtifacts) +set(simdlib_category_aggregate_OPTIMIZED_CODEGEN + SimdLibOptimizedCodegenArtifacts) +set(simdlib_category_aggregate_DEBUG_DIAGNOSTIC + SimdLibDebugDiagnosticArtifacts) +set(simdlib_category_aggregate_COVERAGE_SUPPORT + SimdLibCoverageSupportArtifacts) +set(simdlib_category_aggregate_BENCHMARK + BenchmarkArtifacts) + +set(simdlib_profile_allowed_CUSTOM ${SIMDLIB_VALIDATION_CATEGORIES}) +set(simdlib_profile_selected_CUSTOM + COMPILER_CONTRACT CONSTEXPR_CONTRACT + RUNTIME_VALIDATION CHECKS_VALIDATION SMOKE_VALIDATION + OPTIMIZED_CODEGEN DEBUG_DIAGNOSTIC) + +if(NOT SIMDLIB_VALIDATION_PROFILE STREQUAL "CUSTOM") + if(DEFINED SIMDLIB_SOURCE_DIRECTORY) + set(simdlib_validation_matrix_root "${SIMDLIB_SOURCE_DIRECTORY}") + else() + set(simdlib_validation_matrix_root "${CMAKE_SOURCE_DIR}") + endif() + set(simdlib_validation_matrix + "${simdlib_validation_matrix_root}/tools/validation-matrix.json") + if(NOT EXISTS "${simdlib_validation_matrix}") + message(FATAL_ERROR + "Validation matrix is missing: ${simdlib_validation_matrix}") + endif() + file(READ "${simdlib_validation_matrix}" simdlib_validation_matrix_json) + foreach(simdlib_profile_property IN ITEMS + allowedTargetCategories selectedTargetCategories) + string(JSON simdlib_profile_category_count + ERROR_VARIABLE simdlib_profile_error + LENGTH "${simdlib_validation_matrix_json}" + profiles "${SIMDLIB_VALIDATION_PROFILE}" + "${simdlib_profile_property}") + if(simdlib_profile_error) + message(FATAL_ERROR + "Validation matrix does not define ${simdlib_profile_property} " + "for profile ${SIMDLIB_VALIDATION_PROFILE}: " + "${simdlib_profile_error}") + endif() + set(simdlib_profile_categories "") + if(simdlib_profile_category_count GREATER 0) + math(EXPR simdlib_profile_category_last + "${simdlib_profile_category_count} - 1") + foreach(simdlib_profile_category_index RANGE + ${simdlib_profile_category_last}) + string(JSON simdlib_profile_category GET + "${simdlib_validation_matrix_json}" + profiles "${SIMDLIB_VALIDATION_PROFILE}" + "${simdlib_profile_property}" + ${simdlib_profile_category_index}) + list(APPEND simdlib_profile_categories + "${simdlib_profile_category}") + endforeach() + endif() + if(simdlib_profile_property STREQUAL "allowedTargetCategories") + set(simdlib_profile_allowed_${SIMDLIB_VALIDATION_PROFILE} + ${simdlib_profile_categories}) + else() + set(simdlib_profile_selected_${SIMDLIB_VALIDATION_PROFILE} + ${simdlib_profile_categories}) + endif() + endforeach() +endif() + +set(simdlib_allowed_categories + ${simdlib_profile_allowed_${SIMDLIB_VALIDATION_PROFILE}}) +set(simdlib_selected_categories + ${simdlib_profile_selected_${SIMDLIB_VALIDATION_PROFILE}}) +if(NOT simdlib_allowed_categories) + message(FATAL_ERROR + "No artifact ownership contract exists for profile " + "${SIMDLIB_VALIDATION_PROFILE}") +endif() + +if(SIMDLIB_BUILD_REGISTER_CODEGEN_GATES AND + SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "OFF") + message(FATAL_ERROR + "Register generated-code targets require ENFORCE or RECORD policy") +elseif(NOT SIMDLIB_BUILD_REGISTER_CODEGEN_GATES AND + NOT SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "OFF") + message(FATAL_ERROR + "Register generated-code policy must be OFF when its targets are disabled") +endif() + +if(SIMDLIB_VALIDATION_PROFILE STREQUAL "RELEASE") + foreach(simdlib_release_contract_option IN ITEMS + SIMDLIB_BUILD_CONFIGURATION_PROBES + SIMDLIB_BUILD_CONSTEXPR_PROBES + SIMDLIB_BUILD_HEADER_PROBES + SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES) + if(NOT ${simdlib_release_contract_option}) + message(FATAL_ERROR + "Release validation requires ${simdlib_release_contract_option}=ON") + endif() + endforeach() + if(NOT SIMDLIB_DEFAULT_CHECKS_PROBE STREQUAL "RELEASE") + message(FATAL_ERROR + "Release validation requires SIMDLIB_DEFAULT_CHECKS_PROBE=RELEASE") + endif() + if(SIMDLIB_REGISTER_COMPILER_SUPPORTED) + if(NOT SIMDLIB_BUILD_REGISTER_CODEGEN_GATES OR + NOT SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "ENFORCE") + message(FATAL_ERROR + "Register-capable Release validation requires enforced " + "Register generated-code gates") + endif() + elseif(SIMDLIB_BUILD_REGISTER_CODEGEN_GATES OR + NOT SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "OFF") + message(FATAL_ERROR + "Core-only Release validation cannot enable Register codegen") + endif() +elseif(SIMDLIB_VALIDATION_PROFILE STREQUAL "COMPILER_CONTRACTS") + if(NOT SIMDLIB_BUILD_CONFIGURATION_PROBES OR + NOT SIMDLIB_BUILD_HEADER_PROBES) + message(FATAL_ERROR + "Compiler-contract validation requires configuration and header probes") + endif() + if(NOT SIMDLIB_DEFAULT_CHECKS_PROBE STREQUAL "RELEASE") + message(FATAL_ERROR + "Compiler-contract validation requires the Release default-checks probe") + endif() + if(SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES) + message(FATAL_ERROR + "Compiler-contract validation excludes method-flags generated-code gates") + endif() +elseif(SIMDLIB_VALIDATION_PROFILE MATCHES + "^(DEBUG|SANITIZER|COVERAGE|CODEGEN_DIAGNOSTIC)$") + foreach(simdlib_forbidden_contract_option IN ITEMS + SIMDLIB_BUILD_CONFIGURATION_PROBES + SIMDLIB_BUILD_CONSTEXPR_PROBES + SIMDLIB_BUILD_HEADER_PROBES + SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES) + if(${simdlib_forbidden_contract_option}) + message(FATAL_ERROR + "Validation profile ${SIMDLIB_VALIDATION_PROFILE} excludes " + "${simdlib_forbidden_contract_option}") + endif() + endforeach() + foreach(simdlib_forbidden_public_surface_option IN ITEMS + SIMDLIB_BUILD_EXAMPLES + SIMDLIB_BUILD_SMOKE_TESTS) + if(${simdlib_forbidden_public_surface_option}) + message(FATAL_ERROR + "Validation profile ${SIMDLIB_VALIDATION_PROFILE} excludes " + "${simdlib_forbidden_public_surface_option}") + endif() + endforeach() + if(SIMDLIB_VALIDATION_PROFILE MATCHES "^(DEBUG|SANITIZER)$" AND + NOT SIMDLIB_DEFAULT_CHECKS_PROBE STREQUAL "DEBUG") + message(FATAL_ERROR + "Validation profile ${SIMDLIB_VALIDATION_PROFILE} requires the " + "checks-enabled Debug state probe") + endif() + if(SIMDLIB_VALIDATION_PROFILE STREQUAL "CODEGEN_DIAGNOSTIC") + if(NOT SIMDLIB_BUILD_REGISTER_CODEGEN_GATES OR + NOT SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "RECORD") + message(FATAL_ERROR + "Diagnostic codegen requires record-only Register generated-code targets") + endif() + elseif(SIMDLIB_BUILD_REGISTER_CODEGEN_GATES OR + NOT SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "OFF") + message(FATAL_ERROR + "Validation profile ${SIMDLIB_VALIDATION_PROFILE} excludes " + "Register generated-code targets and policy") + endif() +endif() + +simdlib_collect_project_targets("${CMAKE_CURRENT_SOURCE_DIR}" + simdlib_development_targets) +list(REMOVE_DUPLICATES simdlib_development_targets) +list(FILTER simdlib_development_targets EXCLUDE + REGEX "^(Continuous|Experimental|Nightly)") +list(SORT simdlib_development_targets) + +set(simdlib_owned_targets "") +foreach(simdlib_development_target IN LISTS simdlib_development_targets) + get_target_property(simdlib_development_target_type + ${simdlib_development_target} TYPE) + if(simdlib_development_target_type STREQUAL "INTERFACE_LIBRARY") + continue() + endif() + + get_target_property(simdlib_target_category + ${simdlib_development_target} SIMDLIB_VALIDATION_CATEGORY) + if(NOT simdlib_target_category) + message(FATAL_ERROR + "Development target ${simdlib_development_target} has no " + "validation-category owner") + endif() + if(NOT simdlib_target_category IN_LIST simdlib_allowed_categories) + message(FATAL_ERROR + "Development target ${simdlib_development_target} belongs to " + "${simdlib_target_category}, which is excluded by validation " + "profile ${SIMDLIB_VALIDATION_PROFILE}") + endif() + + list(APPEND simdlib_targets_${simdlib_target_category} + ${simdlib_development_target}) + list(APPEND simdlib_owned_targets ${simdlib_development_target}) +endforeach() + +foreach(simdlib_category IN LISTS SIMDLIB_VALIDATION_CATEGORIES) + list(SORT simdlib_targets_${simdlib_category}) + simdlib_add_category_aggregate( + ${simdlib_category_aggregate_${simdlib_category}} + ${simdlib_category}) + get_target_property(simdlib_aggregate_dependencies + ${simdlib_category_aggregate_${simdlib_category}} + MANUALLY_ADDED_DEPENDENCIES) + if(NOT simdlib_aggregate_dependencies) + set(simdlib_aggregate_dependencies "") + endif() + list(SORT simdlib_aggregate_dependencies) + if(NOT "${simdlib_aggregate_dependencies}" STREQUAL + "${simdlib_targets_${simdlib_category}}") + message(FATAL_ERROR + "Aggregate ${simdlib_category_aggregate_${simdlib_category}} " + "does not exactly own category ${simdlib_category}") + endif() +endforeach() + +add_custom_target(SimdLibSanitizerValidationArtifacts) +add_dependencies(SimdLibSanitizerValidationArtifacts + SimdLibRuntimeValidationArtifacts + SimdLibChecksValidationArtifacts) + +add_custom_target(SimdLibCoverageValidationArtifacts) +add_dependencies(SimdLibCoverageValidationArtifacts + SimdLibRuntimeValidationArtifacts + SimdLibChecksValidationArtifacts) + +add_custom_target(ExhaustiveArtifacts) +if(SIMDLIB_VALIDATION_PROFILE STREQUAL "SANITIZER") + add_dependencies(ExhaustiveArtifacts + SimdLibSanitizerValidationArtifacts) +elseif(SIMDLIB_VALIDATION_PROFILE STREQUAL "COVERAGE") + add_dependencies(ExhaustiveArtifacts + SimdLibCoverageValidationArtifacts) +else() + foreach(simdlib_selected_category IN LISTS simdlib_selected_categories) + add_dependencies(ExhaustiveArtifacts + ${simdlib_category_aggregate_${simdlib_selected_category}}) + endforeach() +endif() + +if(SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS) + set(simdlib_required_nonempty_categories ${simdlib_selected_categories}) + if(SIMDLIB_VALIDATION_PROFILE STREQUAL "RELEASE") + list(APPEND simdlib_required_nonempty_categories BENCHMARK) + endif() + foreach(simdlib_required_category IN LISTS simdlib_required_nonempty_categories) + if(NOT simdlib_targets_${simdlib_required_category}) + message(FATAL_ERROR + "Validation profile ${SIMDLIB_VALIDATION_PROFILE} requires " + "at least one ${simdlib_required_category} target") + endif() + endforeach() +endif() + +set(simdlib_ownership_rows + "target\tcategory\towning_aggregate\tselected") +set(simdlib_profile_targets "") +foreach(simdlib_owned_target IN LISTS simdlib_owned_targets) + get_target_property(simdlib_target_category + ${simdlib_owned_target} SIMDLIB_VALIDATION_CATEGORY) + if(simdlib_target_category IN_LIST simdlib_selected_categories) + set(simdlib_target_selected YES) + list(APPEND simdlib_profile_targets ${simdlib_owned_target}) + else() + set(simdlib_target_selected NO) + endif() + list(APPEND simdlib_ownership_rows + "${simdlib_owned_target}\t${simdlib_target_category}\t${simdlib_category_aggregate_${simdlib_target_category}}\t${simdlib_target_selected}") +endforeach() +list(SORT simdlib_profile_targets) + +set(simdlib_aggregate_targets + ExhaustiveArtifacts + SimdLibCompilerContractArtifacts + SimdLibConstexprContractArtifacts + SimdLibRuntimeValidationArtifacts + SimdLibChecksValidationArtifacts + SimdLibSmokeValidationArtifacts + SimdLibOptimizedCodegenArtifacts + SimdLibDebugDiagnosticArtifacts + SimdLibSanitizerValidationArtifacts + SimdLibCoverageValidationArtifacts + SimdLibCoverageSupportArtifacts + BenchmarkArtifacts) +list(SORT simdlib_aggregate_targets) + +set(simdlib_inventory_targets + ${simdlib_development_targets} ${simdlib_aggregate_targets}) +list(REMOVE_DUPLICATES simdlib_inventory_targets) +list(SORT simdlib_inventory_targets) +string(REPLACE ";" "\n" simdlib_development_target_inventory + "${simdlib_inventory_targets}") +file(WRITE "${CMAKE_BINARY_DIR}/development-targets.txt" + "${simdlib_development_target_inventory}\n") +string(REPLACE ";" "\n" simdlib_profile_target_inventory + "${simdlib_profile_targets}") +file(WRITE "${CMAKE_BINARY_DIR}/development-profile-targets.txt" + "${simdlib_profile_target_inventory}\n") +string(REPLACE ";" "\n" simdlib_ownership_inventory + "${simdlib_ownership_rows}") +file(WRITE "${CMAKE_BINARY_DIR}/development-target-ownership.tsv" + "${simdlib_ownership_inventory}\n") + +set(simdlib_compiler_contract_rows + "target\tcompile_definitions\tcompile_options\tlink_options\tcxx_standard") +set(simdlib_compiler_contract_source_rows "target\tsource") +foreach(simdlib_compiler_contract_target IN LISTS simdlib_targets_COMPILER_CONTRACT) + set(simdlib_contract_property_row "${simdlib_compiler_contract_target}") + foreach(simdlib_contract_property IN ITEMS + COMPILE_DEFINITIONS COMPILE_OPTIONS LINK_OPTIONS CXX_STANDARD) + get_target_property(simdlib_contract_property_value + ${simdlib_compiler_contract_target} ${simdlib_contract_property}) + if(NOT simdlib_contract_property_value) + set(simdlib_contract_property_value "") + endif() + string(REPLACE ";" "," simdlib_contract_property_value + "${simdlib_contract_property_value}") + string(REPLACE "\t" " " simdlib_contract_property_value + "${simdlib_contract_property_value}") + string(APPEND simdlib_contract_property_row + "\t${simdlib_contract_property_value}") + endforeach() + list(APPEND simdlib_compiler_contract_rows + "${simdlib_contract_property_row}") + + get_target_property(simdlib_contract_sources + ${simdlib_compiler_contract_target} SOURCES) + get_target_property(simdlib_contract_source_directory + ${simdlib_compiler_contract_target} SOURCE_DIR) + if(simdlib_contract_sources) + foreach(simdlib_contract_source IN LISTS simdlib_contract_sources) + if(simdlib_contract_source MATCHES "^\\$<") + message(FATAL_ERROR + "Compiler-contract target ${simdlib_compiler_contract_target} " + "uses a generated source expression") + endif() + cmake_path(ABSOLUTE_PATH simdlib_contract_source + BASE_DIRECTORY "${simdlib_contract_source_directory}" + NORMALIZE OUTPUT_VARIABLE simdlib_contract_source_absolute) + list(APPEND simdlib_compiler_contract_source_rows + "${simdlib_compiler_contract_target}\t${simdlib_contract_source_absolute}") + endforeach() + endif() +endforeach() +string(REPLACE ";" "\n" simdlib_compiler_contract_inventory + "${simdlib_compiler_contract_rows}") +file(WRITE "${CMAKE_BINARY_DIR}/compiler-contract-properties.tsv" + "${simdlib_compiler_contract_inventory}\n") +string(REPLACE ";" "\n" simdlib_compiler_contract_source_inventory + "${simdlib_compiler_contract_source_rows}") +file(WRITE "${CMAKE_BINARY_DIR}/compiler-contract-sources.tsv" + "${simdlib_compiler_contract_source_inventory}\n") + +set(simdlib_checks_contract_rows "target\tcompile_definitions\tsources") +foreach(simdlib_checks_contract_target IN LISTS simdlib_targets_CHECKS_VALIDATION) + get_target_property(simdlib_checks_contract_definitions + ${simdlib_checks_contract_target} COMPILE_DEFINITIONS) + if(NOT simdlib_checks_contract_definitions) + set(simdlib_checks_contract_definitions "") + endif() + string(REPLACE ";" "," simdlib_checks_contract_definitions + "${simdlib_checks_contract_definitions}") + + get_target_property(simdlib_checks_contract_sources + ${simdlib_checks_contract_target} SOURCES) + get_target_property(simdlib_checks_contract_source_directory + ${simdlib_checks_contract_target} SOURCE_DIR) + set(simdlib_checks_contract_absolute_sources "") + foreach(simdlib_checks_contract_source IN LISTS simdlib_checks_contract_sources) + cmake_path(ABSOLUTE_PATH simdlib_checks_contract_source + BASE_DIRECTORY "${simdlib_checks_contract_source_directory}" + NORMALIZE OUTPUT_VARIABLE simdlib_checks_contract_source_absolute) + list(APPEND simdlib_checks_contract_absolute_sources + "${simdlib_checks_contract_source_absolute}") + endforeach() + string(REPLACE ";" "," simdlib_checks_contract_absolute_sources + "${simdlib_checks_contract_absolute_sources}") + list(APPEND simdlib_checks_contract_rows + "${simdlib_checks_contract_target}\t${simdlib_checks_contract_definitions}\t${simdlib_checks_contract_absolute_sources}") +endforeach() +string(REPLACE ";" "\n" simdlib_checks_contract_inventory + "${simdlib_checks_contract_rows}") +file(WRITE "${CMAKE_BINARY_DIR}/checks-contract-properties.tsv" + "${simdlib_checks_contract_inventory}\n") + +set(simdlib_aggregate_rows "") +foreach(simdlib_category IN LISTS SIMDLIB_VALIDATION_CATEGORIES) + list(APPEND simdlib_aggregate_rows + "${simdlib_category_aggregate_${simdlib_category}}\t${simdlib_category}") +endforeach() +list(APPEND simdlib_aggregate_rows + "ExhaustiveArtifacts\tPROFILE:${SIMDLIB_VALIDATION_PROFILE}" + "SimdLibSanitizerValidationArtifacts\tPROFILE:SANITIZER" + "SimdLibCoverageValidationArtifacts\tPROFILE:COVERAGE") +list(SORT simdlib_aggregate_rows) +string(REPLACE ";" "\n" simdlib_aggregate_inventory + "${simdlib_aggregate_rows}") +file(WRITE "${CMAKE_BINARY_DIR}/development-aggregates.tsv" + "aggregate\tcategory\n${simdlib_aggregate_inventory}\n") + +set(simdlib_membership_rows "") +foreach(simdlib_category IN LISTS SIMDLIB_VALIDATION_CATEGORIES) + foreach(simdlib_category_target IN LISTS simdlib_targets_${simdlib_category}) + list(APPEND simdlib_membership_rows + "${simdlib_category_aggregate_${simdlib_category}}\t${simdlib_category_target}") + endforeach() +endforeach() +foreach(simdlib_profile_aggregate IN ITEMS + SimdLibSanitizerValidationArtifacts + SimdLibCoverageValidationArtifacts + ExhaustiveArtifacts) + get_target_property(simdlib_profile_dependencies + ${simdlib_profile_aggregate} MANUALLY_ADDED_DEPENDENCIES) + if(simdlib_profile_dependencies) + foreach(simdlib_profile_dependency IN LISTS simdlib_profile_dependencies) + list(APPEND simdlib_membership_rows + "${simdlib_profile_aggregate}\t${simdlib_profile_dependency}") + endforeach() + endif() +endforeach() +list(SORT simdlib_membership_rows) +string(REPLACE ";" "\n" simdlib_membership_inventory + "${simdlib_membership_rows}") +file(WRITE "${CMAKE_BINARY_DIR}/development-aggregate-membership.tsv" + "aggregate\tdependency\n${simdlib_membership_inventory}\n") + +set(simdlib_external_consumer_targets CoreConsumerSmoke) +if(SIMDLIB_REGISTER_COMPILER_SUPPORTED) + list(APPEND simdlib_external_consumer_targets RegisterConsumerSmoke) +endif() +string(REPLACE ";" "\n" simdlib_external_consumer_inventory + "${simdlib_external_consumer_targets}") +file(WRITE "${CMAKE_BINARY_DIR}/external-consumer-targets.txt" + "${simdlib_external_consumer_inventory}\n") + +if(BUILD_TESTING) + add_test(NAME ArtifactAggregates.ProfileMembership + COMMAND ${CMAKE_COMMAND} + "-DOWNERSHIP_FILE=${CMAKE_BINARY_DIR}/development-target-ownership.tsv" + "-DAGGREGATE_FILE=${CMAKE_BINARY_DIR}/development-aggregates.tsv" + "-DMEMBERSHIP_FILE=${CMAKE_BINARY_DIR}/development-aggregate-membership.tsv" + "-DPROFILE=${SIMDLIB_VALIDATION_PROFILE}" + "-DSELECTED_CATEGORIES=${simdlib_selected_categories}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyArtifactAggregateInventory.cmake) + set_tests_properties(ArtifactAggregates.ProfileMembership PROPERTIES + LABELS "CONFIGURATION;ARTIFACT_OWNERSHIP") + simdlib_register_development_test(ArtifactAggregates.ProfileMembership PROFILE_AUDIT) + + add_test(NAME ArtifactAggregates.PublicConsumption + COMMAND ${CMAKE_COMMAND} + "-DOWNERSHIP_FILE=${CMAKE_BINARY_DIR}/development-target-ownership.tsv" + "-DCONSUMER_TARGET_FILE=${CMAKE_BINARY_DIR}/external-consumer-targets.txt" + "-DPROFILE=${SIMDLIB_VALIDATION_PROFILE}" + "-DREGISTER_SUPPORTED=${SIMDLIB_REGISTER_COMPILER_SUPPORTED}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyPublicConsumptionProfile.cmake) + set_tests_properties(ArtifactAggregates.PublicConsumption PROPERTIES + LABELS "CONFIGURATION;ARTIFACT_OWNERSHIP;PUBLIC_CONSUMPTION") + simdlib_register_development_test(ArtifactAggregates.PublicConsumption PROFILE_AUDIT) + + if(simdlib_targets_COMPILER_CONTRACT) + add_test(NAME ArtifactAggregates.CompilerContractIndependence + COMMAND ${CMAKE_COMMAND} + "-DPROPERTY_FILE=${CMAKE_BINARY_DIR}/compiler-contract-properties.tsv" + "-DSOURCE_FILE=${CMAKE_BINARY_DIR}/compiler-contract-sources.tsv" + "-DDEFAULT_CHECKS_PROBE=${SIMDLIB_DEFAULT_CHECKS_PROBE}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyCompilerContractIndependence.cmake) + set_tests_properties( + ArtifactAggregates.CompilerContractIndependence PROPERTIES + LABELS "CONFIGURATION;ARTIFACT_OWNERSHIP;COMPILER_CONTRACT") + simdlib_register_development_test(ArtifactAggregates.CompilerContractIndependence PROFILE_AUDIT) + endif() + + if(simdlib_targets_CHECKS_VALIDATION) + add_test(NAME ArtifactAggregates.ChecksConfiguration + COMMAND ${CMAKE_COMMAND} + "-DPROPERTY_FILE=${CMAKE_BINARY_DIR}/checks-contract-properties.tsv" + "-DDEFAULT_CHECKS_PROBE=${SIMDLIB_DEFAULT_CHECKS_PROBE}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyChecksConfiguration.cmake) + set_tests_properties(ArtifactAggregates.ChecksConfiguration PROPERTIES + LABELS "CONFIGURATION;ARTIFACT_OWNERSHIP;CHECKS") + simdlib_register_development_test(ArtifactAggregates.ChecksConfiguration PROFILE_AUDIT) + endif() + + foreach(simdlib_failure_case IN ITEMS UNOWNED MULTIPLE EXCLUDED) + add_test(NAME ArtifactAggregates.Reject${simdlib_failure_case} + COMMAND ${CMAKE_COMMAND} + "-DCASE=${simdlib_failure_case}" + "-DSOURCE_DIRECTORY=${CMAKE_CURRENT_SOURCE_DIR}" + "-DBINARY_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR}/artifact-aggregate-negative/${simdlib_failure_case}" + "-DGENERATOR=${CMAKE_GENERATOR}" + "-DMAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyArtifactAggregateFailure.cmake) + set_tests_properties( + ArtifactAggregates.Reject${simdlib_failure_case} PROPERTIES + LABELS "CONFIGURATION;ARTIFACT_OWNERSHIP") + simdlib_register_development_test(ArtifactAggregates.Reject${simdlib_failure_case} PROFILE_AUDIT) + endforeach() + + if(SIMDLIB_VALIDATION_PROFILE MATCHES + "^(DEBUG|SANITIZER|COVERAGE|CODEGEN_DIAGNOSTIC)$") + set(simdlib_codegen_isolation_mode OFF) + if(SIMDLIB_VALIDATION_PROFILE STREQUAL "CODEGEN_DIAGNOSTIC") + set(simdlib_codegen_isolation_mode RECORD) + endif() + add_test(NAME ArtifactAggregates.CodegenIsolation + COMMAND ${CMAKE_COMMAND} + "-DBINARY_DIRECTORY=${CMAKE_BINARY_DIR}" + "-DOWNERSHIP_FILE=${CMAKE_BINARY_DIR}/development-target-ownership.tsv" + "-DPROFILE=${SIMDLIB_VALIDATION_PROFILE}" + "-DCODEGEN_MODE=${simdlib_codegen_isolation_mode}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyCodegenProfileIsolation.cmake) + set_tests_properties(ArtifactAggregates.CodegenIsolation PROPERTIES + LABELS "CONFIGURATION;ARTIFACT_OWNERSHIP;CODEGEN_ISOLATION") + simdlib_register_development_test(ArtifactAggregates.CodegenIsolation PROFILE_AUDIT) + endif() + + if(SIMDLIB_VALIDATION_PROFILE MATCHES "^(RELEASE|CODEGEN_DIAGNOSTIC)$") + add_test(NAME CodegenPolicy.RejectRecordAsEnforced + COMMAND ${CMAKE_COMMAND} + "-DSOURCE_DIRECTORY=${CMAKE_CURRENT_SOURCE_DIR}" + "-DBINARY_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR}/codegen-policy-separation" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyCodegenPolicySeparation.cmake) + set_tests_properties(CodegenPolicy.RejectRecordAsEnforced PROPERTIES + LABELS "CONFIGURATION;CODEGEN;CODEGEN_POLICY") + simdlib_register_development_test(CodegenPolicy.RejectRecordAsEnforced PROFILE_AUDIT) + endif() +endif() + +endblock() diff --git a/cmake/development/ArtifactOwnership.cmake b/cmake/development/ArtifactOwnership.cmake new file mode 100644 index 0000000..576b15e --- /dev/null +++ b/cmake/development/ArtifactOwnership.cmake @@ -0,0 +1,69 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "ArtifactOwnership.cmake is available only to top-level SimdLib builds") +endif() + +set(SIMDLIB_VALIDATION_CATEGORIES + COMPILER_CONTRACT + CONSTEXPR_CONTRACT + RUNTIME_VALIDATION + CHECKS_VALIDATION + SMOKE_VALIDATION + OPTIMIZED_CODEGEN + DEBUG_DIAGNOSTIC + COVERAGE_SUPPORT + BENCHMARK) + +# @brief Assigns one development target to its sole validation category. +# @param target Existing project-owned development target. +# @param category One value from SIMDLIB_VALIDATION_CATEGORIES. +function(simdlib_register_development_target target category) + if(NOT TARGET ${target}) + message(FATAL_ERROR + "Cannot assign validation ownership before target ${target} exists") + endif() + if(NOT category IN_LIST SIMDLIB_VALIDATION_CATEGORIES) + message(FATAL_ERROR + "Target ${target} uses unknown validation category ${category}") + endif() + + get_target_property(existing_category ${target} SIMDLIB_VALIDATION_CATEGORY) + if(existing_category) + message(FATAL_ERROR + "Target ${target} has multiple validation owners: " + "${existing_category} and ${category}") + endif() + + set_property(TARGET ${target} PROPERTY + SIMDLIB_VALIDATION_CATEGORY ${category}) +endfunction() +# @brief Assigns one configured CTest test to its sole validation owner. +# @param test Existing CTest test name. +# @param owner Validation target category or the PROFILE_AUDIT test-only owner. +function(simdlib_register_development_test test owner) + get_property(configured_tests DIRECTORY PROPERTY TESTS) + if(NOT test IN_LIST configured_tests) + message(FATAL_ERROR + "Cannot assign validation ownership before test ${test} exists") + endif() + if(NOT owner IN_LIST SIMDLIB_VALIDATION_CATEGORIES AND + NOT owner STREQUAL "PROFILE_AUDIT") + message(FATAL_ERROR + "Test ${test} uses unknown validation owner ${owner}") + endif() + + get_property(existing_labels TEST "${test}" PROPERTY LABELS) + set(existing_owner_labels ${existing_labels}) + list(FILTER existing_owner_labels INCLUDE + REGEX "^SIMDLIB_OWNER_") + if(existing_owner_labels) + message(FATAL_ERROR + "Test ${test} has multiple validation owners: " + "${existing_owner_labels} and ${owner}") + endif() + + list(APPEND existing_labels "SIMDLIB_OWNER_${owner}") + list(REMOVE_DUPLICATES existing_labels) + set_property(TEST "${test}" PROPERTY LABELS "${existing_labels}") +endfunction() diff --git a/cmake/development/Benchmarks.cmake b/cmake/development/Benchmarks.cmake new file mode 100644 index 0000000..5b9c7d5 --- /dev/null +++ b/cmake/development/Benchmarks.cmake @@ -0,0 +1,33 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "Benchmarks.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "Benchmarks.cmake requires the production SimdLib targets") +endif() + +block(SCOPE_FOR VARIABLES) + +if(SIMDLIB_BUILD_BENCHMARKS) + add_executable(Benchmarks benchmarks/Core.benchmarks.cpp) + simdlib_register_development_target(Benchmarks BENCHMARK) + target_link_libraries(Benchmarks PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) + if(SIMDLIB_REGISTER_COMPILER_SUPPORTED) + target_sources(Benchmarks PRIVATE benchmarks/Register.benchmarks.cpp) + target_link_libraries(Benchmarks PRIVATE SimdLib::Register) + endif() + simdlib_enable_development_warnings(Benchmarks) + target_compile_definitions(Benchmarks PRIVATE SIMDLIB_HAS_BMI1=1 SIMDLIB_HAS_BMI2=1) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_definitions(Benchmarks PRIVATE _SILENCE_CXX23_DENORM_DEPRECATION_WARNING) + target_compile_options(Benchmarks PRIVATE /arch:AVX2) + else() + target_compile_options(Benchmarks PRIVATE -mavx2 -mfma -mbmi -mbmi2) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(Benchmarks PRIVATE -Wno-deprecated-declarations) + endif() + endif() +endif() + +endblock() diff --git a/cmake/development/ConfigurationProbes.cmake b/cmake/development/ConfigurationProbes.cmake new file mode 100644 index 0000000..7d6c4c7 --- /dev/null +++ b/cmake/development/ConfigurationProbes.cmake @@ -0,0 +1,339 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "ConfigurationProbes.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "ConfigurationProbes.cmake requires the production SimdLib targets") +endif() + +block(SCOPE_FOR VARIABLES) + +if(SIMDLIB_BUILD_CONFIGURATION_PROBES) + if(SIMDLIB_MSVC_STYLE_DRIVER) + set(simdlib_method_flags_msvc_style 1) + else() + set(simdlib_method_flags_msvc_style 0) + endif() + + foreach(config_probe IN ITEMS + ConfigDefaultProbe + ConfigOverridePreconditionProbe + ConfigDisabledInstructionsProbe + ConfigDisabledPublicHeadersProbe + ConfigClangUnsupportedTargetProbe + ConfigVendorAttributeProbe + MethodFlagsConfigDefaultProbe + MethodFlagsConfigOverrideProbe + MethodFlagsConfigDisabledVectorcallProbe + MethodFlagsConfigUnsupportedTargetProbe) + add_library(${config_probe} OBJECT tests/config/${config_probe}.cpp) + simdlib_register_development_target(${config_probe} COMPILER_CONTRACT) + target_link_libraries(${config_probe} PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(${config_probe}) + endforeach() + + add_library(MethodFlagsContractPass OBJECT + tests/method_flags/MethodFlagsContractPass.cpp) + simdlib_register_development_target(MethodFlagsContractPass COMPILER_CONTRACT) + target_link_libraries(MethodFlagsContractPass PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(MethodFlagsContractPass) + + add_test(NAME MethodFlagsPreprocessor + COMMAND ${CMAKE_COMMAND} + "-DSIMDLIB_METHOD_FLAGS_COMPILER=${CMAKE_CXX_COMPILER}" + "-DSIMDLIB_METHOD_FLAGS_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}" + "-DSIMDLIB_METHOD_FLAGS_MSVC_STYLE=${simdlib_method_flags_msvc_style}" + "-DSIMDLIB_METHOD_FLAGS_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}" + "-DSIMDLIB_METHOD_FLAGS_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyMethodFlagsPreprocessor.cmake) + set_tests_properties(MethodFlagsPreprocessor PROPERTIES + LABELS "CONFIGURATION;METHOD_FLAGS;PREPROCESSOR") + simdlib_register_development_test(MethodFlagsPreprocessor COMPILER_CONTRACT) + + add_test(NAME MethodFlagsConfiguration + COMMAND ${CMAKE_COMMAND} + "-DSIMDLIB_METHOD_FLAGS_COMPILER=${CMAKE_CXX_COMPILER}" + "-DSIMDLIB_METHOD_FLAGS_COMPILER_ID=${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}" + "-DSIMDLIB_METHOD_FLAGS_MSVC_STYLE=${simdlib_method_flags_msvc_style}" + "-DSIMDLIB_METHOD_FLAGS_COMPILER_OPTIONS=${CMAKE_CXX_FLAGS}" + "-DSIMDLIB_METHOD_FLAGS_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}" + "-DSIMDLIB_METHOD_FLAGS_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyMethodFlagsConfiguration.cmake) + set_tests_properties(MethodFlagsConfiguration PROPERTIES + LABELS "CONFIGURATION;METHOD_FLAGS;ADAPTERS;PREPROCESSOR") + simdlib_register_development_test(MethodFlagsConfiguration COMPILER_CONTRACT) + + add_subdirectory( + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/placement + ${CMAKE_CURRENT_BINARY_DIR}/method-flags-placement) +endif() + +if(SIMDLIB_BUILD_CONSTEXPR_PROBES) + add_library(ConstexprProbe OBJECT tests/config/ConstexprProbe.cpp) + simdlib_register_development_target(ConstexprProbe CONSTEXPR_CONTRACT) + target_link_libraries(ConstexprProbe PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(ConstexprProbe) +endif() + +# @brief Adds a compile-only language-availability probe with an exact standard mode. +# @param target Target name used in compiler diagnostics. +# @param source Translation unit containing the availability assertions. +# @param standard C++ standard level requested for the probe. +# @param dependency Public SimdLib target whose usage requirements are under test. +function(simdlib_add_language_probe target source standard dependency) + add_library(${target} OBJECT ${source}) + simdlib_register_development_target(${target} COMPILER_CONTRACT) + target_link_libraries(${target} PRIVATE ${dependency}) + set_target_properties(${target} PROPERTIES + CXX_STANDARD ${standard} + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF) + simdlib_enable_development_warnings(${target}) +endfunction() + +# @brief Verifies that one intentionally invalid translation unit fails with the focused diagnostic. +# @param probe_name Stable name used for the try-compile directory and log. +# @param source Translation unit that must fail to compile. +# @param standard Exact C++ standard level used for the negative probe. +# @param expected_diagnostic Stable diagnostic token required in compiler output. +function(simdlib_expect_language_probe_failure probe_name source standard expected_diagnostic) + try_compile(probe_compiled + SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/${source} + NO_CACHE + CXX_STANDARD ${standard} + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF + CMAKE_FLAGS + -DINCLUDE_DIRECTORIES=${CMAKE_CURRENT_SOURCE_DIR}/include + OUTPUT_VARIABLE probe_output) + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${probe_name}.log" "${probe_output}") + if(probe_compiled) + message(FATAL_ERROR "${probe_name} unexpectedly compiled successfully") + endif() + if(NOT probe_output MATCHES "${expected_diagnostic}") + message(FATAL_ERROR + "${probe_name} did not emit ${expected_diagnostic}; see ${CMAKE_CURRENT_BINARY_DIR}/${probe_name}.log") + endif() +endfunction() + +if(SIMDLIB_BUILD_CONFIGURATION_PROBES) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/include/SimdLib/Config.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/SimdLib/Register.h + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyMethodFlagsConfiguration.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyMethodFlagsPreprocessor.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/MethodFlagsPrototype.h + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/MethodFlagsContractPass.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/InvalidEmpty.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/InvalidUnknown.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/InvalidDuplicate.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/InvalidTooMany.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/InvalidObjectMacroCollision.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/InvalidMissingBoundary.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/InvalidModifierOrder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/placement/CMakeLists.txt + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/placement/MethodFlagsPlacementFixture.h + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/placement/MethodFlagsPlacementCxx20.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/placement/MethodFlagsPlacementCxx23.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/placement/MethodFlagsPlacementAbiDefinition.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/method_flags/placement/MethodFlagsPlacementAbiConsumer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/config/MethodFlagsConfigDefaultProbe.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/config/MethodFlagsConfigOverrideProbe.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/config/MethodFlagsConfigDisabledVectorcallProbe.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/config/MethodFlagsConfigUnsupportedTargetProbe.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterHeaderCxx20.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterRequirementCxx20.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterAvailabilityOverride.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterUnsupportedCompiler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterPartialLaneList.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterOversizedLaneList.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterDynamicTransfer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterImplicitScalar.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterImplicitNative.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterNativeOrder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterUninitialized.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterInvalidShuffleSelector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterWrongShuffleSelectorCount.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterInvalidByteShuffleSelector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterWrongByteShuffleSelectorCount.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/api/ApiInvalidShuffleSelector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/api/ApiWrongShuffleSelectorCount.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/api/ApiUnsuffixedRuntimeImmediate.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterUnsuffixedRuntimeImmediate.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/availability/ImmediateControlSlowPathProbe.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/availability/CompleteRegisterShiftProbe.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/api/ApiNegativeCompleteByteShift.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterNegativeCompleteByteShift.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterInvalidRearrangementImmediate.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterUnsupportedConversionTarget.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterUnavailableWidthChange.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterCompatibilityRearrangement.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/compile_fail/register/RegisterCollectionOperations.cpp) + + simdlib_add_language_probe(RegisterCxx20UmbrellaProbe + tests/availability/RegisterCxx20UmbrellaProbe.cpp 20 SimdLib::SimdLib) + + simdlib_add_language_probe(ImmediateControlSlowPathProbe + tests/availability/ImmediateControlSlowPathProbe.cpp 20 SimdLib::SimdLib) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(ImmediateControlSlowPathProbe PRIVATE /arch:AVX2) + else() + target_compile_options(ImmediateControlSlowPathProbe PRIVATE -mavx2) + endif() + + if(SIMDLIB_REGISTER_COMPILER_SUPPORTED) + simdlib_add_language_probe(CompleteRegisterShiftProbe + tests/availability/CompleteRegisterShiftProbe.cpp 23 SimdLib::Register) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(CompleteRegisterShiftProbe PRIVATE /arch:AVX2) + else() + target_compile_options(CompleteRegisterShiftProbe PRIVATE -mavx2) + endif() + simdlib_expect_language_probe_failure(RegisterNegativeCompleteByteShiftFailure + tests/compile_fail/register/RegisterNegativeCompleteByteShift.cpp 23 + shift_bytes_left) + + simdlib_add_language_probe(RegisterEnabledProbe + tests/availability/RegisterEnabledProbe.cpp 23 SimdLib::Register) + + foreach(register_width IN ITEMS 128 256) + add_library(RegisterRepresentation${register_width} OBJECT + tests/register/RegisterRepresentation.tests.cpp) + simdlib_register_development_target( + RegisterRepresentation${register_width} COMPILER_CONTRACT) + target_link_libraries(RegisterRepresentation${register_width} PRIVATE SimdLib::Register) + target_compile_definitions(RegisterRepresentation${register_width} PRIVATE + SIMDLIB_REGISTER_TEST_WIDTH=${register_width}) + simdlib_enable_development_warnings(RegisterRepresentation${register_width}) + if(register_width EQUAL 128) + simdlib_enable_register_sse42(RegisterRepresentation${register_width}) + else() + simdlib_enable_register_avx2(RegisterRepresentation${register_width}) + endif() + endforeach() + + simdlib_expect_language_probe_failure(RegisterPartialLaneListFailure + tests/compile_fail/register/RegisterPartialLaneList.cpp 23 + SIMDLIB_REGISTER_REJECTS_PARTIAL_LANE_LIST) + simdlib_expect_language_probe_failure(RegisterOversizedLaneListFailure + tests/compile_fail/register/RegisterOversizedLaneList.cpp 23 + SIMDLIB_REGISTER_REJECTS_OVERSIZED_LANE_LIST) + simdlib_expect_language_probe_failure(RegisterDynamicTransferFailure + tests/compile_fail/register/RegisterDynamicTransfer.cpp 23 + SIMDLIB_REGISTER_REJECTS_DYNAMIC_TRANSFER) + simdlib_expect_language_probe_failure(RegisterImplicitScalarFailure + tests/compile_fail/register/RegisterImplicitScalar.cpp 23 + SIMDLIB_REGISTER_REJECTS_IMPLICIT_SCALAR) + simdlib_expect_language_probe_failure(RegisterImplicitNativeFailure + tests/compile_fail/register/RegisterImplicitNative.cpp 23 + SIMDLIB_REGISTER_REJECTS_IMPLICIT_NATIVE) + simdlib_expect_language_probe_failure(RegisterNativeOrderFailure + tests/compile_fail/register/RegisterNativeOrder.cpp 23 + SIMDLIB_REGISTER_REJECTS_NATIVE_ORDER_CONSTRUCTION) + simdlib_expect_language_probe_failure(RegisterUninitializedFailure + tests/compile_fail/register/RegisterUninitialized.cpp 23 + SIMDLIB_REGISTER_REJECTS_UNINITIALIZED_CONSTRUCTION) + simdlib_expect_language_probe_failure(RegisterInvalidShuffleSelectorFailure + tests/compile_fail/register/RegisterInvalidShuffleSelector.cpp 23 + SIMDLIB_REGISTER_REJECTS_INVALID_SHUFFLE_SELECTOR) + simdlib_expect_language_probe_failure(RegisterWrongShuffleSelectorCountFailure + tests/compile_fail/register/RegisterWrongShuffleSelectorCount.cpp 23 + SIMDLIB_REGISTER_REJECTS_WRONG_SHUFFLE_SELECTOR_COUNT) + simdlib_expect_language_probe_failure(RegisterInvalidByteShuffleSelectorFailure + tests/compile_fail/register/RegisterInvalidByteShuffleSelector.cpp 23 + SIMDLIB_REGISTER_REJECTS_INVALID_BYTE_SHUFFLE_SELECTOR) + simdlib_expect_language_probe_failure(RegisterWrongByteShuffleSelectorCountFailure + tests/compile_fail/register/RegisterWrongByteShuffleSelectorCount.cpp 23 + SIMDLIB_REGISTER_REJECTS_WRONG_BYTE_SHUFFLE_SELECTOR_COUNT) + simdlib_expect_language_probe_failure(RegisterInvalidRearrangementImmediateFailure + tests/compile_fail/register/RegisterInvalidRearrangementImmediate.cpp 23 + SIMDLIB_REGISTER_REJECTS_INVALID_REARRANGEMENT_IMMEDIATE) + simdlib_expect_language_probe_failure(RegisterUnsupportedConversionTargetFailure + tests/compile_fail/register/RegisterUnsupportedConversionTarget.cpp 23 + SIMDLIB_REGISTER_REJECTS_UNSUPPORTED_CONVERSION_TARGET) + simdlib_expect_language_probe_failure(RegisterUnavailableWidthChangeFailure + tests/compile_fail/register/RegisterUnavailableWidthChange.cpp 23 + SIMDLIB_REGISTER_REJECTS_UNAVAILABLE_WIDTH_CHANGE) + simdlib_expect_language_probe_failure(RegisterCompatibilityRearrangementFailure + tests/compile_fail/register/RegisterCompatibilityRearrangement.cpp 23 + SIMDLIB_REGISTER_REJECTS_COMPATIBILITY_REARRANGEMENT) + simdlib_expect_language_probe_failure(RegisterCollectionOperationsFailure + tests/compile_fail/register/RegisterCollectionOperations.cpp 23 + SIMDLIB_REGISTER_REJECTS_COLLECTION_OPERATIONS) + simdlib_expect_language_probe_failure(RegisterUnsuffixedRuntimeImmediateFailure + tests/compile_fail/register/RegisterUnsuffixedRuntimeImmediate.cpp 23 + SIMDLIB_REGISTER_REJECTS_UNSUFFIXED_RUNTIME_IMMEDIATE_CONTROLS) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + simdlib_add_language_probe(RegisterMsvcFallbackProbe + tests/availability/RegisterMsvcFallbackProbe.cpp 23 SimdLib::Register) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND SIMDLIB_MSVC_STYLE_DRIVER) + simdlib_add_language_probe(RegisterClangClFallbackExclusionProbe + tests/availability/RegisterClangClFallbackExclusionProbe.cpp 23 SimdLib::SimdLib) + endif() + endif() + + simdlib_expect_language_probe_failure(RegisterHeaderCxx20Failure + tests/compile_fail/register/RegisterHeaderCxx20.cpp 20 + SIMDLIB_REGISTER_HEADER_REQUIRES_CXX23) + simdlib_expect_language_probe_failure(RegisterRequirementCxx20Failure + tests/compile_fail/register/RegisterRequirementCxx20.cpp 20 + SIMDLIB_REGISTER_INTERFACE_UNAVAILABLE) + simdlib_expect_language_probe_failure(RegisterAvailabilityOverrideFailure + tests/compile_fail/register/RegisterAvailabilityOverride.cpp 20 + SIMDLIB_REGISTER_INTERFACE_AVAILABILITY_IS_COMPUTED) + simdlib_expect_language_probe_failure(ApiInvalidShuffleSelectorFailure + tests/compile_fail/api/ApiInvalidShuffleSelector.cpp 20 + SIMDLIB_API_REJECTS_INVALID_SHUFFLE_SELECTOR) + simdlib_expect_language_probe_failure(ApiWrongShuffleSelectorCountFailure + tests/compile_fail/api/ApiWrongShuffleSelectorCount.cpp 20 + SIMDLIB_API_REJECTS_WRONG_SHUFFLE_SELECTOR_COUNT) + simdlib_expect_language_probe_failure(ApiUnsuffixedRuntimeImmediateFailure + tests/compile_fail/api/ApiUnsuffixedRuntimeImmediate.cpp 20 + SIMDLIB_REJECTS_UNSUFFIXED_RUNTIME_IMMEDIATE_CONTROLS) + simdlib_expect_language_probe_failure(ApiNegativeCompleteByteShiftFailure + tests/compile_fail/api/ApiNegativeCompleteByteShift.cpp 20 + shift_bytes_left) + if(NOT SIMDLIB_REGISTER_COMPILER_SUPPORTED) + simdlib_expect_language_probe_failure(RegisterUnsupportedCompilerFailure + tests/compile_fail/register/RegisterUnsupportedCompiler.cpp 23 + SIMDLIB_REGISTER_INTERFACE_UNAVAILABLE) + endif() +endif() + +if(SIMDLIB_BUILD_CONSTEXPR_PROBES AND SIMDLIB_REGISTER_COMPILER_SUPPORTED) + foreach(register_width IN ITEMS 128 256) + add_library(RegisterConstexpr${register_width}Probe OBJECT + tests/constexpr/RegisterConstexpr.tests.cpp) + simdlib_register_development_target( + RegisterConstexpr${register_width}Probe CONSTEXPR_CONTRACT) + target_link_libraries(RegisterConstexpr${register_width}Probe PRIVATE SimdLib::Register) + target_compile_definitions(RegisterConstexpr${register_width}Probe PRIVATE + SIMDLIB_REGISTER_TEST_WIDTH=${register_width}) + simdlib_enable_development_warnings(RegisterConstexpr${register_width}Probe) + if(register_width EQUAL 128) + simdlib_enable_register_sse42(RegisterConstexpr${register_width}Probe) + else() + simdlib_enable_register_avx2(RegisterConstexpr${register_width}Probe) + endif() + endforeach() +endif() + +if(SIMDLIB_BUILD_CONFIGURATION_PROBES) + add_library(AvailabilityDisabledProbe OBJECT tests/availability/ApiDisabledProbe.cpp) + simdlib_register_development_target(AvailabilityDisabledProbe COMPILER_CONTRACT) + target_link_libraries(AvailabilityDisabledProbe PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(AvailabilityDisabledProbe) + + add_library(AvailabilityEnabledProbe OBJECT tests/availability/ApiEnabledProbe.cpp) + simdlib_register_development_target(AvailabilityEnabledProbe COMPILER_CONTRACT) + target_link_libraries(AvailabilityEnabledProbe PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(AvailabilityEnabledProbe) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(AvailabilityEnabledProbe PRIVATE /arch:AVX2) + else() + target_compile_options(AvailabilityEnabledProbe PRIVATE -mavx2) + endif() +endif() + +endblock() diff --git a/cmake/development/ConfigurationStateProbes.cmake b/cmake/development/ConfigurationStateProbes.cmake new file mode 100644 index 0000000..c77f958 --- /dev/null +++ b/cmake/development/ConfigurationStateProbes.cmake @@ -0,0 +1,39 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "ConfigurationStateProbes.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib) + message(FATAL_ERROR "ConfigurationStateProbes.cmake requires the production SimdLib target") +endif() + +block(SCOPE_FOR VARIABLES) + +if(NOT SIMDLIB_DEFAULT_CHECKS_PROBE STREQUAL "NONE") + if(SIMDLIB_DEFAULT_CHECKS_PROBE STREQUAL "RELEASE") + set(default_checks_target ConfigDefaultChecksReleaseProbe) + set(default_checks_expected 0) + elseif(SIMDLIB_DEFAULT_CHECKS_PROBE STREQUAL "DEBUG") + set(default_checks_target ConfigDefaultChecksDebugProbe) + set(default_checks_expected 1) + else() + message(FATAL_ERROR + "Unsupported default-checks probe ${SIMDLIB_DEFAULT_CHECKS_PROBE}") + endif() + + add_library(${default_checks_target} OBJECT + tests/config/ConfigDefaultChecksProbe.cpp) + if(SIMDLIB_DEFAULT_CHECKS_PROBE STREQUAL "DEBUG") + simdlib_register_development_target(${default_checks_target} + CHECKS_VALIDATION) + else() + simdlib_register_development_target(${default_checks_target} + COMPILER_CONTRACT) + endif() + target_link_libraries(${default_checks_target} PRIVATE SimdLib::SimdLib) + target_compile_definitions(${default_checks_target} PRIVATE + SIMDLIB_EXPECT_DEFAULT_CHECKS=${default_checks_expected}) + simdlib_enable_development_warnings(${default_checks_target}) +endif() + +endblock() diff --git a/cmake/development/ConstexprProbes.cmake b/cmake/development/ConstexprProbes.cmake new file mode 100644 index 0000000..7cf5241 --- /dev/null +++ b/cmake/development/ConstexprProbes.cmake @@ -0,0 +1,128 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "ConstexprProbes.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "ConstexprProbes.cmake requires the production SimdLib targets") +endif() + +block(SCOPE_FOR VARIABLES) + +# @brief Adds a compile-only constexpr contract probe. +# @param target Target name used in compiler diagnostics. +# @param source Translation unit containing static assertions. +function(simdlib_add_constexpr_probe target source) + add_library(${target} OBJECT ${source}) + simdlib_register_development_target(${target} CONSTEXPR_CONTRACT) + target_link_libraries(${target} PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(${target}) +endfunction() + +if(SIMDLIB_BUILD_CONSTEXPR_PROBES) + set(simdlib_constexpr_targets "") + + simdlib_add_constexpr_probe(LogicalShuffleOracleConstexprProbe + tests/constexpr/LogicalShuffleOracle.tests.cpp) + list(APPEND simdlib_constexpr_targets LogicalShuffleOracleConstexprProbe) + + # @brief Adds one BMI feature-macro compile profile. + # @param profile_name Profile suffix used in the target name. + # @param bmi1 Whether BMI1 declarations are enabled. + # @param bmi2 Whether BMI2 declarations are enabled. + function(simdlib_add_bmi_constexpr_profile profile_name bmi1 bmi2) + set(target Bmi${profile_name}ConstexprProbe) + simdlib_add_constexpr_probe(${target} tests/constexpr/BmiConstexpr.tests.cpp) + target_compile_definitions(${target} PRIVATE SIMDLIB_HAS_BMI1=${bmi1} SIMDLIB_HAS_BMI2=${bmi2}) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(${target} PRIVATE /arch:AVX2) + else() + target_compile_options(${target} PRIVATE -mno-bmi -mno-bmi2) + if(bmi1) + target_compile_options(${target} PRIVATE -mbmi) + endif() + if(bmi2) + target_compile_options(${target} PRIVATE -mbmi2) + endif() + endif() + set(simdlib_constexpr_targets ${simdlib_constexpr_targets} ${target} PARENT_SCOPE) + endfunction() + + simdlib_add_bmi_constexpr_profile(Portable 0 0) + simdlib_add_bmi_constexpr_profile(1 1 0) + simdlib_add_bmi_constexpr_profile(2 0 1) + simdlib_add_bmi_constexpr_profile(1Bmi2 1 1) + + foreach(uint128_profile IN ITEMS Optimized Portable Scalar) + set(target UInt128${uint128_profile}ConstexprProbe) + simdlib_add_constexpr_probe(${target} tests/constexpr/UInt128Constexpr.tests.cpp) + list(APPEND simdlib_constexpr_targets ${target}) + if(uint128_profile STREQUAL "Portable" OR uint128_profile STREQUAL "Scalar") + target_compile_definitions(${target} PRIVATE SIMDLIB_USE_COMPILER_CARRY_INTRINSICS=0) + endif() + if(uint128_profile STREQUAL "Scalar") + target_compile_definitions(${target} PRIVATE + SIMDLIB_HAS_SSE=0 SIMDLIB_HAS_SSE2=0 SIMDLIB_HAS_SSE3=0 SIMDLIB_HAS_SSSE3=0 + SIMDLIB_HAS_SSE41=0 SIMDLIB_HAS_SSE42=0 SIMDLIB_HAS_AVX=0 SIMDLIB_HAS_AVX2=0 + SIMDLIB_HAS_FMA=0 SIMDLIB_HAS_BMI1=0 SIMDLIB_HAS_BMI2=0) + elseif(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_definitions(${target} PRIVATE + SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) + else() + target_compile_options(${target} PRIVATE -msse4.2) + endif() + endforeach() + + simdlib_add_constexpr_probe(ApiSse42ConstexprProbe tests/constexpr/Api128Constexpr.tests.cpp) + list(APPEND simdlib_constexpr_targets ApiSse42ConstexprProbe) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_definitions(ApiSse42ConstexprProbe PRIVATE + SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(ApiSse42ConstexprProbe PRIVATE /arch:AVX2) + endif() + else() + target_compile_options(ApiSse42ConstexprProbe PRIVATE -msse4.2) + endif() + + simdlib_add_constexpr_probe(ApiAvx2ConstexprProbe tests/constexpr/Api256Constexpr.tests.cpp) + list(APPEND simdlib_constexpr_targets ApiAvx2ConstexprProbe) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(ApiAvx2ConstexprProbe PRIVATE /arch:AVX2) + else() + target_compile_options(ApiAvx2ConstexprProbe PRIVATE -mavx2) + endif() + + simdlib_add_constexpr_probe(ApiDisabledConstexprProbe tests/constexpr/ApiDisabledConstexpr.tests.cpp) + list(APPEND simdlib_constexpr_targets ApiDisabledConstexprProbe) + + set(constexpr_object_expressions "") + foreach(constexpr_target IN LISTS simdlib_constexpr_targets) + list(APPEND constexpr_object_expressions "$") + endforeach() + set(constexpr_record "${CMAKE_CURRENT_BINARY_DIR}/constexpr-probes/artifacts.record") + add_custom_command( + OUTPUT "${constexpr_record}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/constexpr-probes" + COMMAND ${CMAKE_COMMAND} + -DMODE=RECORD + -DRECORD_FILE=${constexpr_record} + "-DARTIFACTS=$" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/RecordArtifactHashes.cmake + DEPENDS ${constexpr_object_expressions} cmake/RecordArtifactHashes.cmake + COMMENT "Recording constexpr probe artifacts" + VERBATIM) + add_custom_target(ConstexprProbes ALL DEPENDS "${constexpr_record}") + simdlib_register_development_target(ConstexprProbes CONSTEXPR_CONTRACT) + add_dependencies(ConstexprProbes ${simdlib_constexpr_targets}) + add_test(NAME ConstexprProbes.Artifacts + COMMAND ${CMAKE_COMMAND} + -DMODE=VALIDATE + -DRECORD_FILE=${constexpr_record} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/RecordArtifactHashes.cmake) + set_tests_properties(ConstexprProbes.Artifacts PROPERTIES + LABELS "CONSTEXPR;COMPILE_ONLY" RUN_SERIAL TRUE) + simdlib_register_development_test(ConstexprProbes.Artifacts CONSTEXPR_CONTRACT) +endif() + +endblock() diff --git a/cmake/development/Coverage.cmake b/cmake/development/Coverage.cmake new file mode 100644 index 0000000..f29b54c --- /dev/null +++ b/cmake/development/Coverage.cmake @@ -0,0 +1,74 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "Coverage.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "Coverage.cmake requires the production SimdLib targets") +endif() + +block(SCOPE_FOR VARIABLES) + +if(SIMDLIB_ENABLE_COVERAGE) + get_filename_component(simdlib_compiler_directory "${CMAKE_CXX_COMPILER}" DIRECTORY) + find_program(SIMDLIB_LLVM_PROFDATA + NAMES llvm-profdata + HINTS "${simdlib_compiler_directory}" + REQUIRED) + find_program(SIMDLIB_LLVM_COV + NAMES llvm-cov + HINTS "${simdlib_compiler_directory}" + REQUIRED) + find_program(SIMDLIB_LLVM_READOBJ + NAMES llvm-readobj + HINTS "${simdlib_compiler_directory}" + REQUIRED) + + get_property(simdlib_coverage_targets GLOBAL PROPERTY SIMDLIB_COVERAGE_TARGETS) + list(REMOVE_DUPLICATES simdlib_coverage_targets) + list(SORT simdlib_coverage_targets) + if(NOT simdlib_coverage_targets) + message(FATAL_ERROR "SIMDLIB_ENABLE_COVERAGE requires at least one executable target") + endif() + + set(simdlib_coverage_manifest "") + foreach(coverage_target IN LISTS simdlib_coverage_targets) + get_target_property(coverage_profile_prefix ${coverage_target} + SIMDLIB_COVERAGE_PROFILE_PREFIX) + if(NOT coverage_profile_prefix) + message(FATAL_ERROR + "Coverage target ${coverage_target} has no CTest profile prefix") + endif() + string(APPEND simdlib_coverage_manifest + "${coverage_target}|$|${coverage_profile_prefix}\n") + endforeach() + set(simdlib_coverage_manifest_file + "${CMAKE_CURRENT_BINARY_DIR}/coverage-targets-$.txt") + file(GENERATE + OUTPUT "${simdlib_coverage_manifest_file}" + CONTENT "${simdlib_coverage_manifest}") + + add_custom_target(CoverageReset + COMMAND ${CMAKE_COMMAND} + -DBINARY_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ResetCoverage.cmake + COMMENT "Removing previous SimdLib coverage data" + VERBATIM) + simdlib_register_development_target(CoverageReset COVERAGE_SUPPORT) + + add_custom_target(CoverageReport + COMMAND ${CMAKE_COMMAND} + -DBINARY_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR} + -DSOURCE_DIRECTORY=${CMAKE_CURRENT_SOURCE_DIR} + -DCOVERAGE_MANIFEST=${simdlib_coverage_manifest_file} + -DLLVM_PROFDATA=${SIMDLIB_LLVM_PROFDATA} + -DLLVM_COV=${SIMDLIB_LLVM_COV} + -DLLVM_READOBJ=${SIMDLIB_LLVM_READOBJ} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/GenerateCoverageReport.cmake + DEPENDS ${simdlib_coverage_targets} + COMMENT "Generating SimdLib LCOV coverage report" + VERBATIM) + simdlib_register_development_target(CoverageReport COVERAGE_SUPPORT) +endif() + +endblock() diff --git a/cmake/development/Dependencies.cmake b/cmake/development/Dependencies.cmake new file mode 100644 index 0000000..d6fea54 --- /dev/null +++ b/cmake/development/Dependencies.cmake @@ -0,0 +1,28 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "Dependencies.cmake is available only to top-level SimdLib builds") +endif() + +block(SCOPE_FOR VARIABLES) + +if(SIMDLIB_BUILD_RUNTIME_TESTS OR SIMDLIB_BUILD_BENCHMARKS) + find_package(Catch2 3 CONFIG QUIET) + if(NOT TARGET Catch2::Catch2WithMain AND SIMDLIB_FETCH_TEST_DEPENDENCIES) + include(FetchContent) + FetchContent_Declare(Catch2 + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG 2b60af89e23d28eefc081bc930831ee9d45ea58b + GIT_SHALLOW TRUE) + FetchContent_MakeAvailable(Catch2) + endif() + if(NOT TARGET Catch2::Catch2WithMain) + message(FATAL_ERROR + "Catch2 3 is required; install it or enable SIMDLIB_FETCH_TEST_DEPENDENCIES") + endif() +endif() + +# Catch2 publishes Catch.cmake through CMAKE_MODULE_PATH for RuntimeTests.cmake. +set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE) + +endblock() diff --git a/cmake/development/Development.cmake b/cmake/development/Development.cmake new file mode 100644 index 0000000..43995f0 --- /dev/null +++ b/cmake/development/Development.cmake @@ -0,0 +1,43 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "Development.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "Development.cmake requires the production SimdLib targets") +endif() + +include(CTest) + +block(SCOPE_FOR VARIABLES) + +set(simdlib_development_modules + Options + TargetConfiguration + ArtifactOwnership + Dependencies + ConfigurationProbes + ConfigurationStateProbes + MethodFlagsCodegen + ConstexprProbes + HeaderProbes + RegisterCodegen + SmokeTests + RuntimeTests + Examples + Benchmarks + Coverage + ArtifactAggregates) +foreach(simdlib_development_module IN LISTS simdlib_development_modules) + set(simdlib_development_module_path + "${CMAKE_CURRENT_LIST_DIR}/${simdlib_development_module}.cmake") + if(NOT EXISTS "${simdlib_development_module_path}") + message(FATAL_ERROR + "Development coordinator cannot locate ${simdlib_development_module_path}") + endif() + include("${simdlib_development_module_path}") +endforeach() + +include("${CMAKE_CURRENT_LIST_FILE}") + +endblock() diff --git a/cmake/development/Examples.cmake b/cmake/development/Examples.cmake new file mode 100644 index 0000000..612ef4b --- /dev/null +++ b/cmake/development/Examples.cmake @@ -0,0 +1,40 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "Examples.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "Examples.cmake requires the production SimdLib targets") +endif() + +block(SCOPE_FOR VARIABLES) + +if(SIMDLIB_BUILD_EXAMPLES) + add_executable(ApiExamples examples/ApiExamples.cpp) + simdlib_register_development_target(ApiExamples SMOKE_VALIDATION) + target_link_libraries(ApiExamples PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(ApiExamples) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(ApiExamples PRIVATE /arch:AVX2) + else() + target_compile_options(ApiExamples PRIVATE -mavx2 -mfma -mbmi -mbmi2) + endif() + add_test(NAME ApiExamples COMMAND ApiExamples) + set_tests_properties(ApiExamples PROPERTIES LABELS "EXAMPLES;AVX2;FMA;BMI") + simdlib_register_development_test(ApiExamples SMOKE_VALIDATION) + simdlib_set_coverage_profile_prefix(ApiExamples "ApiExamples") + + if(SIMDLIB_REGISTER_COMPILER_SUPPORTED) + add_executable(RegisterExamples examples/RegisterExamples.cpp) + simdlib_register_development_target(RegisterExamples SMOKE_VALIDATION) + target_link_libraries(RegisterExamples PRIVATE SimdLib::Register) + simdlib_enable_development_warnings(RegisterExamples) + simdlib_enable_register_sse42(RegisterExamples) + add_test(NAME RegisterExamples COMMAND RegisterExamples) + set_tests_properties(RegisterExamples PROPERTIES LABELS "EXAMPLES;REGISTER;SSE42") + simdlib_register_development_test(RegisterExamples SMOKE_VALIDATION) + simdlib_set_coverage_profile_prefix(RegisterExamples "RegisterExamples") + endif() +endif() + +endblock() diff --git a/cmake/development/HeaderProbes.cmake b/cmake/development/HeaderProbes.cmake new file mode 100644 index 0000000..302c597 --- /dev/null +++ b/cmake/development/HeaderProbes.cmake @@ -0,0 +1,132 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "HeaderProbes.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "HeaderProbes.cmake requires the production SimdLib targets") +endif() + +block(SCOPE_FOR VARIABLES) + +if(SIMDLIB_BUILD_HEADER_PROBES) + set(simdlib_installed_header_root + "${CMAKE_CURRENT_BINARY_DIR}/installed-header-probe/include") + file(GLOB_RECURSE simdlib_installed_headers + CONFIGURE_DEPENDS + RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}/include/SimdLib/*.h") + foreach(simdlib_installed_header IN LISTS simdlib_installed_headers) + get_filename_component(simdlib_installed_header_directory + "${simdlib_installed_header}" DIRECTORY) + file(MAKE_DIRECTORY + "${simdlib_installed_header_root}/${simdlib_installed_header_directory}") + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/include/${simdlib_installed_header}" + "${simdlib_installed_header_root}/${simdlib_installed_header}" + COPYONLY) + endforeach() + + # @brief Configures a compiler-contract target against only the copied public headers. + # @param target Existing target that consumes the isolated header image. + # @param standard Exact C++ language standard required by the target. + function(simdlib_configure_installed_header_probe target standard) + target_include_directories(${target} PRIVATE + "${simdlib_installed_header_root}") + set_target_properties(${target} PROPERTIES + CXX_STANDARD ${standard} + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF) + simdlib_register_development_target(${target} COMPILER_CONTRACT) + simdlib_enable_development_warnings(${target}) + endfunction() + + add_library(InstalledConfigHeaderProbe OBJECT + tests/headers/InstalledConfigHeaderProbe.cpp) + simdlib_configure_installed_header_probe(InstalledConfigHeaderProbe 20) + + add_library(InstalledUmbrellaHeaderProbe OBJECT + tests/headers/InstalledUmbrellaHeaderProbe.cpp) + simdlib_configure_installed_header_probe(InstalledUmbrellaHeaderProbe 20) + + add_library(InstalledDisabledHeaderProbe OBJECT + tests/headers/InstalledDisabledHeaderProbe.cpp) + simdlib_configure_installed_header_probe(InstalledDisabledHeaderProbe 20) + target_compile_definitions(InstalledDisabledHeaderProbe PRIVATE + SIMDLIB_HAS_SSE=0 SIMDLIB_HAS_SSE2=0 SIMDLIB_HAS_SSE3=0 + SIMDLIB_HAS_SSSE3=0 SIMDLIB_HAS_SSE41=0 SIMDLIB_HAS_SSE42=0 + SIMDLIB_HAS_AVX=0 SIMDLIB_HAS_AVX2=0 SIMDLIB_HAS_FMA=0 + SIMDLIB_HAS_BMI1=0 SIMDLIB_HAS_BMI2=0) + + add_executable(InstalledHeaderOdrProbe + tests/headers/InstalledHeaderOdrDefinition.cpp + tests/headers/InstalledHeaderOdrConsumer.cpp + tests/headers/InstalledHeaderOdrFixture.h) + simdlib_configure_installed_header_probe(InstalledHeaderOdrProbe 20) + add_test(NAME InstalledHeaderOdr COMMAND InstalledHeaderOdrProbe) + set_tests_properties(InstalledHeaderOdr PROPERTIES + LABELS "HEADERS;METHOD_FLAGS;ODR") + simdlib_register_development_test(InstalledHeaderOdr COMPILER_CONTRACT) + foreach(header_probe IN ITEMS + Config + TemplateTools + IApi + IImpl + IRegister + IRegisterMask + Api + SimdApi + SimdVector + SimdAlgo + SimdResample + Bmi + UInt128 + Format + SimdLib + PublicSurface) + add_library(Header${header_probe}Probe OBJECT tests/headers/${header_probe}HeaderProbe.cpp) + simdlib_register_development_target(Header${header_probe}Probe + COMPILER_CONTRACT) + target_link_libraries(Header${header_probe}Probe PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(Header${header_probe}Probe) + endforeach() + + if(SIMDLIB_REGISTER_COMPILER_SUPPORTED) + add_library(InstalledRegisterHeaderProbe OBJECT + tests/headers/InstalledRegisterHeaderProbe.cpp) + simdlib_configure_installed_header_probe( + InstalledRegisterHeaderProbe 23) + target_compile_definitions(InstalledRegisterHeaderProbe PRIVATE + SIMDLIB_REQUIRE_REGISTER_INTERFACE=1) + simdlib_enable_register_sse42(InstalledRegisterHeaderProbe) + + add_library(HeaderAliasesProbe OBJECT + tests/headers/AliasesHeaderProbe.cpp) + simdlib_register_development_target(HeaderAliasesProbe COMPILER_CONTRACT) + target_link_libraries(HeaderAliasesProbe PRIVATE SimdLib::Register) + simdlib_enable_development_warnings(HeaderAliasesProbe) + simdlib_enable_register_avx2(HeaderAliasesProbe) + + add_library(HeaderRegisterProbe OBJECT + tests/headers/RegisterHeaderProbe.cpp) + simdlib_register_development_target(HeaderRegisterProbe COMPILER_CONTRACT) + target_link_libraries(HeaderRegisterProbe PRIVATE SimdLib::Register) + simdlib_enable_development_warnings(HeaderRegisterProbe) + + add_library(HeaderRegisterMaskProbe OBJECT + tests/headers/RegisterMaskHeaderProbe.cpp) + simdlib_register_development_target(HeaderRegisterMaskProbe COMPILER_CONTRACT) + target_link_libraries(HeaderRegisterMaskProbe PRIVATE SimdLib::Register) + simdlib_enable_development_warnings(HeaderRegisterMaskProbe) + + add_library(HeaderSimdLibRegisterProbe OBJECT + tests/headers/SimdLibRegisterHeaderProbe.cpp) + simdlib_register_development_target(HeaderSimdLibRegisterProbe + COMPILER_CONTRACT) + target_link_libraries(HeaderSimdLibRegisterProbe PRIVATE SimdLib::Register) + simdlib_enable_development_warnings(HeaderSimdLibRegisterProbe) + simdlib_enable_register_sse42(HeaderSimdLibRegisterProbe) + endif() +endif() + +endblock() diff --git a/cmake/development/MethodFlagsCodegen.cmake b/cmake/development/MethodFlagsCodegen.cmake new file mode 100644 index 0000000..cf2aa2f --- /dev/null +++ b/cmake/development/MethodFlagsCodegen.cmake @@ -0,0 +1,116 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "MethodFlagsCodegen.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib) + message(FATAL_ERROR "MethodFlagsCodegen.cmake requires the production SimdLib target") +endif() + +block(SCOPE_FOR VARIABLES) + +if(SIMDLIB_BUILD_CONFIGURATION_PROBES + AND SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES + AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|i[3-6]86)$") + if(NOT CMAKE_OBJDUMP) + find_program(CMAKE_OBJDUMP NAMES llvm-objdump llvm-objdump.exe objdump) + endif() + if(NOT CMAKE_OBJDUMP) + message(FATAL_ERROR "Method-flags generated-code gates require an objdump-compatible disassembler") + endif() + + add_library(MethodFlagsCodegenRaw OBJECT + tests/method_flags/codegen/MethodFlagsRaw.cpp) + add_library(MethodFlagsCodegenFlagged OBJECT + tests/method_flags/codegen/MethodFlagsFlagged.cpp) + foreach(method_flags_target IN ITEMS MethodFlagsCodegenRaw MethodFlagsCodegenFlagged) + simdlib_register_development_target(${method_flags_target} + OPTIMIZED_CODEGEN) + target_link_libraries(${method_flags_target} PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(${method_flags_target}) + if(SIMDLIB_MSVC_STYLE_DRIVER) + set_property(TARGET ${method_flags_target} PROPERTY MSVC_RUNTIME_CHECKS "") + target_compile_options(${method_flags_target} PRIVATE /O2 /Ob2 /GS) + else() + target_compile_options(${method_flags_target} PRIVATE + -O2 -msse4.2 -fstack-protector-strong) + endif() + endforeach() + + set(method_flags_vectorcall_enabled 0) + if(WIN32 AND (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" + OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")) + set(method_flags_vectorcall_enabled 1) + endif() + if(SIMDLIB_MSVC_STYLE_DRIVER) + set(method_flags_stack_protector_mode "msvc-gs") + else() + set(method_flags_stack_protector_mode "strong") + endif() + + set(method_flags_artifact_directory + "${CMAKE_CURRENT_BINARY_DIR}/method-flags-codegen") + set(method_flags_record + "${method_flags_artifact_directory}/comparison.record.json") + set(method_flags_verification + "${method_flags_artifact_directory}/verification.txt") + add_custom_command( + OUTPUT "${method_flags_record}" "${method_flags_verification}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${method_flags_artifact_directory}" + COMMAND ${CMAKE_COMMAND} -E rm -f "${method_flags_verification}" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${method_flags_artifact_directory} + -DRECORD_FILE=${method_flags_record} + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=128 + -DISA_PROFILE=SSE42 + -DVECTORCALL_ENABLED=${method_flags_vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${method_flags_stack_protector_mode} + -DCODEGEN_PROFILE=method-flags + -DSYMBOL_PATTERN=simdlib_method_flags_codegen_ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + COMMAND ${CMAKE_COMMAND} + -DFLAGGED_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DSTACK_PROTECTOR_MODE=${method_flags_stack_protector_mode} + -DOUTPUT_FILE=${method_flags_verification} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyMethodFlagsCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + cmake/VerifyMethodFlagsCodegen.cmake + COMMENT "Verifying method-flags generated code and stack contract" + VERBATIM) + add_custom_target(MethodFlagsCodegen ALL + DEPENDS "${method_flags_record}" "${method_flags_verification}") + simdlib_register_development_target(MethodFlagsCodegen OPTIMIZED_CODEGEN) + add_dependencies(MethodFlagsCodegen + MethodFlagsCodegenRaw + MethodFlagsCodegenFlagged) + + set(method_flags_record_index + "${method_flags_artifact_directory}/all-records.txt") + file(GENERATE OUTPUT "${method_flags_record_index}" + CONTENT "${method_flags_record}\n") + add_test(NAME MethodFlagsCodegen + COMMAND ${CMAKE_COMMAND} + -DRECORD_INDEX=${method_flags_record_index} + -DVERIFICATION_FILE=${method_flags_verification} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyMethodFlagsCodegenRecords.cmake) + set_tests_properties(MethodFlagsCodegen PROPERTIES + LABELS "CONFIGURATION;METHOD_FLAGS;CODEGEN;ABI;STACK") + simdlib_register_development_test(MethodFlagsCodegen OPTIMIZED_CODEGEN) +endif() + +endblock() diff --git a/cmake/development/Options.cmake b/cmake/development/Options.cmake new file mode 100644 index 0000000..26f52ad --- /dev/null +++ b/cmake/development/Options.cmake @@ -0,0 +1,106 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "Options.cmake is available only to top-level SimdLib builds") +endif() + +block(SCOPE_FOR VARIABLES) + +set(simdlib_retired_options + SIMDLIB_BUILD_TESTS + SIMDLIB_BUILD_TESTS_128 + SIMDLIB_BUILD_TESTS_256 + SIMDLIB_BUILD_TESTS_FMA + SIMDLIB_BUILD_TESTS_OPTIONAL + SIMDLIB_BUILD_CONFIGURATION_TESTS + SIMDLIB_BUILD_HEADER_TESTS + SIMDLIB_BUILD_REGISTER_CODEGEN + SIMDLIB_REGISTER_CODEGEN_RECORD_ONLY) +set(simdlib_retired_replacements + SIMDLIB_BUILD_RUNTIME_TESTS + SIMDLIB_BUILD_API_SSE42_TESTS + SIMDLIB_BUILD_API_AVX2_TESTS + SIMDLIB_BUILD_FMA_TESTS + SIMDLIB_BUILD_BMI_TESTS + SIMDLIB_BUILD_CONFIGURATION_PROBES + SIMDLIB_BUILD_HEADER_PROBES + SIMDLIB_BUILD_REGISTER_CODEGEN_GATES + SIMDLIB_REGISTER_CODEGEN_MODE) +list(LENGTH simdlib_retired_options simdlib_retired_option_count) +math(EXPR simdlib_retired_option_last "${simdlib_retired_option_count} - 1") +foreach(simdlib_retired_option_index RANGE ${simdlib_retired_option_last}) + list(GET simdlib_retired_options ${simdlib_retired_option_index} simdlib_retired_option) + if(DEFINED CACHE{${simdlib_retired_option}}) + list(GET simdlib_retired_replacements ${simdlib_retired_option_index} + simdlib_retired_replacement) + message(FATAL_ERROR + "Retired CMake option ${simdlib_retired_option} was supplied. " + "Use ${simdlib_retired_replacement}; compatibility aliases are intentionally unavailable.") + endif() +endforeach() + +option(SIMDLIB_BUILD_SMOKE_TESTS "Build header-only ODR smoke tests" ON) +option(SIMDLIB_BUILD_RUNTIME_TESTS "Build Catch2 runtime tests" OFF) +option(SIMDLIB_BUILD_API_SSE42_TESTS "Build Api SSE4.2 tests" ON) +option(SIMDLIB_BUILD_API_AVX2_TESTS "Build Api AVX2 tests" ON) +option(SIMDLIB_BUILD_FMA_TESTS "Build FMA tests" ON) +option(SIMDLIB_BUILD_BMI_TESTS "Build BMI profile tests" OFF) +option(SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS + "Build SimdVector, SimdAlgo, and resampling parity tests" ON) +option(SIMDLIB_BUILD_BENCHMARKS "Build Catch2 benchmarks" OFF) +option(SIMDLIB_BUILD_EXAMPLES "Build executable API examples" OFF) +option(SIMDLIB_BUILD_CONFIGURATION_PROBES + "Build compile-only configuration probes" ON) +option(SIMDLIB_BUILD_CONSTEXPR_PROBES + "Build compile-only constant-evaluation contract probes" ON) +option(SIMDLIB_BUILD_HEADER_PROBES + "Build first-and-only public-header probes" ON) +option(SIMDLIB_FETCH_TEST_DEPENDENCIES + "Fetch missing development-only dependencies" ON) +option(SIMDLIB_STRICT_WARNINGS + "Treat warnings in SimdLib-owned development targets as errors" OFF) +option(SIMDLIB_ENABLE_COVERAGE + "Instrument SimdLib-owned development targets for source coverage" OFF) +option(SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES + "Build method-attribute generated-code comparisons" OFF) +option(SIMDLIB_BUILD_REGISTER_CODEGEN_GATES + "Build Register generated-code comparisons" OFF) +option(SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS + "Require every category selected by the validation profile to contain owned targets" OFF) + +set(SIMDLIB_DEFAULT_CHECKS_PROBE "NONE" CACHE STRING + "Default checks-state contract: NONE, RELEASE, or DEBUG") +set_property(CACHE SIMDLIB_DEFAULT_CHECKS_PROBE PROPERTY STRINGS + NONE RELEASE DEBUG) +if(NOT SIMDLIB_DEFAULT_CHECKS_PROBE MATCHES "^(NONE|RELEASE|DEBUG)$") + message(FATAL_ERROR + "SIMDLIB_DEFAULT_CHECKS_PROBE has unsupported value " + "'${SIMDLIB_DEFAULT_CHECKS_PROBE}'") +endif() + +set(SIMDLIB_VALIDATION_PROFILE "CUSTOM" CACHE STRING + "Validation ownership profile: CUSTOM, RELEASE, DEBUG, SANITIZER, COVERAGE, CODEGEN_DIAGNOSTIC, or COMPILER_CONTRACTS") +set_property(CACHE SIMDLIB_VALIDATION_PROFILE PROPERTY STRINGS + CUSTOM RELEASE DEBUG SANITIZER COVERAGE CODEGEN_DIAGNOSTIC COMPILER_CONTRACTS) +if(NOT SIMDLIB_VALIDATION_PROFILE MATCHES + "^(CUSTOM|RELEASE|DEBUG|SANITIZER|COVERAGE|CODEGEN_DIAGNOSTIC|COMPILER_CONTRACTS)$") + message(FATAL_ERROR + "SIMDLIB_VALIDATION_PROFILE has unsupported value " + "'${SIMDLIB_VALIDATION_PROFILE}'") +endif() + +set(SIMDLIB_REGISTER_CODEGEN_MODE "OFF" CACHE STRING + "Register generated-code policy: OFF, ENFORCE, or RECORD") +set_property(CACHE SIMDLIB_REGISTER_CODEGEN_MODE PROPERTY STRINGS + OFF ENFORCE RECORD) +if(NOT SIMDLIB_REGISTER_CODEGEN_MODE MATCHES "^(OFF|ENFORCE|RECORD)$") + message(FATAL_ERROR + "SIMDLIB_REGISTER_CODEGEN_MODE must be OFF, ENFORCE, or RECORD; got " + "'${SIMDLIB_REGISTER_CODEGEN_MODE}'") +endif() + +if(SIMDLIB_ENABLE_COVERAGE) + set(CTEST_TEST_COVERAGE_TOOL "LLVM-COV" PARENT_SCOPE) +endif() + +endblock() diff --git a/cmake/development/RegisterCodegen.cmake b/cmake/development/RegisterCodegen.cmake new file mode 100644 index 0000000..68235c8 --- /dev/null +++ b/cmake/development/RegisterCodegen.cmake @@ -0,0 +1,677 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "RegisterCodegen.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "RegisterCodegen.cmake requires the production SimdLib targets") +endif() + +block(SCOPE_FOR VARIABLES) + +# @brief Adds paired wrapper/raw object fixtures and a mandatory disassembly comparison. +# @param register_width Width of the compared native and wrapped register values. +# @param isa_profile Instruction-set profile used to compile both sides of the comparison. +function(simdlib_add_register_codegen_gate register_width isa_profile) + if(SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "ENFORCE") + set(codegen_validation_category OPTIMIZED_CODEGEN) + else() + set(codegen_validation_category DEBUG_DIAGNOSTIC) + endif() + if(NOT isa_profile STREQUAL "SSE42" AND NOT isa_profile STREQUAL "AVX2") + message(FATAL_ERROR "Unsupported Register codegen ISA profile: ${isa_profile}") + endif() + if(isa_profile STREQUAL "SSE42" AND NOT register_width EQUAL 128) + message(FATAL_ERROR "The SSE4.2 Register codegen profile supports only 128-bit registers") + endif() + if(isa_profile STREQUAL "SSE42") + set(target_suffix "${register_width}Sse42") + set(artifact_profile "sse42") + set(codegen_comparison_record_only ON) + else() + set(target_suffix "${register_width}Avx2") + set(artifact_profile "avx2") + if(SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "RECORD") + set(codegen_comparison_record_only ON) + else() + set(codegen_comparison_record_only OFF) + endif() + endif() + set(composition_record_only ${codegen_comparison_record_only}) + set(composition_difference_reason "non-release-differential") + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + set(composition_record_only ON) + set(composition_difference_reason "msvc-gs-composition-cookie") + endif() + set(modulus_record_only ${codegen_comparison_record_only}) + set(modulus_difference_reason "non-release-differential") + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND isa_profile STREQUAL "AVX2" AND register_width EQUAL 256) + set(modulus_record_only ON) + set(modulus_difference_reason "msvc-scalar-remainder-scheduling") + endif() + set(vectorcall_enabled 0) + set(stack_protector_mode "msvc-gs") + if(WIN32 AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|i[3-6]86)$" AND + (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")) + set(vectorcall_enabled 1) + endif() + if(NOT SIMDLIB_MSVC_STYLE_DRIVER AND + (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")) + set(stack_protector_mode "strong") + endif() + set(wrapper_target RegisterCodegenWrapper${target_suffix}) + set(raw_target RegisterCodegenRaw${target_suffix}) + set(default_wrapper_target RegisterDefaultAbiWrapper${target_suffix}) + set(default_raw_target RegisterDefaultAbiRaw${target_suffix}) + set(abi_wrapper_target RegisterAbiWrapper${target_suffix}) + set(abi_raw_target RegisterAbiRaw${target_suffix}) + set(specialized_wrapper_target RegisterSpecializedWrapper${target_suffix}) + set(specialized_raw_target RegisterSpecializedRaw${target_suffix}) + set(fma_enabled_wrapper_target RegisterFmaEnabledWrapper${target_suffix}) + set(fma_enabled_raw_target RegisterFmaEnabledRaw${target_suffix}) + set(fma_disabled_wrapper_target RegisterFmaDisabledWrapper${target_suffix}) + set(fma_disabled_raw_target RegisterFmaDisabledRaw${target_suffix}) + set(rearrangement_wrapper_target RegisterRearrangementWrapper${target_suffix}) + set(rearrangement_raw_target RegisterRearrangementRaw${target_suffix}) + set(type_matrix_wrapper_target RegisterTypeMatrixWrapper${target_suffix}) + set(type_matrix_raw_target RegisterTypeMatrixRaw${target_suffix}) + add_library(${wrapper_target} OBJECT tests/codegen/RegisterCodegen.cpp) + add_library(${raw_target} OBJECT tests/codegen/RegisterCodegenRaw.cpp) + add_library(${default_wrapper_target} OBJECT tests/codegen/RegisterDefaultAbi.cpp) + add_library(${default_raw_target} OBJECT tests/codegen/RegisterDefaultAbiRaw.cpp) + add_library(${abi_wrapper_target} OBJECT tests/codegen/RegisterAbi.cpp) + add_library(${abi_raw_target} OBJECT tests/codegen/RegisterAbiRaw.cpp) + add_library(${specialized_wrapper_target} OBJECT tests/codegen/RegisterSpecializedCodegen.cpp) + add_library(${specialized_raw_target} OBJECT tests/codegen/RegisterSpecializedCodegenRaw.cpp) + if(isa_profile STREQUAL "AVX2") + add_library(${fma_enabled_wrapper_target} OBJECT tests/codegen/RegisterFmaCodegen.cpp) + add_library(${fma_enabled_raw_target} OBJECT tests/codegen/RegisterFmaCodegenRaw.cpp) + endif() + add_library(${fma_disabled_wrapper_target} OBJECT tests/codegen/RegisterFmaCodegen.cpp) + add_library(${fma_disabled_raw_target} OBJECT tests/codegen/RegisterFmaCodegenRaw.cpp) + add_library(${rearrangement_wrapper_target} OBJECT tests/codegen/RegisterRearrangementCodegen.cpp) + add_library(${rearrangement_raw_target} OBJECT tests/codegen/RegisterRearrangementCodegenRaw.cpp) + add_library(${type_matrix_wrapper_target} OBJECT tests/codegen/RegisterTypeMatrixCodegen.cpp) + add_library(${type_matrix_raw_target} OBJECT tests/codegen/RegisterTypeMatrixCodegenRaw.cpp) + set(codegen_object_targets + ${wrapper_target} ${raw_target} ${default_wrapper_target} ${default_raw_target} + ${abi_wrapper_target} ${abi_raw_target} + ${specialized_wrapper_target} ${specialized_raw_target} + ${fma_disabled_wrapper_target} ${fma_disabled_raw_target} + ${rearrangement_wrapper_target} ${rearrangement_raw_target} + ${type_matrix_wrapper_target} ${type_matrix_raw_target}) + if(isa_profile STREQUAL "AVX2") + list(APPEND codegen_object_targets + ${fma_enabled_wrapper_target} ${fma_enabled_raw_target}) + endif() + foreach(target IN LISTS codegen_object_targets) + simdlib_register_development_target(${target} + ${codegen_validation_category}) + target_link_libraries(${target} PRIVATE SimdLib::Register) + target_compile_definitions(${target} PRIVATE SIMDLIB_REGISTER_TEST_WIDTH=${register_width}) + simdlib_enable_development_warnings(${target}) + if(isa_profile STREQUAL "SSE42") + simdlib_enable_register_sse42(${target}) + else() + simdlib_enable_register_avx2(${target}) + endif() + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(${target} PRIVATE /GS) + else() + target_compile_options(${target} PRIVATE -fstack-protector-strong) + endif() + if(SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "ENFORCE") + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(${target} PRIVATE /O2) + else() + target_compile_options(${target} PRIVATE -O2) + endif() + endif() + endforeach() + set_property(GLOBAL APPEND PROPERTY + SIMDLIB_REGISTER_CODEGEN_OBJECT_TARGETS ${codegen_object_targets}) + if(isa_profile STREQUAL "AVX2") + foreach(target IN ITEMS ${fma_enabled_wrapper_target} ${fma_enabled_raw_target}) + target_compile_definitions(${target} PRIVATE SIMDLIB_HAS_FMA=1) + if(NOT SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(${target} PRIVATE -mfma) + endif() + endforeach() + endif() + foreach(target IN ITEMS ${fma_disabled_wrapper_target} ${fma_disabled_raw_target}) + target_compile_definitions(${target} PRIVATE SIMDLIB_HAS_FMA=0) + if(NOT SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(${target} PRIVATE -mno-fma) + endif() + endforeach() + + set(artifact_directory "${CMAKE_CURRENT_BINARY_DIR}/register-codegen/${artifact_profile}/${register_width}") + set(composition_stamp_file "${artifact_directory}/primary-composition/comparison.record.json") + set(immediate_shift_stamp_file "${artifact_directory}/complete-byte-shift-immediate/comparison.record.json") + set(immediate_shift_instruction_stamp_file "${artifact_directory}/complete-byte-shift-immediate/instructions.verified") + set(register_only_stamp_file "${artifact_directory}/register-only/comparison.record.json") + set(reassignment_stamp_file "${artifact_directory}/reassignment/comparison.record.json") + set(default_abi_stamp_file "${artifact_directory}/default-abi.record.json") + set(abi_stamp_file "${artifact_directory}/abi/comparison.record.json") + set(consumer_abi_stamp_file "${artifact_directory}/consumer-abi/comparison.record.json") + set(specialized_stamp_file "${artifact_directory}/specialized/comparison.record.json") + set(fma_enabled_stamp_file "${artifact_directory}/fma/enabled/comparison.record.json") + set(fma_disabled_stamp_file "${artifact_directory}/fma/disabled/comparison.record.json") + set(rearrangement_stamp_file "${artifact_directory}/rearrangement-conversion/comparison.record.json") + set(type_matrix_stamp_file "${artifact_directory}/type-matrix/common/comparison.record.json") + set(type_matrix_modulus_stamp_file "${artifact_directory}/type-matrix/modulus/comparison.record.json") + add_custom_command( + OUTPUT "${composition_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/primary-composition" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/primary-composition + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${composition_record_only} + -DRECORDED_DIFFERENCE_REASON=${composition_difference_reason} + -DCODEGEN_PROFILE=primary-composition + "-DSYMBOL_PATTERN=simdlib_codegen_(load_operate_store|aligned_transfer|byte_transfer|mutate|complete_shift_static|complete_shift_runtime|complete_byte_shift|opaque)" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit composed and memory-capable Register code" + VERBATIM) + add_custom_command( + OUTPUT "${immediate_shift_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/complete-byte-shift-immediate" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/complete-byte-shift-immediate + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=OFF + -DCODEGEN_PROFILE=complete-byte-shift-immediate + -DSYMBOL_PATTERN=simdlib_codegen_shift_bytes_ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit immediate byte-shift Register and Api generated code" + VERBATIM) + add_custom_command( + OUTPUT "${immediate_shift_instruction_stamp_file}" + COMMAND ${CMAKE_COMMAND} + -DOBJECT_FILE=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DOUTPUT_FILE=${immediate_shift_instruction_stamp_file} + -DREGISTER_WIDTH=${register_width} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyCompleteRegisterShiftCodegen.cmake + DEPENDS + $ + cmake/VerifyCompleteRegisterShiftCodegen.cmake + COMMENT "Verifying ${register_width}-bit immediate byte-shift instruction selection" + VERBATIM) + + add_custom_command( + OUTPUT "${register_only_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/register-only" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/register-only + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${codegen_comparison_record_only} + "-DSYMBOL_PATTERN=simdlib_codegen_(ternary|mask_combine|mask_select|mask_bits|mask_any|mask_all|native|broadcast_reuse|lane_last|special_members|pressure|basic_bitwise|basic_broadcast_chain|basic_shift_left_immediate)" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit register-only wrapper and raw generated code" + VERBATIM) + add_custom_command( + OUTPUT "${specialized_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/specialized" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/specialized + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${codegen_comparison_record_only} + -DCODEGEN_PROFILE=specialized + -DSYMBOL_PATTERN=simdlib_specialized_codegen_ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit specialized Register code" + VERBATIM) + if(isa_profile STREQUAL "AVX2") + add_custom_command( + OUTPUT "${fma_enabled_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/fma/enabled" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/fma/enabled + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${codegen_comparison_record_only} + -DCODEGEN_PROFILE=fma-enabled + -DFMA_EXPECTATION=enabled + "-DSYMBOL_PATTERN=simdlib_fma_codegen_multiply_add_(f32|f64)" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit isolated Register multiply-add code with FMA enabled" + VERBATIM) + endif() + add_custom_command( + OUTPUT "${fma_disabled_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/fma/disabled" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/fma/disabled + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${codegen_comparison_record_only} + -DCODEGEN_PROFILE=fma-disabled + -DFMA_EXPECTATION=disabled + "-DSYMBOL_PATTERN=simdlib_fma_codegen_multiply_add_(f32|f64)" + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit isolated Register multiply-add code with FMA disabled" + VERBATIM) + add_custom_command( + OUTPUT "${rearrangement_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/rearrangement-conversion" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/rearrangement-conversion + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${codegen_comparison_record_only} + -DCODEGEN_PROFILE=rearrangement-conversion + -DSYMBOL_PATTERN=simdlib_rearrangement_codegen_ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit rearrangement and conversion wrapper and raw generated code" + VERBATIM) + add_custom_command( + OUTPUT "${type_matrix_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/type-matrix/common" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/type-matrix/common + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${codegen_comparison_record_only} + -DCODEGEN_PROFILE=common-type-matrix + -DSYMBOL_PATTERN=simdlib_type_matrix_ + -DEXCLUDE_SYMBOL_PATTERN=simdlib_type_matrix_modulus_ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit common non-modulus operations across every Register element type" + VERBATIM) + add_custom_command( + OUTPUT "${type_matrix_modulus_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/type-matrix/modulus" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/type-matrix/modulus + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${modulus_record_only} + -DRECORDED_DIFFERENCE_REASON=${modulus_difference_reason} + -DCODEGEN_PROFILE=modulus-type-matrix + -DSYMBOL_PATTERN=simdlib_type_matrix_modulus_ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit modulus operations across every integer Register element type" + VERBATIM) + add_custom_command( + OUTPUT "${reassignment_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/reassignment" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/reassignment + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${codegen_comparison_record_only} + -DSYMBOL_PATTERN=simdlib_codegen_reassignment_arithmetic + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit reassignment wrapper and raw generated code" + VERBATIM) + add_custom_command( + OUTPUT "${abi_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/abi" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/abi + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${codegen_comparison_record_only} + -DSYMBOL_PATTERN=simdlib_abi_ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit explicit-object and raw ABI mirrors" + VERBATIM) + add_custom_command( + OUTPUT "${default_abi_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory} + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/RecordRegisterDefaultAbi.cmake + DEPENDS + $ + $ + cmake/RecordRegisterDefaultAbi.cmake + COMMENT "Recording ${register_width}-bit platform-default Register ABI" + VERBATIM) + add_custom_command( + OUTPUT "${consumer_abi_stamp_file}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${artifact_directory}/consumer-abi" + COMMAND ${CMAKE_COMMAND} + -DWRAPPER_OBJECT=$ + -DRAW_OBJECT=$ + -DOBJDUMP=${CMAKE_OBJDUMP} + -DARTIFACT_DIRECTORY=${artifact_directory}/consumer-abi + -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} + -DCOMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} + -DCOMPILER_PATH=${CMAKE_CXX_COMPILER} + -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} + -DSYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCONFIGURATION=$ + -DREGISTER_WIDTH=${register_width} + -DISA_PROFILE=${isa_profile} + -DVECTORCALL_ENABLED=${vectorcall_enabled} + -DSTACK_PROTECTOR_MODE=${stack_protector_mode} + -DRECORD_ONLY=${codegen_comparison_record_only} + -DSYMBOL_PATTERN=simdlib_consumer_abi_ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareRegisterCodegen.cmake + DEPENDS + $ + $ + cmake/CompareRegisterCodegen.cmake + COMMENT "Comparing ${register_width}-bit downstream Register wrappers and raw ABI boundaries" + VERBATIM) + set(expression_codegen_gate_outputs + "${composition_stamp_file}" "${immediate_shift_stamp_file}" "${register_only_stamp_file}" "${reassignment_stamp_file}" + "${specialized_stamp_file}" "${fma_disabled_stamp_file}" + "${rearrangement_stamp_file}" "${type_matrix_stamp_file}" "${type_matrix_modulus_stamp_file}") + if(isa_profile STREQUAL "AVX2") + list(APPEND expression_codegen_gate_outputs "${fma_enabled_stamp_file}") + endif() + add_custom_target(RegisterExpressionCodegen${target_suffix} + DEPENDS ${expression_codegen_gate_outputs} "${immediate_shift_instruction_stamp_file}") + simdlib_register_development_target( + RegisterExpressionCodegen${target_suffix} + ${codegen_validation_category}) + add_dependencies(RegisterExpressionCodegen${target_suffix} ${codegen_object_targets}) + add_custom_target(RegisterConsumerAbi${target_suffix} + DEPENDS "${consumer_abi_stamp_file}") + simdlib_register_development_target(RegisterConsumerAbi${target_suffix} + ${codegen_validation_category}) + add_dependencies(RegisterConsumerAbi${target_suffix} + ${abi_wrapper_target} ${abi_raw_target}) + set(codegen_gate_outputs + ${expression_codegen_gate_outputs} "${consumer_abi_stamp_file}" "${abi_stamp_file}" "${default_abi_stamp_file}") + set(codegen_classification_outputs + "${composition_stamp_file}" + "${immediate_shift_stamp_file}" + "${register_only_stamp_file}" + "${reassignment_stamp_file}" + "${specialized_stamp_file}" + "${fma_disabled_stamp_file}" + "${rearrangement_stamp_file}" + "${type_matrix_stamp_file}" + "${type_matrix_modulus_stamp_file}" + "${consumer_abi_stamp_file}" + "${abi_stamp_file}") + set(codegen_classification_record_only + ${composition_record_only} + OFF + ${codegen_comparison_record_only} + ${codegen_comparison_record_only} + ${codegen_comparison_record_only} + ${codegen_comparison_record_only} + ${codegen_comparison_record_only} + ${codegen_comparison_record_only} + ${modulus_record_only} + ${codegen_comparison_record_only} + ${codegen_comparison_record_only}) + if(isa_profile STREQUAL "AVX2") + list(APPEND codegen_classification_outputs "${fma_enabled_stamp_file}") + list(APPEND codegen_classification_record_only + ${codegen_comparison_record_only}) + endif() + set(enforced_codegen_gate_outputs "") + set(diagnostic_codegen_gate_outputs "${default_abi_stamp_file}") + list(LENGTH codegen_classification_outputs codegen_classification_count) + math(EXPR codegen_classification_last "${codegen_classification_count} - 1") + foreach(codegen_classification_index RANGE ${codegen_classification_last}) + list(GET codegen_classification_outputs + ${codegen_classification_index} codegen_classification_output) + list(GET codegen_classification_record_only + ${codegen_classification_index} codegen_classification_is_record_only) + if(codegen_classification_is_record_only) + list(APPEND diagnostic_codegen_gate_outputs + "${codegen_classification_output}") + else() + list(APPEND enforced_codegen_gate_outputs + "${codegen_classification_output}") + endif() + endforeach() + add_custom_target(RegisterCodegen${target_suffix} ALL + DEPENDS "${abi_stamp_file}" "${default_abi_stamp_file}") + simdlib_register_development_target(RegisterCodegen${target_suffix} + ${codegen_validation_category}) + add_dependencies(RegisterCodegen${target_suffix} + RegisterExpressionCodegen${target_suffix} + RegisterConsumerAbi${target_suffix}) + set(codegen_record_index "${artifact_directory}/all-records.txt") + set(enforced_codegen_record_index + "${artifact_directory}/enforced-records.txt") + set(diagnostic_codegen_record_index + "${artifact_directory}/diagnostic-records.txt") + file(GENERATE OUTPUT "${codegen_record_index}" + CONTENT "$\n") + file(GENERATE OUTPUT "${enforced_codegen_record_index}" + CONTENT "$\n") + file(GENERATE OUTPUT "${diagnostic_codegen_record_index}" + CONTENT "$\n") + set(require_enforced_records OFF) + if(SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "ENFORCE" AND + isa_profile STREQUAL "AVX2") + set(require_enforced_records ON) + endif() + add_test(NAME RegisterCodegen.${target_suffix} + COMMAND ${CMAKE_COMMAND} + -DENFORCED_RECORD_INDEX=${enforced_codegen_record_index} + -DDIAGNOSTIC_RECORD_INDEX=${diagnostic_codegen_record_index} + -DCODEGEN_MODE=${SIMDLIB_REGISTER_CODEGEN_MODE} + -DCONFIGURATION=$ + -DREQUIRE_ENFORCED_RECORDS=${require_enforced_records} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ValidateRegisterCodegenProfile.cmake) + set_tests_properties(RegisterCodegen.${target_suffix} PROPERTIES + LABELS "REGISTER;CODEGEN;ABI;${isa_profile}" RUN_SERIAL TRUE) + simdlib_register_development_test(RegisterCodegen.${target_suffix} + ${codegen_validation_category}) +endfunction() + +if(SIMDLIB_BUILD_REGISTER_CODEGEN_GATES AND SIMDLIB_REGISTER_COMPILER_SUPPORTED) + if(NOT CMAKE_OBJDUMP) + find_program(CMAKE_OBJDUMP NAMES llvm-objdump llvm-objdump.exe) + endif() + if(NOT CMAKE_OBJDUMP) + message(FATAL_ERROR "Register generated-code gates require an objdump-compatible disassembler") + endif() + simdlib_add_register_codegen_gate(128 SSE42) + simdlib_add_register_codegen_gate(128 AVX2) + simdlib_add_register_codegen_gate(256 AVX2) + get_property(register_codegen_object_targets GLOBAL PROPERTY + SIMDLIB_REGISTER_CODEGEN_OBJECT_TARGETS) + add_custom_target(RegisterCodegenFixtureObjects + DEPENDS ${register_codegen_object_targets}) + if(SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "ENFORCE") + simdlib_register_development_target( + RegisterCodegenFixtureObjects OPTIMIZED_CODEGEN) + else() + simdlib_register_development_target( + RegisterCodegenFixtureObjects DEBUG_DIAGNOSTIC) + endif() + add_custom_target(RegisterCodegen DEPENDS + RegisterCodegen128Sse42 + RegisterCodegen128Avx2 + RegisterCodegen256Avx2) + if(SIMDLIB_REGISTER_CODEGEN_MODE STREQUAL "ENFORCE") + simdlib_register_development_target(RegisterCodegen OPTIMIZED_CODEGEN) + else() + simdlib_register_development_target(RegisterCodegen DEBUG_DIAGNOSTIC) + endif() +endif() + +endblock() diff --git a/cmake/development/RuntimeTests.cmake b/cmake/development/RuntimeTests.cmake new file mode 100644 index 0000000..2134084 --- /dev/null +++ b/cmake/development/RuntimeTests.cmake @@ -0,0 +1,382 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "RuntimeTests.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "RuntimeTests.cmake requires the production SimdLib targets") +endif() + +block(SCOPE_FOR VARIABLES) + +if(SIMDLIB_BUILD_RUNTIME_TESTS) + include(Catch) + # Build receipts inventory CTest immediately after compilation, so keeping + # discovery at build time avoids hidden discovery work during receipt reuse. + set(CMAKE_CATCH_DISCOVER_TESTS_DISCOVERY_MODE POST_BUILD) + + # @brief Applies labels after Catch2 has populated its deferred discovery list. + # @param test_list_variable Name of the Catch2-generated test-list variable. + # @param labels Semicolon-separated labels applied to every discovered test. + # @param owner Validation category that owns every discovered test. + function(simdlib_label_discovered_tests test_list_variable labels owner) + set(label_file "${CMAKE_CURRENT_BINARY_DIR}/${test_list_variable}-labels.cmake") + file(WRITE "${label_file}" + "foreach(discovered_test IN LISTS ${test_list_variable})\n" + " set_tests_properties(\"\${discovered_test}\" PROPERTIES LABELS \"${labels};SIMDLIB_OWNER_${owner}\")\n" + "endforeach()\n") + set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES "${label_file}") + endfunction() + + # @brief Adds and discovers one Catch2 executable with stable labels. + # @param target Development executable target name. + # @param source Translation unit that owns the Catch2 cases. + # @param test_prefix Prefix applied to every discovered CTest identity. + # @param labels Semicolon-separated labels applied to every discovered case. + # @param category Optional validation category; defaults to RUNTIME_VALIDATION. + function(simdlib_add_catch_test target source test_prefix labels) + set(validation_category RUNTIME_VALIDATION) + if(ARGC GREATER 4) + set(validation_category ${ARGV4}) + endif() + add_executable(${target} ${source}) + simdlib_register_development_target(${target} ${validation_category}) + target_link_libraries(${target} PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) + simdlib_enable_development_warnings(${target}) + simdlib_set_coverage_profile_prefix(${target} "${test_prefix}") + set(test_list_variable "${target}_DISCOVERED_TESTS") + catch_discover_tests(${target} + TEST_PREFIX "${test_prefix}." + TEST_LIST ${test_list_variable}) + simdlib_label_discovered_tests(${test_list_variable} "${labels}" + ${validation_category}) + endfunction() + + if(SIMDLIB_REGISTER_COMPILER_SUPPORTED) + simdlib_add_catch_test(RegisterAvx2Tests tests/Register.tests.cpp + Register.AVX2 "REGISTER;AVX2") + target_sources(RegisterAvx2Tests PRIVATE + tests/RegisterBasicOperations.tests.cpp + tests/RegisterSpecializedOperations.tests.cpp + tests/RegisterRearrangementConversion.tests.cpp + tests/LogicalShuffleRegister.tests.cpp + tests/RegisterOperationMatrix.tests.cpp) + target_link_libraries(RegisterAvx2Tests PRIVATE SimdLib::Register) + target_compile_definitions(RegisterAvx2Tests PRIVATE + SIMDLIB_REGISTER_TEST_ENABLE_256=1) + simdlib_enable_register_avx2(RegisterAvx2Tests) + + simdlib_add_catch_test(RegisterSse42Tests tests/Register.tests.cpp + Register.SSE42 "REGISTER;SSE42") + target_sources(RegisterSse42Tests PRIVATE + tests/RegisterBasicOperations.tests.cpp + tests/RegisterSpecializedOperations.tests.cpp + tests/RegisterRearrangementConversion.tests.cpp + tests/LogicalShuffleRegister.tests.cpp + tests/RegisterOperationMatrix.tests.cpp) + target_link_libraries(RegisterSse42Tests PRIVATE SimdLib::Register) + target_compile_definitions(RegisterSse42Tests PRIVATE + SIMDLIB_REGISTER_TEST_ENABLE_256=0) + simdlib_enable_register_sse42(RegisterSse42Tests) + + add_executable(RegisterPreconditionTests tests/RegisterPreconditionFailure.tests.cpp) + simdlib_register_development_target(RegisterPreconditionTests + CHECKS_VALIDATION) + target_link_libraries(RegisterPreconditionTests PRIVATE SimdLib::Register Catch2::Catch2WithMain) + simdlib_enable_development_warnings(RegisterPreconditionTests) + simdlib_set_coverage_profile_prefix(RegisterPreconditionTests + "Register.AVX2Preconditions") + simdlib_enable_register_sse42(RegisterPreconditionTests) + catch_discover_tests(RegisterPreconditionTests + TEST_PREFIX "Register.AVX2Preconditions." + TEST_LIST RegisterPreconditionTests_DISCOVERED_TESTS + PROPERTIES + PASS_REGULAR_EXPRESSION "SIMDLIB_REGISTER_PRECONDITION_FAILURE_EXPECTED_61B4C2" + TIMEOUT 10) + simdlib_label_discovered_tests(RegisterPreconditionTests_DISCOVERED_TESTS + "REGISTER;PRECONDITIONS;AVX2" CHECKS_VALIDATION) + endif() + + simdlib_add_catch_test(BmiPortableTests tests/Bmi.tests.cpp + BmiPortable "BMI;PORTABLE") + target_compile_definitions(BmiPortableTests PRIVATE + SIMDLIB_HAS_BMI1=0 SIMDLIB_HAS_BMI2=0 + SIMDLIB_BMI_EXPECT_BMI1=0 SIMDLIB_BMI_EXPECT_BMI2=0) + if(NOT SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(BmiPortableTests PRIVATE -mno-bmi -mno-bmi2) + endif() + + simdlib_add_catch_test(FormatTests tests/Format.tests.cpp + Format "FORMAT;SSE42") + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_definitions(FormatTests PRIVATE + SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(FormatTests PRIVATE /arch:AVX2) + endif() + else() + target_compile_options(FormatTests PRIVATE -msse4.2) + endif() + + if(SIMDLIB_BUILD_SMOKE_TESTS) + add_executable(FormatOdr + tests/format_odr/main.cpp + tests/format_odr/second_translation_unit.cpp) + simdlib_register_development_target(FormatOdr SMOKE_VALIDATION) + target_link_libraries(FormatOdr PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(FormatOdr) + add_test(NAME FormatOdr COMMAND FormatOdr) + set_tests_properties(FormatOdr PROPERTIES LABELS "FORMAT;ODR") + simdlib_register_development_test(FormatOdr SMOKE_VALIDATION) + simdlib_set_coverage_profile_prefix(FormatOdr "FormatOdr") + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_definitions(FormatOdr PRIVATE + SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(FormatOdr PRIVATE /arch:AVX2) + endif() + else() + target_compile_options(FormatOdr PRIVATE -msse4.2) + endif() + endif() + + if(SIMDLIB_BUILD_API_SSE42_TESTS) + simdlib_add_catch_test(LogicalShuffleImpl128Tests tests/LogicalShuffleImpl128.tests.cpp + LogicalShuffle.Impl128 "LOGICAL_SHUFFLE;SSE42") + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_definitions(LogicalShuffleImpl128Tests PRIVATE + SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(LogicalShuffleImpl128Tests PRIVATE /arch:AVX2) + endif() + else() + target_compile_options(LogicalShuffleImpl128Tests PRIVATE -msse4.2) + endif() + + simdlib_add_catch_test(ApiSse42Tests tests/Api128.tests.cpp + Api.SSE42 "SSE42") + target_sources(ApiSse42Tests PRIVATE + tests/LogicalShuffleApi.tests.cpp + tests/ImmediateControlSlowPaths.tests.cpp + tests/CompleteRegisterShift.tests.cpp) + target_compile_definitions(ApiSse42Tests PRIVATE + SIMDLIB_LOGICAL_SHUFFLE_TEST_WIDTH=128 + SIMDLIB_IMMEDIATE_CONTROL_TEST_WIDTH=128 + SIMDLIB_COMPLETE_SHIFT_TEST_WIDTH=128) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_definitions(ApiSse42Tests PRIVATE + SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(ApiSse42Tests PRIVATE /arch:AVX2) + endif() + else() + target_compile_options(ApiSse42Tests PRIVATE -msse4.2) + endif() + + simdlib_add_catch_test(UInt128OptimizedTests tests/UInt128.tests.cpp + UInt128Optimized "UINT128;OPTIMIZED;SSE42") + simdlib_add_catch_test(UInt128PortableTests tests/UInt128.tests.cpp + UInt128Portable "UINT128;PORTABLE;SSE42") + simdlib_add_catch_test(UInt128ScalarTests tests/UInt128.tests.cpp + UInt128Scalar "UINT128;PORTABLE;SCALAR") + foreach(uint128_target IN ITEMS + UInt128OptimizedTests UInt128PortableTests UInt128ScalarTests) + target_compile_definitions(${uint128_target} PRIVATE + SIMDLIB_TEST_CONSTEXPR_ASSERTIONS=$) + endforeach() + target_compile_definitions(UInt128PortableTests PRIVATE + SIMDLIB_USE_COMPILER_CARRY_INTRINSICS=0 SIMDLIB_EXPECT_CARRY_PATH=0) + target_compile_definitions(UInt128ScalarTests PRIVATE SIMDLIB_EXPECT_CARRY_PATH=0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_definitions(UInt128OptimizedTests PRIVATE SIMDLIB_EXPECT_CARRY_PATH=1) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + target_compile_definitions(UInt128OptimizedTests PRIVATE SIMDLIB_EXPECT_CARRY_PATH=2) + endif() + target_compile_definitions(UInt128ScalarTests PRIVATE + SIMDLIB_USE_COMPILER_CARRY_INTRINSICS=0 + SIMDLIB_HAS_SSE=0 SIMDLIB_HAS_SSE2=0 SIMDLIB_HAS_SSE3=0 SIMDLIB_HAS_SSSE3=0 + SIMDLIB_HAS_SSE41=0 SIMDLIB_HAS_SSE42=0 SIMDLIB_HAS_AVX=0 SIMDLIB_HAS_AVX2=0 + SIMDLIB_HAS_FMA=0 SIMDLIB_HAS_BMI1=0 SIMDLIB_HAS_BMI2=0) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_definitions(UInt128OptimizedTests PRIVATE + SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) + target_compile_definitions(UInt128PortableTests PRIVATE + SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(UInt128OptimizedTests PRIVATE /arch:AVX2) + target_compile_options(UInt128PortableTests PRIVATE /arch:AVX2) + endif() + else() + target_compile_options(UInt128OptimizedTests PRIVATE -msse4.2) + target_compile_options(UInt128PortableTests PRIVATE -msse4.2) + endif() + add_test(NAME UInt128ResultSetEquivalence + COMMAND ${CMAKE_COMMAND} + -DPORTABLE_EXECUTABLE=$ + -DOPTIMIZED_EXECUTABLE=$ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareUInt128ResultSets.cmake) + set_tests_properties(UInt128ResultSetEquivalence PROPERTIES LABELS "UINT128;EQUIVALENCE;SSE42") + simdlib_register_development_test(UInt128ResultSetEquivalence RUNTIME_VALIDATION) + + add_test(NAME UInt128ScalarResultSetEquivalence + COMMAND ${CMAKE_COMMAND} + -DPORTABLE_EXECUTABLE=$ + -DOPTIMIZED_EXECUTABLE=$ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareUInt128ResultSets.cmake) + set_tests_properties(UInt128ScalarResultSetEquivalence PROPERTIES LABELS "UINT128;EQUIVALENCE;SCALAR") + simdlib_register_development_test(UInt128ScalarResultSetEquivalence RUNTIME_VALIDATION) + endif() + + if(SIMDLIB_BUILD_API_AVX2_TESTS) + simdlib_add_catch_test(LogicalShuffleImpl256Tests tests/LogicalShuffleImpl256.tests.cpp + LogicalShuffle.Impl256 "LOGICAL_SHUFFLE;AVX2") + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(LogicalShuffleImpl256Tests PRIVATE /arch:AVX2) + else() + target_compile_options(LogicalShuffleImpl256Tests PRIVATE -mavx2) + endif() + + simdlib_add_catch_test(ApiAvx2Tests tests/Api256.tests.cpp + Api.AVX2 "AVX2") + target_sources(ApiAvx2Tests PRIVATE + tests/LogicalShuffleApi.tests.cpp + tests/ImmediateControlSlowPaths.tests.cpp + tests/CompleteRegisterShift.tests.cpp) + target_compile_definitions(ApiAvx2Tests PRIVATE + SIMDLIB_LOGICAL_SHUFFLE_TEST_WIDTH=256 + SIMDLIB_IMMEDIATE_CONTROL_TEST_WIDTH=256 + SIMDLIB_COMPLETE_SHIFT_TEST_WIDTH=256) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(ApiAvx2Tests PRIVATE /arch:AVX2) + else() + target_compile_options(ApiAvx2Tests PRIVATE -mavx2) + endif() + endif() + + if(SIMDLIB_BUILD_FMA_TESTS) + simdlib_add_catch_test(FmaEnabledTests tests/SimdFma.tests.cpp + FMA.Enabled "FMA;ENABLED") + simdlib_add_catch_test(FmaDisabledTests tests/SimdFma.tests.cpp + FMA.Disabled "FMA;DISABLED") + target_compile_definitions(FmaEnabledTests PRIVATE SIMDLIB_HAS_FMA=1 SIMDLIB_EXPECT_FMA=1) + target_compile_definitions(FmaDisabledTests PRIVATE SIMDLIB_HAS_FMA=0 SIMDLIB_EXPECT_FMA=0) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(FmaEnabledTests PRIVATE /arch:AVX2) + target_compile_options(FmaDisabledTests PRIVATE /arch:AVX2) + else() + target_compile_options(FmaEnabledTests PRIVATE -mavx2 -mfma) + target_compile_options(FmaDisabledTests PRIVATE -mavx2 -mno-fma) + endif() + endif() + + if(SIMDLIB_BUILD_BMI_TESTS) + # @brief Adds one runtime test executable for a BMI feature combination. + # @param profile_name Stable suffix identifying the enabled BMI features. + # @param bmi1 Whether BMI1 is enabled for this profile. + # @param bmi2 Whether BMI2 is enabled for this profile. + function(simdlib_add_bmi_profile profile_name bmi1 bmi2) + set(target Bmi${profile_name}Tests) + set(test_name Bmi.Bmi${profile_name}) + simdlib_add_catch_test(${target} tests/Bmi.tests.cpp ${test_name} + "BMI;${profile_name};OPTIONAL") + target_compile_definitions(${target} PRIVATE + SIMDLIB_HAS_BMI1=${bmi1} SIMDLIB_HAS_BMI2=${bmi2} + SIMDLIB_BMI_EXPECT_BMI1=${bmi1} SIMDLIB_BMI_EXPECT_BMI2=${bmi2}) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(${target} PRIVATE /arch:AVX2) + else() + target_compile_options(${target} PRIVATE -mno-bmi -mno-bmi2) + if(bmi1) + target_compile_options(${target} PRIVATE -mbmi) + endif() + if(bmi2) + target_compile_options(${target} PRIVATE -mbmi2) + endif() + endif() + set(equivalence_name Bmi.Bmi${profile_name}.Equivalence) + add_test(NAME ${equivalence_name} + COMMAND ${CMAKE_COMMAND} + -DPORTABLE_EXECUTABLE=$ + -DENABLED_EXECUTABLE=$ + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CompareBmiResultSets.cmake) + set_tests_properties(${equivalence_name} PROPERTIES LABELS "BMI;EQUIVALENCE;${profile_name};OPTIONAL") + simdlib_register_development_test(${equivalence_name} RUNTIME_VALIDATION) + endfunction() + + simdlib_add_bmi_profile(1 1 0) + simdlib_add_bmi_profile(2 0 1) + simdlib_add_bmi_profile(1Bmi2 1 1) + endif() + + if(SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS) + add_executable(VectorAlgorithmsTests + tests/SimdVector.tests.cpp + tests/SimdAlgo.tests.cpp + tests/PreconditionBoundary.tests.cpp + tests/SimdResample.tests.cpp) + simdlib_register_development_target(VectorAlgorithmsTests + RUNTIME_VALIDATION) + target_link_libraries(VectorAlgorithmsTests PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) + simdlib_enable_development_warnings(VectorAlgorithmsTests) + simdlib_set_coverage_profile_prefix(VectorAlgorithmsTests + "VectorAlgorithms") + catch_discover_tests(VectorAlgorithmsTests + TEST_PREFIX "VectorAlgorithms." + TEST_LIST VectorAlgorithmsTests_DISCOVERED_TESTS) + simdlib_label_discovered_tests(VectorAlgorithmsTests_DISCOVERED_TESTS + "VECTOR_ALGORITHMS;AVX2" RUNTIME_VALIDATION) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(VectorAlgorithmsTests PRIVATE /arch:AVX2) + else() + target_compile_options(VectorAlgorithmsTests PRIVATE -mavx2 -mfma) + endif() + + simdlib_add_catch_test(VectorChecksTests tests/SimdVectorChecks.tests.cpp + VectorChecks "VECTOR_ALGORITHMS;AVX2;CHECKS" CHECKS_VALIDATION) + target_compile_definitions(VectorChecksTests PRIVATE SIMDLIB_ENABLE_CHECKS=1) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(VectorChecksTests PRIVATE /arch:AVX2) + else() + target_compile_options(VectorChecksTests PRIVATE -mavx2 -mfma) + endif() + + add_executable(PreconditionTests tests/PreconditionFailure.tests.cpp) + simdlib_register_development_target(PreconditionTests CHECKS_VALIDATION) + target_link_libraries(PreconditionTests PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) + simdlib_enable_development_warnings(PreconditionTests) + simdlib_set_coverage_profile_prefix(PreconditionTests + "Preconditions") + target_compile_definitions(PreconditionTests PRIVATE SIMDLIB_ENABLE_CHECKS=1) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(PreconditionTests PRIVATE /arch:AVX2) + else() + target_compile_options(PreconditionTests PRIVATE -mavx2 -mfma) + endif() + catch_discover_tests(PreconditionTests + TEST_PREFIX "Preconditions." + TEST_LIST PreconditionTests_DISCOVERED_TESTS + PROPERTIES + PASS_REGULAR_EXPRESSION "SIMDLIB_PRECONDITION_FAILURE_EXPECTED_18A7E3" + TIMEOUT 10) + simdlib_label_discovered_tests(PreconditionTests_DISCOVERED_TESTS + "PRECONDITIONS;CHECKS;AVX2" CHECKS_VALIDATION) + + add_executable(ResampleScalarTests tests/SimdResample.tests.cpp) + simdlib_register_development_target(ResampleScalarTests + RUNTIME_VALIDATION) + target_link_libraries(ResampleScalarTests PRIVATE SimdLib::SimdLib Catch2::Catch2WithMain) + simdlib_enable_development_warnings(ResampleScalarTests) + simdlib_set_coverage_profile_prefix(ResampleScalarTests + "ResampleScalar") + target_compile_definitions(ResampleScalarTests PRIVATE + SIMDLIB_HAS_SSE3=0 SIMDLIB_HAS_SSSE3=0 SIMDLIB_HAS_SSE41=0 SIMDLIB_HAS_SSE42=0 + SIMDLIB_HAS_AVX=0 SIMDLIB_HAS_AVX2=0 SIMDLIB_HAS_FMA=0) + catch_discover_tests(ResampleScalarTests + TEST_PREFIX "ResampleScalar." + TEST_LIST ResampleScalarTests_DISCOVERED_TESTS) + simdlib_label_discovered_tests(ResampleScalarTests_DISCOVERED_TESTS + "VECTOR_ALGORITHMS;SCALAR" RUNTIME_VALIDATION) + endif() +endif() + +endblock() diff --git a/cmake/development/SmokeTests.cmake b/cmake/development/SmokeTests.cmake new file mode 100644 index 0000000..13d3e6e --- /dev/null +++ b/cmake/development/SmokeTests.cmake @@ -0,0 +1,39 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "SmokeTests.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "SmokeTests.cmake requires the production SimdLib targets") +endif() + +block(SCOPE_FOR VARIABLES) + +if(SIMDLIB_BUILD_SMOKE_TESTS) + add_executable(HeaderOnlySmoke + tests/smoke/main.cpp + tests/smoke/second_translation_unit.cpp) + simdlib_register_development_target(HeaderOnlySmoke SMOKE_VALIDATION) + target_link_libraries(HeaderOnlySmoke PRIVATE SimdLib::SimdLib) + simdlib_enable_development_warnings(HeaderOnlySmoke) + add_test(NAME HeaderOnlySmoke COMMAND HeaderOnlySmoke) + simdlib_register_development_test(HeaderOnlySmoke SMOKE_VALIDATION) + simdlib_set_coverage_profile_prefix(HeaderOnlySmoke + "HeaderOnlySmoke") + + if(SIMDLIB_REGISTER_COMPILER_SUPPORTED) + add_executable(RegisterOdr + tests/register_odr/main.cpp + tests/register_odr/second_translation_unit.cpp) + simdlib_register_development_target(RegisterOdr SMOKE_VALIDATION) + target_link_libraries(RegisterOdr PRIVATE SimdLib::Register) + simdlib_enable_development_warnings(RegisterOdr) + simdlib_enable_register_sse42(RegisterOdr) + add_test(NAME RegisterOdr COMMAND RegisterOdr) + set_tests_properties(RegisterOdr PROPERTIES LABELS "REGISTER;ODR;SSE42") + simdlib_register_development_test(RegisterOdr SMOKE_VALIDATION) + simdlib_set_coverage_profile_prefix(RegisterOdr "RegisterOdr") + endif() +endif() + +endblock() diff --git a/cmake/development/TargetConfiguration.cmake b/cmake/development/TargetConfiguration.cmake new file mode 100644 index 0000000..54fed9f --- /dev/null +++ b/cmake/development/TargetConfiguration.cmake @@ -0,0 +1,89 @@ +include_guard(GLOBAL) + +if(NOT PROJECT_IS_TOP_LEVEL) + message(FATAL_ERROR "TargetConfiguration.cmake is available only to top-level SimdLib builds") +endif() +if(NOT TARGET SimdLib OR NOT TARGET SimdLibRegister) + message(FATAL_ERROR "TargetConfiguration.cmake requires the production SimdLib targets") +endif() + +add_library(DevelopmentWarnings INTERFACE) +if(SIMDLIB_ENABLE_COVERAGE) + if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR SIMDLIB_MSVC_STYLE_DRIVER) + message(FATAL_ERROR "SIMDLIB_ENABLE_COVERAGE currently requires Clang with its GNU-like command-line driver") + endif() + target_compile_options(DevelopmentWarnings INTERFACE + -fprofile-instr-generate -fcoverage-mapping) + target_link_options(DevelopmentWarnings INTERFACE + -fprofile-instr-generate) +endif() +if(SIMDLIB_STRICT_WARNINGS) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(DevelopmentWarnings INTERFACE + /W4 /WX /permissive- + -Wno-unknown-attributes -Wno-ignored-attributes -Wno-c2y-extensions) + else() + target_compile_options(DevelopmentWarnings INTERFACE + -Wall -Wextra -Wpedantic -Werror + -Wno-unknown-attributes -Wno-ignored-attributes -Wno-c2y-extensions) + endif() + elseif(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(DevelopmentWarnings INTERFACE /W4 /WX /permissive-) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_compile_options(DevelopmentWarnings INTERFACE + -Wall -Wextra -Wpedantic -Werror + -Wno-attributes -Wno-ignored-attributes) + endif() +endif() + +# @brief Applies shared development warnings and coverage instrumentation. +# @param target Target that receives the development usage requirements. +function(simdlib_enable_development_warnings target) + target_link_libraries(${target} PRIVATE DevelopmentWarnings) + if(SIMDLIB_ENABLE_COVERAGE) + get_target_property(target_type ${target} TYPE) + if(target_type STREQUAL "EXECUTABLE") + set_property(GLOBAL APPEND PROPERTY SIMDLIB_COVERAGE_TARGETS ${target}) + endif() + endif() +endfunction() + +# @brief Compiles one target for the supported 128-bit SSE4.2 Register profile. +# @param target Target that must not acquire AVX-family availability. +function(simdlib_enable_register_sse42 target) + target_compile_definitions(${target} PRIVATE + SIMDLIB_HAS_SSE=1 SIMDLIB_HAS_SSE2=1 SIMDLIB_HAS_SSE3=1 + SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1 + SIMDLIB_HAS_AVX=0 SIMDLIB_HAS_AVX2=0 SIMDLIB_HAS_FMA=0) + if(SIMDLIB_MSVC_STYLE_DRIVER) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(${target} PRIVATE + /clang:-msse4.2 /clang:-mno-avx /clang:-mno-avx2 /clang:-mno-fma) + endif() + else() + target_compile_options(${target} PRIVATE + -msse4.2 -mno-avx -mno-avx2 -mno-fma) + endif() +endfunction() + +# @brief Compiles one target for the supported 128-bit and 256-bit AVX2 Register profile. +# @param target Target that receives AVX2 code-generation options. +function(simdlib_enable_register_avx2 target) + if(SIMDLIB_MSVC_STYLE_DRIVER) + target_compile_options(${target} PRIVATE /arch:AVX2) + else() + target_compile_options(${target} PRIVATE -mavx2) + endif() +endfunction() + +# @brief Associates an instrumented executable with the prefix CTest uses for +# the profiles produced by that executable. +# @param target Instrumented executable target. +# @param profile_prefix Prefix shared by every CTest profile for the target. +function(simdlib_set_coverage_profile_prefix target profile_prefix) + if(SIMDLIB_ENABLE_COVERAGE) + set_property(TARGET ${target} PROPERTY + SIMDLIB_COVERAGE_PROFILE_PREFIX "${profile_prefix}") + endif() +endfunction() diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..7780d4d --- /dev/null +++ b/compose.yml @@ -0,0 +1,63 @@ +name: simdlib-container + +x-simdlib-service: &simdlib-service + init: true + working_dir: /workspace/source + read_only: true + user: "${SIMDLIB_HOST_UID:-1000}:${SIMDLIB_HOST_GID:-1000}" + volumes: + - ./:/workspace/source:ro + - ./out/pipeline:/workspace/out:rw + tmpfs: + - /tmp:exec,mode=1777 + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + environment: + BUILDKITE: "${BUILDKITE:-}" + CI: "${CI:-}" + CIRCLECI: "${CIRCLECI:-}" + CMAKE_BUILD_PARALLEL_LEVEL: "${SIMDLIB_PARALLEL_LEVEL:-2}" + CTEST_PARALLEL_LEVEL: "${SIMDLIB_PARALLEL_LEVEL:-2}" + GITHUB_ACTIONS: "${GITHUB_ACTIONS:-}" + GITLAB_CI: "${GITLAB_CI:-}" + HOME: /tmp + JENKINS_URL: "${JENKINS_URL:-}" + SIMDLIB_BUILD_REVISION: "${SIMDLIB_BUILD_REVISION:-unknown}" + TEAMCITY_VERSION: "${TEAMCITY_VERSION:-}" + TF_BUILD: "${TF_BUILD:-}" + command: + - --operation + - build-validation + - --preset + - "${SIMDLIB_CONTAINER_PRESET:-container-release-contracts}" + - --build-profile + - "${SIMDLIB_CONTAINER_BUILD_PROFILE:-Release}" + - --sanitizer + - "${SIMDLIB_CONTAINER_SANITIZER:-none}" + +services: + gcc13: + <<: *simdlib-service + image: simdlib/gcc13:local + build: + context: . + dockerfile: containers/Dockerfile.gcc13 + profiles: [compilers] + + gcc14: + <<: *simdlib-service + image: simdlib/gcc14:local + build: + context: . + dockerfile: containers/Dockerfile.gcc14 + profiles: [compilers] + + clang22: + <<: *simdlib-service + image: simdlib/clang22:local + build: + context: . + dockerfile: containers/Dockerfile.clang22 + profiles: [compilers] diff --git a/containers/Dockerfile.clang22 b/containers/Dockerfile.clang22 new file mode 100644 index 0000000..a9ddb9c --- /dev/null +++ b/containers/Dockerfile.clang22 @@ -0,0 +1,93 @@ +# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e + +ARG ALPINE_IMAGE=alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b + +FROM ${ALPINE_IMAGE} AS tools + +ARG CMAKE_VERSION=4.4.0 +ARG CMAKE_SHA256=65757f442fdd242e27f1728fc26dc0cba4164f7a0791a5c788631c00080369bc +ARG CATCH2_COMMIT=2b60af89e23d28eefc081bc930831ee9d45ea58b + +RUN apk add --no-cache \ + ca-certificates \ + cmake=4.2.3-r0 \ + g++=15.2.0-r5 \ + gcc=15.2.0-r5 \ + git=2.54.0-r0 \ + linux-headers=7.0.0-r1 \ + make=4.4.1-r4 \ + ninja-is-really-ninja=1.13.2-r1 \ + openssl-dev=3.5.7-r0 \ + && wget -q "https://cmake.org/files/v4.4/cmake-${CMAKE_VERSION}.tar.gz" -O /tmp/cmake.tar.gz \ + && echo "${CMAKE_SHA256} /tmp/cmake.tar.gz" | sha256sum -c - \ + && mkdir /tmp/cmake-source \ + && tar -xzf /tmp/cmake.tar.gz -C /tmp/cmake-source --strip-components=1 \ + && cmake -S /tmp/cmake-source -B /tmp/cmake-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/cmake \ + -DBUILD_TESTING=OFF \ + && cmake --build /tmp/cmake-build --parallel \ + && cmake --install /tmp/cmake-build \ + && strip /opt/cmake/bin/cmake /opt/cmake/bin/ctest \ + && rm -f /opt/cmake/bin/cpack \ + && rm -rf /opt/cmake/doc /opt/cmake/man /opt/cmake/share/aclocal \ + /opt/cmake/share/bash-completion /opt/cmake/share/emacs \ + /opt/cmake/share/vim /opt/cmake/share/cmake-4.4/Help \ + /tmp/cmake.tar.gz /tmp/cmake-source /tmp/cmake-build + +RUN git init /opt/catch2 \ + && git -C /opt/catch2 remote add origin https://github.com/catchorg/Catch2.git \ + && git -C /opt/catch2 fetch --depth 1 origin "${CATCH2_COMMIT}" \ + && git -C /opt/catch2 checkout --detach FETCH_HEAD \ + && test "$(git -C /opt/catch2 rev-parse HEAD)" = "${CATCH2_COMMIT}" \ + && rm -rf /opt/catch2/.github /opt/catch2/docs /opt/catch2/examples \ + /opt/catch2/tests /opt/catch2/fuzzing /opt/catch2/benchmark \ + /opt/catch2/.git + +FROM ${ALPINE_IMAGE} + +LABEL org.opencontainers.image.title="SimdLib Clang 22 validation" \ + org.opencontainers.image.description="Pinned Alpine/musl Clang 22 environment for SimdLib" \ + org.opencontainers.image.source="https://github.com/dsisco11/SimdLib" \ + org.opencontainers.image.version="clang-22.1.3-cmake-4.4.0" \ + org.simdlib.base.digest="sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" \ + org.simdlib.cmake.sha256="65757f442fdd242e27f1728fc26dc0cba4164f7a0791a5c788631c00080369bc" \ + org.simdlib.catch2.commit="2b60af89e23d28eefc081bc930831ee9d45ea58b" + +RUN apk add --no-cache \ + binutils=2.45.1-r1 \ + ca-certificates \ + clang22=22.1.3-r2 \ + compiler-rt=22.1.3-r0 \ + libc++-dev=22.1.3-r0 \ + lld22=22.1.3-r0 \ + musl-dev=1.2.6-r2 \ + ninja-is-really-ninja=1.13.2-r1 \ + llvm-libunwind-dev=22.1.3-r0 \ + openssl=3.5.7-r0 \ + strace=6.19-r1 \ + && addgroup -g 1000 simdlib \ + && adduser -D -u 1000 -G simdlib simdlib \ + && mkdir -p /workspace/out \ + && chown -R simdlib:simdlib /workspace + +COPY --from=tools /opt/cmake /opt/cmake +COPY --from=tools /opt/catch2 /opt/catch2 +COPY --chmod=755 containers/container-entrypoint.sh /usr/local/bin/simdlib-container + +ENV PATH="/opt/cmake/bin:${PATH}" \ + CC=clang-22 \ + CXX=clang++-22 \ + SIMDLIB_COMPILER_ID=clang22 \ + SIMDLIB_CATCH2_SOURCE=/opt/catch2 \ + SIMDLIB_BASE_IMAGE="alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" \ + SIMDLIB_REQUIRED_CXX_FLAGS="-stdlib=libc++" \ + SIMDLIB_REQUIRED_LINKER_FLAGS="-fuse-ld=lld --rtlib=compiler-rt --unwindlib=libunwind" \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + TZ=UTC + +USER simdlib +WORKDIR /workspace/source +ENTRYPOINT ["/usr/local/bin/simdlib-container"] +CMD ["--operation", "build-validation", "--preset", "clang22-release-exhaustive"] diff --git a/containers/Dockerfile.gcc13 b/containers/Dockerfile.gcc13 new file mode 100644 index 0000000..41b674a --- /dev/null +++ b/containers/Dockerfile.gcc13 @@ -0,0 +1,87 @@ +# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e + +ARG ALPINE_IMAGE=alpine:3.20.8@sha256:765942a4039992336de8dd5db680586e1a206607dd06170ff0a37267a9e01958 + +FROM ${ALPINE_IMAGE} AS tools + +ARG CMAKE_VERSION=4.4.0 +ARG CMAKE_SHA256=65757f442fdd242e27f1728fc26dc0cba4164f7a0791a5c788631c00080369bc +ARG CATCH2_COMMIT=2b60af89e23d28eefc081bc930831ee9d45ea58b + +RUN apk add --no-cache \ + ca-certificates \ + cmake=3.29.3-r0 \ + g++=13.2.1_git20240309-r1 \ + gcc=13.2.1_git20240309-r1 \ + git=2.45.4-r0 \ + linux-headers=6.6-r0 \ + make=4.4.1-r2 \ + ninja-is-really-ninja=1.12.1-r0 \ + openssl-dev=3.3.7-r0 \ + && wget -q "https://cmake.org/files/v4.4/cmake-${CMAKE_VERSION}.tar.gz" -O /tmp/cmake.tar.gz \ + && echo "${CMAKE_SHA256} /tmp/cmake.tar.gz" | sha256sum -c - \ + && mkdir /tmp/cmake-source \ + && tar -xzf /tmp/cmake.tar.gz -C /tmp/cmake-source --strip-components=1 \ + && cmake -S /tmp/cmake-source -B /tmp/cmake-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/cmake \ + -DBUILD_TESTING=OFF \ + && cmake --build /tmp/cmake-build --parallel \ + && cmake --install /tmp/cmake-build \ + && strip /opt/cmake/bin/cmake /opt/cmake/bin/ctest \ + && rm -f /opt/cmake/bin/cpack \ + && rm -rf /opt/cmake/doc /opt/cmake/man /opt/cmake/share/aclocal \ + /opt/cmake/share/bash-completion /opt/cmake/share/emacs \ + /opt/cmake/share/vim /opt/cmake/share/cmake-4.4/Help \ + /tmp/cmake.tar.gz /tmp/cmake-source /tmp/cmake-build + +RUN git init /opt/catch2 \ + && git -C /opt/catch2 remote add origin https://github.com/catchorg/Catch2.git \ + && git -C /opt/catch2 fetch --depth 1 origin "${CATCH2_COMMIT}" \ + && git -C /opt/catch2 checkout --detach FETCH_HEAD \ + && test "$(git -C /opt/catch2 rev-parse HEAD)" = "${CATCH2_COMMIT}" \ + && rm -rf /opt/catch2/.github /opt/catch2/docs /opt/catch2/examples \ + /opt/catch2/tests /opt/catch2/fuzzing /opt/catch2/benchmark \ + /opt/catch2/.git + +FROM ${ALPINE_IMAGE} + +LABEL org.opencontainers.image.title="SimdLib GCC 13.2 validation" \ + org.opencontainers.image.description="Pinned Alpine/musl GCC 13.2 environment for SimdLib" \ + org.opencontainers.image.source="https://github.com/dsisco11/SimdLib" \ + org.opencontainers.image.version="gcc-13.2.1-cmake-4.4.0" \ + org.simdlib.base.digest="sha256:765942a4039992336de8dd5db680586e1a206607dd06170ff0a37267a9e01958" \ + org.simdlib.cmake.sha256="65757f442fdd242e27f1728fc26dc0cba4164f7a0791a5c788631c00080369bc" \ + org.simdlib.catch2.commit="2b60af89e23d28eefc081bc930831ee9d45ea58b" + +RUN apk add --no-cache \ + ca-certificates \ + g++=13.2.1_git20240309-r1 \ + gcc=13.2.1_git20240309-r1 \ + musl-dev=1.2.5-r3 \ + ninja-is-really-ninja=1.12.1-r0 \ + openssl=3.3.7-r0 \ + strace=6.9-r0 \ + && addgroup -g 1000 simdlib \ + && adduser -D -u 1000 -G simdlib simdlib \ + && mkdir -p /workspace/out \ + && chown -R simdlib:simdlib /workspace + +COPY --from=tools /opt/cmake /opt/cmake +COPY --from=tools /opt/catch2 /opt/catch2 +COPY --chmod=755 containers/container-entrypoint.sh /usr/local/bin/simdlib-container + +ENV PATH="/opt/cmake/bin:${PATH}" \ + CC=gcc \ + CXX=g++ \ + SIMDLIB_COMPILER_ID=gcc13 \ + SIMDLIB_CATCH2_SOURCE=/opt/catch2 \ + SIMDLIB_BASE_IMAGE="alpine:3.20.8@sha256:765942a4039992336de8dd5db680586e1a206607dd06170ff0a37267a9e01958" \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + TZ=UTC + +USER simdlib +WORKDIR /workspace/source +ENTRYPOINT ["/usr/local/bin/simdlib-container"] +CMD ["--operation", "build-validation", "--preset", "gcc13-core-release-exhaustive"] diff --git a/containers/Dockerfile.gcc14 b/containers/Dockerfile.gcc14 new file mode 100644 index 0000000..6f0fb44 --- /dev/null +++ b/containers/Dockerfile.gcc14 @@ -0,0 +1,87 @@ +# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e + +ARG ALPINE_IMAGE=alpine:3.22.5@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce + +FROM ${ALPINE_IMAGE} AS tools + +ARG CMAKE_VERSION=4.4.0 +ARG CMAKE_SHA256=65757f442fdd242e27f1728fc26dc0cba4164f7a0791a5c788631c00080369bc +ARG CATCH2_COMMIT=2b60af89e23d28eefc081bc930831ee9d45ea58b + +RUN apk add --no-cache \ + ca-certificates \ + cmake=3.31.7-r1 \ + g++=14.2.0-r6 \ + gcc=14.2.0-r6 \ + git=2.49.1-r0 \ + linux-headers=6.14.2-r0 \ + make=4.4.1-r3 \ + ninja-is-really-ninja=1.12.1-r1 \ + openssl-dev=3.5.7-r0 \ + && wget -q "https://cmake.org/files/v4.4/cmake-${CMAKE_VERSION}.tar.gz" -O /tmp/cmake.tar.gz \ + && echo "${CMAKE_SHA256} /tmp/cmake.tar.gz" | sha256sum -c - \ + && mkdir /tmp/cmake-source \ + && tar -xzf /tmp/cmake.tar.gz -C /tmp/cmake-source --strip-components=1 \ + && cmake -S /tmp/cmake-source -B /tmp/cmake-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/cmake \ + -DBUILD_TESTING=OFF \ + && cmake --build /tmp/cmake-build --parallel \ + && cmake --install /tmp/cmake-build \ + && strip /opt/cmake/bin/cmake /opt/cmake/bin/ctest \ + && rm -f /opt/cmake/bin/cpack \ + && rm -rf /opt/cmake/doc /opt/cmake/man /opt/cmake/share/aclocal \ + /opt/cmake/share/bash-completion /opt/cmake/share/emacs \ + /opt/cmake/share/vim /opt/cmake/share/cmake-4.4/Help \ + /tmp/cmake.tar.gz /tmp/cmake-source /tmp/cmake-build + +RUN git init /opt/catch2 \ + && git -C /opt/catch2 remote add origin https://github.com/catchorg/Catch2.git \ + && git -C /opt/catch2 fetch --depth 1 origin "${CATCH2_COMMIT}" \ + && git -C /opt/catch2 checkout --detach FETCH_HEAD \ + && test "$(git -C /opt/catch2 rev-parse HEAD)" = "${CATCH2_COMMIT}" \ + && rm -rf /opt/catch2/.github /opt/catch2/docs /opt/catch2/examples \ + /opt/catch2/tests /opt/catch2/fuzzing /opt/catch2/benchmark \ + /opt/catch2/.git + +FROM ${ALPINE_IMAGE} + +LABEL org.opencontainers.image.title="SimdLib GCC 14 validation" \ + org.opencontainers.image.description="Pinned Alpine/musl GCC 14 environment for SimdLib" \ + org.opencontainers.image.source="https://github.com/dsisco11/SimdLib" \ + org.opencontainers.image.version="gcc-14.2.0-cmake-4.4.0" \ + org.simdlib.base.digest="sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce" \ + org.simdlib.cmake.sha256="65757f442fdd242e27f1728fc26dc0cba4164f7a0791a5c788631c00080369bc" \ + org.simdlib.catch2.commit="2b60af89e23d28eefc081bc930831ee9d45ea58b" + +RUN apk add --no-cache \ + ca-certificates \ + g++=14.2.0-r6 \ + gcc=14.2.0-r6 \ + musl-dev=1.2.5-r12 \ + ninja-is-really-ninja=1.12.1-r1 \ + openssl=3.5.7-r0 \ + strace=6.13-r0 \ + && addgroup -g 1000 simdlib \ + && adduser -D -u 1000 -G simdlib simdlib \ + && mkdir -p /workspace/out \ + && chown -R simdlib:simdlib /workspace + +COPY --from=tools /opt/cmake /opt/cmake +COPY --from=tools /opt/catch2 /opt/catch2 +COPY --chmod=755 containers/container-entrypoint.sh /usr/local/bin/simdlib-container + +ENV PATH="/opt/cmake/bin:${PATH}" \ + CC=gcc \ + CXX=g++ \ + SIMDLIB_COMPILER_ID=gcc14 \ + SIMDLIB_CATCH2_SOURCE=/opt/catch2 \ + SIMDLIB_BASE_IMAGE="alpine:3.22.5@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce" \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + TZ=UTC + +USER simdlib +WORKDIR /workspace/source +ENTRYPOINT ["/usr/local/bin/simdlib-container"] +CMD ["--operation", "build-validation", "--preset", "gcc14-release-exhaustive"] diff --git a/containers/container-entrypoint.sh b/containers/container-entrypoint.sh new file mode 100644 index 0000000..ad645e6 --- /dev/null +++ b/containers/container-entrypoint.sh @@ -0,0 +1,778 @@ +#!/bin/sh +set -eu + +source_directory=/workspace/source +operation= +preset=container-release-contracts +test_regex= +test_label= +build_profile= +sanitizer=none +instrumentation=none +codegen_mode=OFF +aggregate=ExhaustiveArtifacts +matrix_cell= +consumer_scope=none +artifact_root="/workspace/out/${SIMDLIB_COMPILER_ID:-unknown}" +fingerprint_sha256= + +## @brief Prints the supported container operation arguments. +print_usage() +{ + cat <<'EOF' +Usage: simdlib-container --operation OPERATION [options] + --operation NAME build-validation, test, test-compiler-contracts, record-codegen, + build-benchmarks, run-benchmarks, or inspect-environment + --preset NAME Owning CMake configure preset + --test-regex REGEX Run only matching CTest tests during test + --test-label REGEX Run only matching CTest labels during test + --build-profile NAME Release or Debug; must agree with the selected preset + --sanitizer MODE none or asan-ubsan + --codegen-mode MODE OFF, ENFORCE, or RECORD + --aggregate NAME Scoped CMake aggregate owned by this operation + --matrix-cell NAME Canonical validation-matrix cell identifier + --consumer-scope SCOPE none or compiler-release + --artifact-root PATH Writable compiler-specific artifact root + --fingerprint-sha256 Full SHA256 of the canonical build-cell fingerprint + --help Show this help +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --operation) operation=$2; shift 2 ;; + --preset) preset=$2; shift 2 ;; + --test-regex) test_regex=$2; shift 2 ;; + --test-label) test_label=$2; shift 2 ;; + --build-profile) build_profile=$2; shift 2 ;; + --sanitizer) sanitizer=$2; shift 2 ;; + --instrumentation) instrumentation=$2; shift 2 ;; + --codegen-mode) codegen_mode=$2; shift 2 ;; + --aggregate) aggregate=$2; shift 2 ;; + --matrix-cell) matrix_cell=$2; shift 2 ;; + --consumer-scope) consumer_scope=$2; shift 2 ;; + --artifact-root) artifact_root=$2; shift 2 ;; + --fingerprint-sha256) fingerprint_sha256=$2; shift 2 ;; + --help) print_usage; exit 0 ;; + *) echo "Unknown argument: $1" >&2; print_usage >&2; exit 2 ;; + esac +done + +case "$operation" in + build-validation|test|test-compiler-contracts|record-codegen|build-benchmarks|run-benchmarks|inspect-environment) ;; + *) echo "A supported --operation is required: ${operation:-}" >&2; exit 2 ;; +esac +case "$artifact_root" in + /workspace/out/*) ;; + *) echo "Artifact root must be below /workspace/out: $artifact_root" >&2; exit 2 ;; +esac +case "$fingerprint_sha256" in + *[!0-9a-f]*|'') echo "A lowercase 64-character --fingerprint-sha256 is required" >&2; exit 2 ;; +esac +[ "${#fingerprint_sha256}" -eq 64 ] || { + echo "A lowercase 64-character --fingerprint-sha256 is required" >&2 + exit 2 +} +fingerprint_prefix=$(printf '%s' "$fingerprint_sha256" | cut -c 1-16) +case "${artifact_root##*/}" in + *-$fingerprint_prefix) ;; + *) echo "Artifact root does not match fingerprint prefix: $artifact_root" >&2; exit 2 ;; +esac +case "$sanitizer" in + none|asan-ubsan) ;; + *) echo "Unsupported sanitizer mode: $sanitizer" >&2; exit 2 ;; +esac +case "$codegen_mode" in + OFF|ENFORCE|RECORD) ;; + *) echo "Unsupported codegen mode: $codegen_mode" >&2; exit 2 ;; +esac +case "$aggregate" in + ExhaustiveArtifacts|SimdLibCompilerContractArtifacts|SimdLibDebugDiagnosticArtifacts) ;; + *) echo "Unsupported scoped aggregate: $aggregate" >&2; exit 2 ;; +esac +[ -n "$matrix_cell" ] || { + echo "A canonical --matrix-cell is required" >&2 + exit 2 +} +case "$consumer_scope" in + none|compiler-release) ;; + *) echo "Unsupported consumer scope: $consumer_scope" >&2; exit 2 ;; +esac +if [ "$operation" = record-codegen ] && [ "$codegen_mode" != RECORD ]; then + echo "The record-codegen operation requires --codegen-mode RECORD" >&2 + exit 2 +fi +case "$preset" in + *debug*|*asan-ubsan-codegen-diagnostic) expected_build_profile=Debug ;; + *) expected_build_profile=Release ;; +esac +[ -n "$build_profile" ] || build_profile=$expected_build_profile +[ "$build_profile" = "$expected_build_profile" ] || { + echo "Build profile $build_profile does not match preset $preset ($expected_build_profile)" >&2 + exit 2 +} +[ "$consumer_scope" != compiler-release ] || [ "$build_profile" = Release ] || { + echo "Compiler Release consumer scope requires a Release profile" >&2 + exit 2 +} +case "$operation" in + build-benchmarks|run-benchmarks) + [ "$build_profile" = Release ] || { + echo "Benchmark operations require a Release fingerprint: $preset" >&2 + exit 2 + } + ;; +esac + +build_directory="$artifact_root/build" +consumer_directory="$artifact_root/consumer" +report_directory="$artifact_root/reports" +provenance_directory="$artifact_root/provenance" +fingerprint_document="$provenance_directory/fingerprint.json" +validation_manifest="$provenance_directory/validation-build.manifest" +benchmark_manifest="$provenance_directory/benchmark-build.manifest" +main_inventory="$provenance_directory/main-test-artifacts.inventory" +consumer_inventory="$provenance_directory/consumer-test-artifacts.inventory" +codegen_record_index="$provenance_directory/codegen-records.index" +target_inventory="$build_directory/development-profile-targets.txt" +matrix_contract="$source_directory/tools/validation-matrix.json" +validation_inventory_audit="$report_directory/validation-inventory.audit.json" +codegen_diagnostic_provenance="$provenance_directory/codegen-diagnostic.json" +mkdir -p "$report_directory" "$provenance_directory" +[ -f "$fingerprint_document" ] || { + echo "Canonical fingerprint document is missing: $fingerprint_document" >&2 + exit 2 +} +[ "$(sha256sum "$fingerprint_document" | cut -d ' ' -f 1)" = "$fingerprint_sha256" ] || { + echo "Canonical fingerprint document does not match --fingerprint-sha256" >&2 + exit 2 +} + +## @brief Runs a test-only operation under process tracing and rejects build processes. +run_traced_test_operation() +{ + trace_temporary="$report_directory/test-only.execve.trace.tmp" + trace_file="$report_directory/test-only.execve.trace" + rm -f "$trace_temporary" + set -- --operation "$operation" --preset "$preset" --build-profile "$build_profile" \ + --sanitizer "$sanitizer" --artifact-root "$artifact_root" \ + --codegen-mode "$codegen_mode" --aggregate "$aggregate" \ + --matrix-cell "$matrix_cell" --consumer-scope "$consumer_scope" \ + --fingerprint-sha256 "$fingerprint_sha256" + [ -z "$test_regex" ] || set -- "$@" --test-regex "$test_regex" + [ -z "$test_label" ] || set -- "$@" --test-label "$test_label" + set +e + strace -f -qq -e trace=execve -o "$trace_temporary" \ + env SIMDLIB_TEST_TRACE_ACTIVE=1 "$0" "$@" + test_status=$? + set -e + if grep -E 'execve\("([^"]*/)?cmake(\.exe)?", \[[^]]*"(--build|--preset)"' \ + "$trace_temporary" >/dev/null || + grep -E 'execve\("([^"]*/)?cmake(\.exe)?", \[[^]]*"-S", "/workspace/source"' \ + "$trace_temporary" >/dev/null || + grep -E 'execve\("([^"]*/)?(ninja|make|msbuild)(\.exe)?"' "$trace_temporary" | + grep -v -- '"--version"' >/dev/null; then + echo "Test-only process trace contains an artifact-tree configure or build invocation" >&2 + test_status=5 + fi + mv "$trace_temporary" "$trace_file" + exit "$test_status" +} + +# LeakSanitizer refuses to execute under ptrace. Sanitizer cells retain the same +# inner test-only operation without tracing; ordinary cells own the trace gate. +if { [ "$operation" = test ] || [ "$operation" = test-compiler-contracts ]; } && + [ -z "${SIMDLIB_TEST_TRACE_ACTIVE:-}" ] && + [ "$sanitizer" != asan-ubsan ]; then + run_traced_test_operation +fi + +## @brief Computes a stable digest of source inputs that affect configured artifacts. +compute_source_digest() +{ + { + for source_file in CMakeLists.txt CMakePresets.json compose.yml .clang-format; do + [ ! -f "$source_directory/$source_file" ] || printf '%s\n' "$source_directory/$source_file" + done + for source_tree in include cmake tests examples benchmarks containers tools; do + [ ! -d "$source_directory/$source_tree" ] || + find "$source_directory/$source_tree" -type f + done + } | LC_ALL=C sort | while IFS= read -r source_file; do + relative_file=${source_file#"$source_directory/"} + printf '%s\0' "$relative_file" + printf '%s\n' "$(sha256sum "$source_file" | cut -d ' ' -f 1)" + done | sha256sum | cut -d ' ' -f 1 +} + +## @brief Returns the runtime CPU features required by the selected fingerprint. +required_cpu_features() +{ + case "$preset" in + *release-contracts) printf '%s\n' sse4_2 ;; + *release-exhaustive|*debug-diagnostics|*debug-asan-ubsan) + printf '%s\n' "sse4_2 avx2 fma bmi1 bmi2" + ;; + *) printf '%s\n' "" ;; + esac +} + +## @brief Validates every required host CPU feature with an exact diagnostic. +validate_cpu_features() +{ + cpuinfo_file=${SIMDLIB_CPUINFO_PATH:-/proc/cpuinfo} + [ -r "$cpuinfo_file" ] || { + echo "Host CPU feature inventory is unavailable: $cpuinfo_file" >&2 + exit 4 + } + flags=" $(sed -n 's/^flags[[:space:]]*: //p' "$cpuinfo_file" | head -n 1) " + for required_flag in $(required_cpu_features); do + case "$flags" in + *" $required_flag "*) ;; + *) echo "Host CPU does not expose required flag: $required_flag" >&2; exit 4 ;; + esac + done +} + +## @brief Reads one exact key from an owned build manifest. +manifest_value() +{ + manifest_file=$1 + manifest_key=$2 + sed -n "s/^${manifest_key}=//p" "$manifest_file" +} + +## @brief Verifies shared toolchain and compiler invariants. +validate_environment() +{ + command -v "$CXX" >/dev/null 2>&1 || { + echo "Configured C++ compiler is unavailable: $CXX" >&2 + exit 3 + } + case "$($CXX -dumpversion)" in + 13.*|14.*|22.*) ;; + *) echo "Unexpected compiler version from $CXX: $($CXX -dumpfullversion -dumpversion)" >&2; exit 3 ;; + esac + test "$(cmake --version | sed -n '1s/.* //p')" = 4.4.0 || { + echo "Container requires exactly CMake 4.4.0" >&2 + exit 3 + } +} + +## @brief Writes shared compiler, image, host, and operation provenance. +write_provenance() +{ + provenance_file="$provenance_directory/environment.txt" + { + echo "compiler_id=${SIMDLIB_COMPILER_ID:-unknown}" + echo "operation=$operation" + echo "build_profile=$build_profile" + echo "preset=$preset" + echo "sanitizer=$sanitizer" + echo "instrumentation=$instrumentation" + echo "codegen_mode=$codegen_mode" + echo "aggregate=$aggregate" + echo "base_image=${SIMDLIB_BASE_IMAGE:-unknown}" + echo "architecture=$(uname -m)" + echo "os_release=$(tr '\n' ' ' &1 | head -n 1)" + echo "catch2_commit=2b60af89e23d28eefc081bc930831ee9d45ea58b" + echo "packages=$(apk info -v 2>/dev/null | sort | tr '\n' ' ')" + echo "cpu_flags=$(sed -n 's/^flags[[:space:]]*: //p' /proc/cpuinfo | head -n 1)" + } | tee "$provenance_file" +} + +## @brief Runs a command into a report while preserving and displaying its failure. +run_reported() +{ + report_file=$1 + shift + if "$@" >"$report_file" 2>&1; then + cat "$report_file" + else + command_status=$? + cat "$report_file" >&2 + return "$command_status" + fi +} + +## @brief Configures the owning main-project tree, applying CI freshness only here. +configure_main_project() +{ + export SIMDLIB_BUILD_DIRECTORY="$build_directory" + cxx_flags=${SIMDLIB_REQUIRED_CXX_FLAGS:-} + linker_flags=${SIMDLIB_REQUIRED_LINKER_FLAGS:-} + set -- --preset "$preset" -S "$source_directory" \ + -DFETCHCONTENT_SOURCE_DIR_CATCH2="$SIMDLIB_CATCH2_SOURCE" \ + -DCMAKE_CXX_FLAGS="$cxx_flags" \ + -DCMAKE_EXE_LINKER_FLAGS="$linker_flags" + for ci_indicator in \ + "${CI:-}" "${GITHUB_ACTIONS:-}" "${GITLAB_CI:-}" "${TF_BUILD:-}" \ + "${BUILDKITE:-}" "${CIRCLECI:-}" "${JENKINS_URL:-}" "${TEAMCITY_VERSION:-}" + do + [ -z "$ci_indicator" ] || { + set -- --fresh "$@" + break + } + done + run_reported "$report_directory/main-configure.log" cmake "$@" +} + +## @brief Resolves the concrete consumer inventory owned by this compiler cell. +resolve_external_consumer_scope() +{ + [ "$consumer_scope" != none ] || { + printf '%s\n' none + return + } + capability_file="$build_directory/external-consumer-targets.txt" + [ -f "$capability_file" ] || { + echo "External-consumer capability inventory is missing: $capability_file" >&2 + exit 6 + } + consumer_targets=$(LC_ALL=C sort -u "$capability_file" | tr '\n' '|') + case "$consumer_targets" in + CoreConsumerSmoke\|) printf '%s\n' core ;; + CoreConsumerSmoke\|RegisterConsumerSmoke\|) + printf '%s\n' core-register + ;; + *) + echo "Unsupported external-consumer capability inventory: $consumer_targets" >&2 + exit 6 + ;; + esac +} + +## @brief Configures and builds the assigned external-consumer tree. +build_external_consumer() +{ + concrete_scope=$1 + cxx_flags=${SIMDLIB_REQUIRED_CXX_FLAGS:-} + linker_flags=${SIMDLIB_REQUIRED_LINKER_FLAGS:-} + register_consumer=OFF + [ "$concrete_scope" != core-register ] || register_consumer=ON + run_reported "$report_directory/consumer-configure.log" cmake \ + -S "$source_directory/tests/consumer" -B "$consumer_directory" -G Ninja \ + -DCMAKE_BUILD_TYPE="$build_profile" \ + -DSIMDLIB_SOURCE_DIR="$source_directory" \ + -DSIMDLIB_BUILD_REGISTER_CONSUMER="$register_consumer" \ + -DCMAKE_CXX_FLAGS="$cxx_flags" \ + -DCMAKE_EXE_LINKER_FLAGS="$linker_flags" + run_reported "$report_directory/consumer-build.log" \ + cmake --build "$consumer_directory" --parallel +} + +## @brief Records the built CTest executables for pre-test staleness checks. +record_test_inventory() +{ + test_directory=$1 + inventory_file=$2 + cmake -DMODE=RECORD \ + -DTEST_DIRECTORY="$test_directory" \ + -DINVENTORY_FILE="$inventory_file" \ + -DCMAKE_CTEST_COMMAND="$(command -v ctest)" \ + -P "$source_directory/cmake/RecordTestInventory.cmake" +} + +## @brief Audits generated target and CTest ownership against the matrix contract. +audit_validation_inventory() +{ + cmake -DMATRIX_FILE="$matrix_contract" \ + -DCELL_ID="$matrix_cell" \ + -DBUILD_DIRECTORY="$build_directory" \ + -DCMAKE_CTEST_COMMAND="$(command -v ctest)" \ + -DRESULT_FILE="$validation_inventory_audit" \ + -P "$source_directory/cmake/AuditValidationInventory.cmake" +} +## @brief Writes the aggregate generated-code record index from CMake-owned indexes. +write_codegen_record_index() +{ + { + for owner_index in \ + "$build_directory/method-flags-codegen/all-records.txt" \ + "$build_directory/register-codegen/sse42/128/all-records.txt" \ + "$build_directory/register-codegen/avx2/128/all-records.txt" \ + "$build_directory/register-codegen/avx2/256/all-records.txt"; do + [ ! -f "$owner_index" ] || cat "$owner_index" + done + } | sed '/^[[:space:]]*$/d' | LC_ALL=C sort -u >"$codegen_record_index" + if [ "$codegen_mode" != OFF ] && [ ! -s "$codegen_record_index" ]; then + echo "No CMake-owned generated-code records were found under $build_directory" >&2 + exit 6 + fi +} + +## @brief Validates a recorded CTest executable inventory before running tests. +validate_test_inventory() +{ + test_directory=$1 + inventory_file=$2 + cmake -DMODE=VALIDATE \ + -DTEST_DIRECTORY="$test_directory" \ + -DINVENTORY_FILE="$inventory_file" \ + -DCMAKE_CTEST_COMMAND="$(command -v ctest)" \ + -P "$source_directory/cmake/RecordTestInventory.cmake" +} + +## @brief Verifies mandatory runtime-test labels and families before execution. +audit_runtime_test_inventory() +{ + register_required=ON + [ "${SIMDLIB_COMPILER_ID:-}" != gcc13 ] || register_required=OFF + cmake -DTEST_DIRECTORY="$build_directory" \ + -DCMAKE_CTEST_COMMAND="$(command -v ctest)" \ + -DAUDIT_FILE="$report_directory/runtime-test-inventory.audit.txt" \ + -DREGISTER_REQUIRED="$register_required" \ + -P "$source_directory/cmake/VerifyRuntimeTestInventory.cmake" +} + +## @brief Records an atomic completed-operation manifest after all assigned builds succeed. +write_completed_manifest() +{ + manifest_file=$1 + manifest_operation=$2 + source_digest=$3 + concrete_consumer_scope=$(resolve_external_consumer_scope) + cache_hash=$(sha256sum "$build_directory/CMakeCache.txt" | cut -d ' ' -f 1) + source_revision=${SIMDLIB_BUILD_REVISION:-unknown} + if [ "$source_revision" = unknown ]; then + source_revision=$(git -C "$source_directory" rev-parse HEAD 2>/dev/null || printf '%s' unknown) + fi + manifest_aggregate=$aggregate + [ "$manifest_operation" != build-benchmarks ] || manifest_aggregate=BenchmarkArtifacts + target_inventory_hash=none + main_inventory_hash=none + consumer_inventory_hash=none + codegen_record_index_hash=none + matrix_contract_hash=$(sha256sum "$matrix_contract" | cut -d ' ' -f 1) + validation_inventory_audit_hash=none + main_ctest_metadata_hash=none + consumer_ctest_metadata_hash=none + [ ! -f "$target_inventory" ] || + target_inventory_hash=$(sha256sum "$target_inventory" | cut -d ' ' -f 1) + [ ! -f "$main_inventory" ] || + main_inventory_hash=$(sha256sum "$main_inventory" | cut -d ' ' -f 1) + [ ! -f "$consumer_inventory" ] || + consumer_inventory_hash=$(sha256sum "$consumer_inventory" | cut -d ' ' -f 1) + [ ! -f "$codegen_record_index" ] || + codegen_record_index_hash=$(sha256sum "$codegen_record_index" | cut -d ' ' -f 1) + [ ! -f "$validation_inventory_audit" ] || + validation_inventory_audit_hash=$(sha256sum "$validation_inventory_audit" | cut -d ' ' -f 1) + [ ! -f "$build_directory/CTestTestfile.cmake" ] || + main_ctest_metadata_hash=$(sha256sum "$build_directory/CTestTestfile.cmake" | cut -d ' ' -f 1) + [ ! -f "$consumer_directory/CTestTestfile.cmake" ] || + consumer_ctest_metadata_hash=$(sha256sum "$consumer_directory/CTestTestfile.cmake" | cut -d ' ' -f 1) + temporary_manifest="${manifest_file}.tmp" + rm -f "$manifest_file" "$temporary_manifest" + { + echo "schema=simdlib.build-manifest.v1" + echo "operation=$manifest_operation" + echo "status=complete" + echo "source_revision=$source_revision" + echo "source_digest=$source_digest" + echo "fingerprint_sha256=$fingerprint_sha256" + echo "fingerprint_document=$fingerprint_document" + echo "compiler_id=${SIMDLIB_COMPILER_ID:-unknown}" + echo "compiler=$($CXX --version | head -n 1)" + echo "base_image=${SIMDLIB_BASE_IMAGE:-unknown}" + echo "preset=$preset" + echo "build_profile=$build_profile" + echo "sanitizer=$sanitizer" + echo "instrumentation=$instrumentation" + echo "codegen_mode=$codegen_mode" + echo "aggregate=$manifest_aggregate" + echo "matrix_cell=$matrix_cell" + echo "consumer_owner=$consumer_scope" + echo "target_inventory=$target_inventory" + echo "target_inventory_sha256=$target_inventory_hash" + echo "matrix_contract_sha256=$matrix_contract_hash" + echo "validation_inventory_audit=$validation_inventory_audit" + echo "validation_inventory_audit_sha256=$validation_inventory_audit_hash" + echo "consumer_scope=$concrete_consumer_scope" + echo "build_directory=$build_directory" + echo "consumer_directory=$consumer_directory" + echo "cmake_cache_sha256=$cache_hash" + echo "required_cpu_features=$(required_cpu_features | tr ' ' ',')" + echo "main_test_inventory=$main_inventory" + echo "main_test_inventory_sha256=$main_inventory_hash" + echo "main_ctest_metadata_sha256=$main_ctest_metadata_hash" + echo "consumer_test_inventory=$consumer_inventory" + echo "consumer_test_inventory_sha256=$consumer_inventory_hash" + echo "consumer_ctest_metadata_sha256=$consumer_ctest_metadata_hash" + echo "codegen_record_index=$codegen_record_index" + echo "codegen_record_index_sha256=$codegen_record_index_hash" + } >"$temporary_manifest" + mv "$temporary_manifest" "$manifest_file" +} + +## @brief Validates the owning completed build and all pre-test artifacts. +validate_validation_manifest() +{ + [ -f "$validation_manifest" ] || { + echo "Required validation build manifest is missing: $validation_manifest" >&2 + exit 6 + } + [ "$(manifest_value "$validation_manifest" schema)" = simdlib.build-manifest.v1 ] && + [ "$(manifest_value "$validation_manifest" operation)" = build-validation ] && + [ "$(manifest_value "$validation_manifest" status)" = complete ] || + { + echo "Validation build manifest is incomplete or incompatible: $validation_manifest" >&2 + exit 6 + } + [ "$(manifest_value "$validation_manifest" preset)" = "$preset" ] && + [ "$(manifest_value "$validation_manifest" fingerprint_sha256)" = "$fingerprint_sha256" ] && + [ "$(manifest_value "$validation_manifest" fingerprint_document)" = "$fingerprint_document" ] && + [ "$(manifest_value "$validation_manifest" build_profile)" = "$build_profile" ] && + [ "$(manifest_value "$validation_manifest" sanitizer)" = "$sanitizer" ] && + [ "$(manifest_value "$validation_manifest" codegen_mode)" = "$codegen_mode" ] && + [ "$(manifest_value "$validation_manifest" aggregate)" = "$aggregate" ] && + [ "$(manifest_value "$validation_manifest" matrix_cell)" = "$matrix_cell" ] && + [ "$(manifest_value "$validation_manifest" consumer_owner)" = "$consumer_scope" ] && + [ "$(manifest_value "$validation_manifest" compiler_id)" = "${SIMDLIB_COMPILER_ID:-unknown}" ] && + [ "$(manifest_value "$validation_manifest" base_image)" = "${SIMDLIB_BASE_IMAGE:-unknown}" ] || + { + echo "Validation build manifest does not match the requested fingerprint: $validation_manifest" >&2 + exit 6 + } + [ -f "$build_directory/CMakeCache.txt" ] || { + echo "Required CMake cache is missing: $build_directory/CMakeCache.txt" >&2 + exit 6 + } + current_source_digest=$(compute_source_digest) + [ "$(manifest_value "$validation_manifest" source_digest)" = "$current_source_digest" ] || { + echo "Validation build manifest is stale for the current source inputs: $validation_manifest" >&2 + exit 6 + } + current_cache_hash=$(sha256sum "$build_directory/CMakeCache.txt" | cut -d ' ' -f 1) + [ "$(manifest_value "$validation_manifest" cmake_cache_sha256)" = "$current_cache_hash" ] || { + echo "Validation build manifest is stale for the current CMake cache: $validation_manifest" >&2 + exit 6 + } + concrete_consumer_scope=$(resolve_external_consumer_scope) + [ "$(manifest_value "$validation_manifest" consumer_scope)" = "$concrete_consumer_scope" ] || { + echo "Validation consumer scope does not match compiler capabilities" >&2 + exit 6 + } + [ "$(manifest_value "$validation_manifest" target_inventory_sha256)" = \ + "$(sha256sum "$target_inventory" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$validation_manifest" matrix_contract_sha256)" = \ + "$(sha256sum "$matrix_contract" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$validation_manifest" validation_inventory_audit_sha256)" = \ + "$(sha256sum "$validation_inventory_audit" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$validation_manifest" main_test_inventory_sha256)" = \ + "$(sha256sum "$main_inventory" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$validation_manifest" consumer_test_inventory_sha256)" = \ + "$(sha256sum "$consumer_inventory" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$validation_manifest" codegen_record_index_sha256)" = \ + "$(sha256sum "$codegen_record_index" | cut -d ' ' -f 1)" ] || + { + echo "Validation artifact indexes are missing or stale: $validation_manifest" >&2 + exit 6 + } + [ "$(manifest_value "$validation_manifest" main_ctest_metadata_sha256)" = \ + "$(sha256sum "$build_directory/CTestTestfile.cmake" | cut -d ' ' -f 1)" ] || { + echo "Generated main CTest metadata is missing or stale: $validation_manifest" >&2 + exit 6 + } + validate_test_inventory "$build_directory" "$main_inventory" + if [ "$concrete_consumer_scope" = none ]; then + [ "$(manifest_value "$validation_manifest" consumer_ctest_metadata_sha256)" = none ] && + [ ! -f "$consumer_directory/CTestTestfile.cmake" ] && + [ ! -s "$consumer_inventory" ] || { + echo "Consumer-free cell contains external-consumer artifacts" >&2 + exit 6 + } + else + [ "$(manifest_value "$validation_manifest" consumer_ctest_metadata_sha256)" = \ + "$(sha256sum "$consumer_directory/CTestTestfile.cmake" | cut -d ' ' -f 1)" ] || { + echo "Generated consumer CTest metadata is missing or stale" >&2 + exit 6 + } + validate_test_inventory "$consumer_directory" "$consumer_inventory" + fi + cmake -DRECORD_INDEX="$codegen_record_index" \ + -P "$source_directory/cmake/ValidateCodegenRecords.cmake" +} + +## @brief Validates the completed benchmark build without rebuilding it. +validate_benchmark_manifest() +{ + [ -f "$benchmark_manifest" ] || { + echo "Required benchmark build manifest is missing: $benchmark_manifest" >&2 + exit 6 + } + [ "$(manifest_value "$benchmark_manifest" operation)" = build-benchmarks ] && + [ "$(manifest_value "$benchmark_manifest" status)" = complete ] && + [ "$(manifest_value "$benchmark_manifest" preset)" = "$preset" ] && + [ "$(manifest_value "$benchmark_manifest" fingerprint_sha256)" = "$fingerprint_sha256" ] && + [ "$(manifest_value "$benchmark_manifest" fingerprint_document)" = "$fingerprint_document" ] && + [ "$(manifest_value "$benchmark_manifest" build_profile)" = "$build_profile" ] && + [ "$(manifest_value "$benchmark_manifest" sanitizer)" = "$sanitizer" ] && + [ "$(manifest_value "$benchmark_manifest" aggregate)" = BenchmarkArtifacts ] && + [ "$(manifest_value "$benchmark_manifest" matrix_cell)" = "$matrix_cell" ] && + [ "$(manifest_value "$benchmark_manifest" target_inventory_sha256)" = \ + "$(sha256sum "$target_inventory" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$benchmark_manifest" matrix_contract_sha256)" = \ + "$(sha256sum "$matrix_contract" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$benchmark_manifest" validation_inventory_audit_sha256)" = \ + "$(sha256sum "$validation_inventory_audit" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$benchmark_manifest" compiler_id)" = "${SIMDLIB_COMPILER_ID:-unknown}" ] && + [ "$(manifest_value "$benchmark_manifest" base_image)" = "${SIMDLIB_BASE_IMAGE:-unknown}" ] || + { + echo "Benchmark build manifest is incomplete: $benchmark_manifest" >&2 + exit 6 + } + [ "$(manifest_value "$benchmark_manifest" source_digest)" = "$(compute_source_digest)" ] || { + echo "Benchmark build manifest is stale for the current source inputs: $benchmark_manifest" >&2 + exit 6 + } + [ "$(manifest_value "$benchmark_manifest" cmake_cache_sha256)" = \ + "$(sha256sum "$build_directory/CMakeCache.txt" | cut -d ' ' -f 1)" ] || { + echo "Benchmark build manifest is stale for the current CMake cache: $benchmark_manifest" >&2 + exit 6 + } + [ -x "$build_directory/Benchmarks" ] || { + echo "Required benchmark executable is missing: $build_directory/Benchmarks" >&2 + exit 6 + } +} + +## @brief Reports whether the existing owning tree matches the completed validation build. +can_reuse_validation_configuration() +{ + [ -f "$validation_manifest" ] && + [ -f "$build_directory/CMakeCache.txt" ] && + [ "$(manifest_value "$validation_manifest" schema)" = simdlib.build-manifest.v1 ] && + [ "$(manifest_value "$validation_manifest" operation)" = build-validation ] && + [ "$(manifest_value "$validation_manifest" status)" = complete ] && + [ "$(manifest_value "$validation_manifest" fingerprint_sha256)" = "$fingerprint_sha256" ] && + [ "$(manifest_value "$validation_manifest" fingerprint_document)" = "$fingerprint_document" ] && + [ "$(manifest_value "$validation_manifest" preset)" = "$preset" ] && + [ "$(manifest_value "$validation_manifest" build_profile)" = "$build_profile" ] && + [ "$(manifest_value "$validation_manifest" sanitizer)" = "$sanitizer" ] && + [ "$(manifest_value "$validation_manifest" codegen_mode)" = "$codegen_mode" ] && + [ "$(manifest_value "$validation_manifest" aggregate)" = "$aggregate" ] && + [ "$(manifest_value "$validation_manifest" matrix_cell)" = "$matrix_cell" ] && + [ "$(manifest_value "$validation_manifest" consumer_owner)" = "$consumer_scope" ] && + [ "$(manifest_value "$validation_manifest" compiler_id)" = "${SIMDLIB_COMPILER_ID:-unknown}" ] && + [ "$(manifest_value "$validation_manifest" base_image)" = "${SIMDLIB_BASE_IMAGE:-unknown}" ] && + [ "$(manifest_value "$validation_manifest" source_digest)" = "$(compute_source_digest)" ] && + [ "$(manifest_value "$validation_manifest" cmake_cache_sha256)" = \ + "$(sha256sum "$build_directory/CMakeCache.txt" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$validation_manifest" matrix_contract_sha256)" = \ + "$(sha256sum "$matrix_contract" | cut -d ' ' -f 1)" ] && + [ "$(manifest_value "$validation_manifest" validation_inventory_audit_sha256)" = \ + "$(sha256sum "$validation_inventory_audit" | cut -d ' ' -f 1)" ] +} + +validate_environment +write_provenance +[ "$operation" != inspect-environment ] || exit 0 + +case "$operation" in + build-validation) + rm -f "$validation_manifest" + source_digest=$(compute_source_digest) + configure_main_project + run_reported "$report_directory/main-build.log" \ + cmake --build "$build_directory" --parallel --target "$aggregate" + concrete_consumer_scope=$(resolve_external_consumer_scope) + if [ "$concrete_consumer_scope" != none ]; then + build_external_consumer "$concrete_consumer_scope" + elif [ -e "$consumer_directory" ]; then + echo "Consumer-free cell contains an external-consumer tree" >&2 + exit 6 + fi + record_test_inventory "$build_directory" "$main_inventory" + if [ "$concrete_consumer_scope" = none ]; then + : >"$consumer_inventory" + else + record_test_inventory "$consumer_directory" "$consumer_inventory" + fi + write_codegen_record_index + audit_validation_inventory + write_completed_manifest "$validation_manifest" build-validation "$source_digest" + ;; + record-codegen) + source_digest=$(compute_source_digest) + configure_main_project + compilation_started=$(date +%s) + run_reported "$report_directory/codegen-compilation.log" \ + cmake --build "$build_directory" --parallel --target RegisterCodegenFixtureObjects + compilation_finished=$(date +%s) + comparison_started=$(date +%s) + run_reported "$report_directory/codegen-comparison.log" \ + cmake --build "$build_directory" --parallel --target SimdLibDebugDiagnosticArtifacts + comparison_finished=$(date +%s) + compilation_seconds=$((compilation_finished - compilation_started)) + comparison_seconds=$((comparison_finished - comparison_started)) + write_codegen_record_index + cmake -DRECORD_INDEX="$codegen_record_index" \ + -DEXPECTED_POLICY_MODE=RECORD \ + -DEXPECTED_CONFIGURATION=Debug \ + -DREQUIRE_RECORDS=ON \ + -P "$source_directory/cmake/ValidateCodegenRecords.cmake" + cmake -DBINARY_DIRECTORY="$build_directory" \ + -DOWNERSHIP_FILE="$build_directory/development-target-ownership.tsv" \ + -DPROFILE=CODEGEN_DIAGNOSTIC -DCODEGEN_MODE=RECORD \ + -P "$source_directory/cmake/VerifyCodegenProfileIsolation.cmake" + audit_validation_inventory + cmake -DRECORD_INDEX="$codegen_record_index" \ + -DOUTPUT_FILE="$codegen_diagnostic_provenance" \ + -DCOMPILE_COMMANDS="$build_directory/compile_commands.json" \ + -DSOURCE_REVISION="${SIMDLIB_BUILD_REVISION:-unknown}" \ + -DSOURCE_DIGEST="$source_digest" \ + -DFINGERPRINT="$fingerprint_sha256" \ + -DCOMPILER_ID="${SIMDLIB_COMPILER_ID:-unknown}" \ + -DPRESET="$preset" -DCONFIGURATION="$build_profile" \ + -DSANITIZER="$sanitizer" \ + -DCOMPILATION_SECONDS="$compilation_seconds" \ + -DCOMPARISON_SECONDS="$comparison_seconds" \ + -P "$source_directory/cmake/SummarizeCodegenDiagnostic.cmake" + printf 'Codegen diagnostic provenance: %s\n' "$codegen_diagnostic_provenance" + ;; + build-benchmarks) + rm -f "$benchmark_manifest" + source_digest=$(compute_source_digest) + if ! can_reuse_validation_configuration; then + echo "Benchmark build requires a current validated Release configuration: $validation_manifest" >&2 + exit 6 + fi + printf 'Reusing validated Release configuration: %s\n' "$build_directory" | + tee "$report_directory/benchmark-configure.log" + run_reported "$report_directory/benchmark-build.log" \ + cmake --build "$build_directory" --parallel --target BenchmarkArtifacts + write_completed_manifest "$benchmark_manifest" build-benchmarks "$source_digest" + ;; + test-compiler-contracts) + validate_validation_manifest + set -- --test-dir "$build_directory" --output-on-failure \ + --output-junit "$report_directory/compiler-contract-tests.xml" + [ -z "$test_regex" ] || set -- "$@" --tests-regex "$test_regex" + [ -z "$test_label" ] || set -- "$@" --label-regex "$test_label" + ctest "$@" + ;; + test) + validate_validation_manifest + validate_cpu_features + audit_runtime_test_inventory + set -- --test-dir "$build_directory" --output-on-failure \ + --output-junit "$report_directory/main-test.xml" + [ -z "$test_regex" ] || set -- "$@" --tests-regex "$test_regex" + [ -z "$test_label" ] || set -- "$@" --label-regex "$test_label" + ctest "$@" + if [ "$(manifest_value "$validation_manifest" consumer_scope)" != none ]; then + ctest --test-dir "$consumer_directory" --output-on-failure \ + --output-junit "$report_directory/consumer-test.xml" + fi + ;; + run-benchmarks) + validate_benchmark_manifest + validate_cpu_features + run_reported "$report_directory/benchmark-execution.txt" \ + "$build_directory/Benchmarks" '[simdlib][benchmark]' --benchmark-samples 25 + ;; +esac diff --git a/docs/ApiOperationMatrix.md b/docs/ApiOperationMatrix.md index 6512eab..59ee634 100644 --- a/docs/ApiOperationMatrix.md +++ b/docs/ApiOperationMatrix.md @@ -1,27 +1,38 @@ # Api Operation and Type Matrix -This matrix records the public `SimdLib::Api` contract. Unless a cell says -otherwise, **tested** means a runtime public-API test exists at both 128 and -256 bits. **Unavailable** means the operation is intentionally constrained away -for that lane family. **Compile-time-only** identifies a contract proved only by -a compile-time probe. **Clarification needed** identifies a supported-looking -cell that cannot be classified until its intended behavior is decided. +This matrix records the public `SimdLib::Api` contract. A checkmark (**✓**) means +a runtime public-API test exists at both 128 and 256 bits unless the cell names a +specific width. An X (**✗**) means the operation is intentionally constrained +away for that lane family. A shared marker identifies types that use the same +generic overload as the separately tested cell rather than a type-specific +implementation. + +`Api` remains the controlling backend-availability record and the supported +C++20 surface. In a supported C++23 translation unit, the preferred spelling +for an operation on exactly one complete register is `Register` or +`NativeRegister`. Register availability intentionally follows the +corresponding `Api` cell rather than inventing a second implementation policy. | Public operation family | `i8` | `u8` | `i16` | `u16` | `i32` | `u32` | `i64` | `u64` | `float` | `double` | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| Construction, transfer, `set1`, and lane extraction/replacement | tested | tested | tested | tested | tested | tested | tested | tested | tested | tested | -| Addition, subtraction, multiplication, and bitwise operations | tested | tested | tested | tested | tested | tested | tested | tested | tested | tested | -| Logical lane shifts | tested | tested | tested | tested | tested | tested | tested | tested | unavailable | unavailable | -| Arithmetic lane shifts | tested | unavailable | tested | unavailable | tested | unavailable | tested | unavailable | unavailable | unavailable | -| Integer divide, remainder, absolute value, minimum, and maximum | tested | tested | tested | tested | tested | tested | tested | tested | unavailable | unavailable | -| Integer comparisons and `min_position`/`max_position` | tested | tested | tested | tested | tested | tested | tested | tested | unavailable | unavailable | -| Integer conversion | unavailable | unavailable | unavailable | unavailable | tested | tested | unavailable | unavailable | unavailable | unavailable | -| Floating absolute value, comparison helpers, and element extraction | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | tested | tested | -| Floating `set1` and bitwise operations | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | tested | tested | -| `uint64_t::multiply_add_adjacent` | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | tested | unavailable | unavailable | -| Whole-register byte shifts | 128 tested; 256 unavailable | 128 tested; 256 unavailable | 128 tested; 256 unavailable | 128 tested; 256 unavailable | 128 tested; 256 unavailable | 128 tested; 256 unavailable | 128 tested; 256 unavailable | 128 tested; 256 unavailable | unavailable | unavailable | -| `transform_pack` | tested | tested | tested | tested | tested | tested | tested | tested | unavailable | unavailable | -| Span transforms (in-place unary, separate-output unary, and binary) | same generic overload | same generic overload | same generic overload | same generic overload | same generic overload | tested | same generic overload | same generic overload | same generic overload | same generic overload | +| Construction, transfer, `set1`, and lane extraction/replacement | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Addition, subtraction, multiplication, and bitwise operations | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Logical lane shifts | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | +| Arithmetic lane shifts | ✓ | ✗ | ✓ | ✗ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | +| Integer divide, remainder, absolute value, minimum, and maximum | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | +| Integer comparisons and `min_position`/`max_position` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | +| Integer conversion | ✗ | ✗ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | +| Floating absolute value, comparison helpers, and element extraction | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | ✓ | +| Floating `set1` and bitwise operations | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | ✓ | +| Compile-time logical `shuffle` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `uint64_t::multiply_add_adjacent` | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | +| Whole-register byte shifts | 128 ✓ / 256 ✓ | 128 ✓ / 256 ✓ | 128 ✓ / 256 ✓ | 128 ✓ / 256 ✓ | 128 ✓ / 256 ✓ | 128 ✓ / 256 ✓ | 128 ✓ / 256 ✓ | 128 ✓ / 256 ✓ | ✗ | ✗ | +| `transform_pack` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | +| Span transforms (in-place unary, separate-output unary, and binary) | shared¹ | shared¹ | shared¹ | shared¹ | shared¹ | ✓ | shared¹ | shared¹ | shared¹ | shared¹ | + +¹ The span-transform entry point is one generic overload. Its runtime gate uses +`u32`; the other element columns do not represent separate implementations or +separately tested overloads. ## Backend-routing audit @@ -32,10 +43,19 @@ through public `Api`, `SimdVector`, `SimdAlgo`, `SimdResample`, `Bmi`, or `uint128_t` entry points. No direct `Detail` test is retained as a supported test seam. -## Runtime evidence +## Register migration boundary + +| Operation category | Preferred supported surface | +| --- | --- | +| Complete-register construction, exact-width transfer, arithmetic, bitwise operations, shifts, comparisons, masks, selection, reductions, rearrangements, and constrained conversions | `Register` or `NativeRegister` in C++23 | +| Target-selected backend access in C++20 | `NativeApi` | +| Explicit-width backend access, specialized low-level operations, and compatibility call sites | `Api` | +| Span transforms, packed transforms, collection tails, and partial-register staging | `Api`, `SimdAlgo`, or the owning higher-level algorithm | +| Partial lane lists, partial or dynamic-extent transfers, native-order construction, generic implementation-specific shuffles, and runtime extraction | Intentionally absent from `Register`; retain the existing owning abstraction where available | -The matrix is exercised by `tests/Api128.tests.cpp`, -`tests/Api256.tests.cpp`, and the public contract helpers in -`tests/TestSupport.h`. The focused MSVC Release and Clang coverage runs each -contain 21 SSE4.2 tests and 19 AVX2 tests; all 40 pass in both configurations. -The complete MSVC Release suite passes all 187 tests. +The complete one-register mapping is audited by +`tests/RegisterOperationMatrix.tests.cpp`. Behavioral correctness remains +independently checked against scalar references so agreement between +`Register` and `Api` cannot hide a shared defect. Per-run results and exact +compiler counts belong in generated build reports and CI artifacts, not in this +enduring availability matrix. diff --git a/docs/BmiContractMatrix.md b/docs/BmiContractMatrix.md index 3e07380..e0fbe6f 100644 --- a/docs/BmiContractMatrix.md +++ b/docs/BmiContractMatrix.md @@ -1,6 +1,6 @@ # BMI public contract matrix -Phase 1 classifies every symbol in `SimdLib::Bmi` that is outside its nested +This matrix classifies every symbol in `SimdLib::Bmi` that is outside its nested `Detail` namespace. All tests use the public `Bmi` entry points; `Detail` contains implementation alternatives and is not a supported test seam. @@ -24,15 +24,3 @@ Signed `int32_t`/`int64_t` object-representation checks protect the signed contracts. The portable, BMI1-only, BMI2-only, and combined profiles must produce the same deterministic digest; their CTest equivalence tests are the configuration proof. - -## Phase 1 validation record - -On 2026-07-18, the `clang-coverage` build ran 125 CTest entries successfully. -The BMI subset ran 47 entries: eleven public-contract tests in each of the -portable, BMI1-only, BMI2-only, and combined configurations, followed by the -three enabled-versus-portable deterministic-digest equivalence tests. All 47 -passed. The exhaustive 8-bit contracts exposed and fixed narrow-integer -promotion defects in the AND-NOT and unset/trailing-mask helper families. The -deterministic seeds remain `0xC001D00D12345678`, -`0x9E3779B97F4A7C15`, `0xD1B54A32D192ED03`, and -`0xA0761D6478BD642F`. diff --git a/docs/BuildPipeline.md b/docs/BuildPipeline.md new file mode 100644 index 0000000..3be8773 --- /dev/null +++ b/docs/BuildPipeline.md @@ -0,0 +1,347 @@ +# Unified build and validation + +SimdLib has one repository-owned build command and one correctness-validation +command. A complete local build is: + +```powershell +tools/Build.ps1 -Scope All +``` + +This builds MSVC Release and the representative MSVC Debug cell, clang-cl +Release, native Clang Debug coverage, GCC 13 core-only Release, GCC 14 Release, +and Clang 22 Release plus the representative ASan+UBSan Debug cell. It builds +the correctness, ABI, sanitizer, consumer, coverage, probe, example, and +header-validation artifacts, plus the mandatory optimized generated-code gates +in Release. The default matrix does not build ordinary clang-cl, GCC 13, +GCC 14, or Clang 22 Debug cells. Debug, sanitizer, and coverage cells do not +compile Register generated-code fixtures. The command does not compile +benchmark targets or run any executable. + +Before starting compiler cells, `Build.ps1` performs two focused validations. +`tools/Validate-PipelineTooling.ps1` validates matrix topology, pipeline +regressions, configured ownership rules, and no-rebuild behavior once for the +reviewed tooling/configuration digest. It writes +`out/pipeline/provenance/pipeline-validation-.json`, and the unified +receipt binds the result path, hash, status, schema, and tooling digest. +Ordinary production-source changes do not rerun these synthetic tooling tests. +`tools/Test-PublicConsumerBoundary.ps1` separately checks every public example +and consumer fixture before compiler-cell execution and rejects use of +`SimdLib::Detail`; it is a source-boundary check, not a general repository +audit. + +The corresponding complete validation command is: + +```powershell +tools/Run-Tests.ps1 -Scope All +``` + +`Run-Tests.ps1` requires the matching receipt from a prior `Build.ps1` +invocation, validates its exact manifest and source ownership, and then starts +test-only operations. It rejects missing, stale, incomplete, or mismatched +evidence without configuring or building. The coverage cell +resets profiles, runs its instrumented tests, and generates `coverage.info` +plus `coverage-provenance.tsv`. The provenance file records the executable +identity and profile count used for every independently merged coverage target. +Benchmark compilation and execution remain separate: + +```powershell +tools/Build-Benchmarks.ps1 -Scope All +tools/Run-Benchmarks.ps1 -Scope All +``` + +## Prerequisites and explicit scope + +The complete `All` scope requires a Windows x64 host with: + +- Visual Studio 2022 and the MSVC x64 C++ tools; +- LLVM 20 or newer with `clang-cl`, `clang++`, `llvm-profdata`, `llvm-cov`, and + `llvm-readobj` available on `PATH`; +- CMake 3.31 or newer; and +- Docker Desktop with a running Linux-container daemon. + +`Build.ps1` deliberately has no implicit scope. Calling it without `-Scope` +fails, because silently falling back to only the current platform would make +an incomplete build look complete. Hosts that own only Linux container +validation use: + +```powershell +tools/Build.ps1 -Scope Containers +tools/Run-Tests.ps1 -Scope Containers +``` + +Focused development and CI ownership use compiler filters: + +```powershell +tools/Build.ps1 -Scope Native -Compiler Msvc +tools/Run-Tests.ps1 -Scope Native -Compiler ClangCl +tools/Run-Tests.ps1 -Scope Containers -Compiler Gcc14,Clang22 +``` + +Native filters are `Msvc`, `ClangCl`, and `ClangCoverage`. Container filters +are `Gcc13`, `Gcc14`, and `Clang22`. A filter from the wrong scope is an error. +Compiler filters retain the default ownership policy: for example, selecting +`ClangCl` builds clang-cl Release, while selecting `Clang22` builds Clang 22 +Release and ASan+UBSan Debug. + +Retired ordinary Debug cells remain directly available for troubleshooting but +do not produce manifests accepted by the unified default receipt: + +```powershell +tools/Run-NativeMatrix.ps1 -Action Build -Compiler ClangCl -Cell Debug +tools/Run-ContainerMatrix.ps1 -Action Build -Compiler Gcc13 -Cell Debug +tools/Run-ContainerMatrix.ps1 -Action Build -Compiler Gcc14 -Cell Debug +tools/Run-ContainerMatrix.ps1 -Action Build -Compiler Clang22 -Cell Debug +``` + +## Artifact reuse and manifests + +Every compiler/configuration cell has an independent directory: + +```text +out/pipeline/-/-/ + build/ + consumer/ + reports/ + provenance/ +``` + +The readable prefix is followed by the first 16 hexadecimal characters of a +SHA-256 over the canonical compilation fingerprint. The complete fingerprint +is retained in `fingerprint.json`; a short-name collision with different +canonical data is rejected. Compiler identity, generator, configuration, +instrumentation, language policy, dependencies, and required CPU features +participate in the fingerprint. Source inputs do not: their separate digest is +bound into each completed build manifest so editing a source file invalidates +test-only reuse without creating a new toolchain directory. + +## Scoped CMake artifact graph + +### Validation ownership policy + +Every validation artifact has one logical category and the narrowest compiler, +configuration, and instrumentation scope that proves its contract. Pipeline +tooling validation is keyed by its reviewed configuration inputs, while +compiler-front-end and compile-time +contracts belong to applicable Release compiler identities; runtime and +checks/precondition contracts additionally run in the representative MSVC +Debug and Clang ASan+UBSan cells; public examples, smoke, ODR, external +consumer, and optimized generated-code contracts belong to applicable Release +cells. Coverage and sanitizer describe how runtime contracts are compiled and +executed rather than creating duplicate logical owners. + +MSVC Debug is the sole ordinary Debug cell in the default matrix because it +owns the distinct unoptimized Windows and default-check configuration +contract. The clang-cl, GCC 13, GCC 14, and Clang ordinary Debug cells remain +available only for focused troubleshooting: their compiler, language, ABI, +runtime, consumer, and optimizer contracts are already owned by their Release +cells, while the Clang ASan+UBSan cell owns instrumented Linux Debug behavior. + +`tools/validation-matrix.json` is the single machine-readable authority for +cell, operation order, profile, category, test-owner, consumer, +instrumentation, and generated-code policy. Native and container runners +resolve their cells from this file, and CMake reads its profile category +definitions directly. A new +compiler, configuration, instrumentation mode, target, or test may join the +default matrix only when it proves a stated contract that no existing owner +proves. New development targets must declare one scoped category; generated +inventory audits reject missing ownership, duplicate ownership, and profile +membership outside the matrix contract. + +Every top-level development target declares exactly one validation category +when it is created. Configuration fails if a project-owned target is unowned, +is assigned more than once, or belongs to a category forbidden by the selected +`SIMDLIB_VALIDATION_PROFILE`. The supported profiles are `RELEASE`, `DEBUG`, +`SANITIZER`, `COVERAGE`, `COMPILER_CONTRACTS`, `CODEGEN_DIAGNOSTIC`, and +`CUSTOM` for explicitly configured local development trees. + +Compiler-tree category targets are exposed through globally unique aggregates: + +- `SimdLibCompilerContractArtifacts`; +- `SimdLibConstexprContractArtifacts`; +- `SimdLibRuntimeValidationArtifacts`; +- `SimdLibChecksValidationArtifacts`; +- `SimdLibSmokeValidationArtifacts`; +- `SimdLibOptimizedCodegenArtifacts`; +- `SimdLibDebugDiagnosticArtifacts`; and +- `SimdLibCoverageSupportArtifacts`. + +`ExhaustiveArtifacts` is the profile umbrella used by the pipeline. It depends +only on the category aggregates selected by its configured profile. Sanitizer +and coverage trees use `SimdLibSanitizerValidationArtifacts` and +`SimdLibCoverageValidationArtifacts`, respectively, so inherited development +options cannot pull compiler probes, constexpr probes, or generated-code work +into those builds. `BenchmarkArtifacts` remains a separate Release-only +aggregate and is never a dependency of `ExhaustiveArtifacts`. + +External consumers remain separate CMake projects because a main-tree marker +target could not truthfully represent their configure and build operations. +Their applicable targets are recorded in `external-consumer-targets.txt` for +the pipeline orchestrator. +Each supported compiler's Release cell configures, builds, and tests that +project once. Ordinary Debug, sanitizer, coverage, and diagnostic cells record +`consumer_scope=none` and contain no consumer tree. The build manifest binds +the owning scope and consumer test-artifact inventory, so `Run-Tests` cannot +substitute a consumer-free cell for Release evidence. + +`ApiExamples` is the executable C++20 public-API usage contract, and +`RegisterExamples` is its C++23 Register counterpart. `HeaderOnlySmoke` proves +multi-translation-unit umbrella-header linkage, `FormatOdr` proves formatter +specializations link across translation units, and `RegisterOdr` proves the +same multi-translation-unit contract for Register and RegisterMask. Applicable +Release cells own these compiler-facing public-surface contracts; GCC 13 owns +only the core variants because its supported surface is core-only. + +Each configured tree writes deterministic audit inputs: + +- `development-targets.txt` lists configured project targets and aggregates; +- `development-profile-targets.txt` lists targets selected by the profile; +- `development-target-ownership.tsv` maps every development target to its + category, owning aggregate, and selection state; and +- `development-aggregate-membership.tsv` records exact aggregate dependency + membership. + +Compiler-front-end contracts are Release-owned for each compiler and supported +language/feature profile. Ordinary Debug, sanitizer, and coverage trees do not +configure header, availability, language-failure, representation, constexpr, +or method-flags contract families. `ConfigDefaultChecksReleaseProbe` verifies +the Release default. The retained MSVC Debug and Clang sanitizer cells build +`ConfigDefaultChecksDebugProbe`, which also rejects `NDEBUG`; these narrow +targets are the only deliberate default-check configuration probes. +`VectorChecksTests` and `PreconditionTests` explicitly define +`SIMDLIB_ENABLE_CHECKS=1`, while `RegisterPreconditionTests` installs its +failure hook before including the Register API, so their contracts do not +depend on the selected build type. + +`Run-Tests.ps1` always consumes existing artifacts. It succeeds only when the +matching unified-build receipt contains exactly the requested cells, its +source-input digest matches the current tree and every embedded manifest, every +manifest is unchanged, and its pipeline-tooling validation result remains +current and unchanged. Receipt schema v5 binds the pipeline-validation schema, +status, tooling digest, path, and hash alongside each cell's canonical matrix +identity, scoped aggregate, target and test inventory hashes, configured-tree +inventory result, matrix-contract hash, configuration, instrumentation, +generated-code mode, and consumer scope. Test operations contain no +artifact-tree configure or build command. + +The expected default, benchmark, compiler-contract, coverage, sanitizer, and +optional diagnostic cells are defined in `tools/validation-matrix.json`. +Generated target and CTest inventories can be checked directly with: + +```powershell +tools/Audit-ValidationMatrix.ps1 ` + -Cell msvc-release ` + -BuildDirectory out/pipeline/windows-msvc//build ` + -Configuration Release +``` + +The audit rejects duplicate targets or tests, missing ownership, and categories +that are not permitted by the selected profile. Build manifests bind the audit +result, and `Run-Tests.ps1` rejects a receipt whose matrix contract or inventory +audit is stale or belongs to a different cell. + +Focused compiler-front-end diagnosis has explicit lower-level operations that +do not enter the default receipt: + +```powershell +tools/Run-NativeMatrix.ps1 -Action BuildCompilerContracts -Compiler Msvc -Cell Release +tools/Run-NativeMatrix.ps1 -Action TestCompilerContracts -Compiler Msvc -Cell Release +tools/Run-ContainerMatrix.ps1 -Action BuildCompilerContracts -Compiler Clang22 -Cell Release +tools/Run-ContainerMatrix.ps1 -Action TestCompilerContracts -Compiler Clang22 -Cell Release +``` + +Benchmark compilation and execution are intentionally isolated: + +```powershell +tools/Build-Benchmarks.ps1 -Scope All +tools/Run-Benchmarks.ps1 -Scope All +``` + +`Build-Benchmarks.ps1` requires completed validation manifests and builds only +`BenchmarkArtifacts` in their existing exhaustive Release trees. It does not +create a benchmark-specific configure tree or rebuild the validation +aggregates. `Run-Benchmarks.ps1` requires current completed benchmark manifests +and never configures or builds. `Run-Tests.ps1` does not require benchmark +artifacts or manifests. + +## Instrumentation boundaries + +Release, Debug, Clang ASan+UBSan, and native Clang coverage are incompatible +compilation fingerprints and always use separate trees. Debug diagnostics do +not inherit Release optimization enforcement. Sanitizer objects are never +consumed by ordinary Debug tests, and coverage objects are never consumed by a +non-instrumented cell. Benchmark compilation is the sole additional aggregate +that reuses an existing fingerprint, and it reuses only validated Release +trees. + +Compile-only constant-evaluation contracts are owned by each compiler's +exhaustive Release tree instead of being repeated under Debug, sanitizer, or +coverage instrumentation. Native Clang coverage builds only execution-bearing +runtime and checks targets. Examples, header smoke tests, and ODR tests are +public-surface contracts owned by applicable Release compiler identities. +Runtime tests continue to exercise Debug and sanitizer behavior. + +Coverage profiles are matched to executable build identities before merging. +Raw profiles are merged only within one executable identity, so mutually +exclusive feature configurations never share a raw-profile merge. The +per-executable LCOV traces are combined only after LLVM has interpreted each +profile against its owning executable. `coverage-provenance.tsv` records that +mapping and states that compile-only constexpr evidence is excluded. + +Catch2 discovery uses `POST_BUILD` explicitly. Build receipts record the +complete CTest inventory immediately after compilation; `PRE_TEST` would move +discovery into that inventory-recording step rather than remove it from the +receipt-producing workflow. + +Register generated-code diagnostics are explicit supplemental operations: + +```powershell +tools/Record-Codegen.ps1 -Scope Native -Compiler Msvc -Cell Debug +tools/Record-Codegen.ps1 -Scope Containers -Compiler Clang22 -Cell Debug +tools/Record-Codegen.ps1 -Scope Containers -Compiler Clang22 -Cell AsanUbsan +``` + +The command requires one compiler and one cell. It configures a diagnostic-only +tree, builds only the Register fixture objects and record comparisons, and +writes dedicated provenance containing the compiler flags, stack-protector +mode, disassembly tools, source identity, record index, and separate compilation +and comparison timings. These `RECORD` results cannot satisfy an `ENFORCE` +Release gate. Debug diagnostics are run only for a compiler involved in an +active investigation; the sanitizer variant is reserved for investigating how +instrumentation changes wrapper/raw memory, control-flow, or ABI paths. + +Coverage is development infrastructure owned only by a top-level SimdLib +build. The root CMake boundary does not load development modules for +`add_subdirectory` consumers, and the external-consumer contract fails if a +coverage option, instrumented test, or report target leaks downstream. + +The external-consumer project snapshots the parent cache before +`add_subdirectory`, requires the nested target inventory to contain only the +two production interface targets, and rejects nested tests or development +options. It independently verifies the C++20 core target, the C++23 Register +target where supported, and their published usage requirements. Because both +production targets are header-only, Debug CRT and sanitizer propagation do not +create additional consumer contracts. + +## Diagnostic runners and cleanup + +`Run-NativeMatrix.ps1` and `Run-ContainerMatrix.ps1` are lower-level diagnostic +and CI implementation interfaces. Normal repository workflows use `Build.ps1`, +`Run-Tests.ps1`, `Build-Benchmarks.ps1`, and `Run-Benchmarks.ps1`; the +lower-level scripts do not define additional mandatory modes. + +Container images and selected Linux fingerprint roots can be removed with: + +```powershell +tools/Run-ContainerMatrix.ps1 -Action Clean +tools/Run-ContainerMatrix.ps1 -Action Clean -Compiler Clang22 +``` + +All pipeline output is generated below the ignored `out/pipeline` directory. +When no pipeline command is running, removing that directory discards every +native and container fingerprint, report, log, and receipt without touching +source files. A later `Build.ps1` invocation recreates only its selected scope. + +Pre-release option and mode names have no compatibility aliases. Supplying a +retired CMake option is a configuration error with a replacement diagnostic; +the PowerShell commands accept only the canonical action, scope, compiler, and +cell vocabulary documented here. diff --git a/docs/ConstexprCompilerEvidence.md b/docs/ConstexprCompilerEvidence.md deleted file mode 100644 index f353b89..0000000 --- a/docs/ConstexprCompilerEvidence.md +++ /dev/null @@ -1,43 +0,0 @@ -# Constexpr and Compiler-Path Evidence - -## Compile-only matrix - -All targets are ordinary CMake object-library probes. They are dependencies of `SimdLibConstexprProbes`, are built by the default build, and are also exposed through `SimdLib.ConstexprProbes.Build` so assertion diagnostics retain their source file and expression in build or CTest output. - -| Contract source | Compile profiles | Result | -| --- | --- | --- | -| `BmiConstexpr.tests.cpp` | portable, BMI1 only, BMI2 only, BMI1 and BMI2 | MSVC Release and Clang coverage builds pass all four profiles. | -| `UInt128Constexpr.tests.cpp` | compiler carry, portable carry, scalar with SIMD/BMI/FMA disabled | MSVC Release and Clang coverage builds pass all three profiles. | -| `Api128Constexpr.tests.cpp` | SSE4.2 public API and four-lane `SimdVector` | MSVC Release and Clang coverage builds pass. | -| `Api256Constexpr.tests.cpp` | AVX2 public API and eight-lane `SimdVector` | MSVC Release and Clang coverage builds pass. | -| `ApiDisabledConstexpr.tests.cpp` | all instruction families disabled | MSVC Release and Clang coverage builds pass and confirm the SIMD facades are unavailable. | - -The reusable contracts in `tests/constexpr/ApiConstexprContracts.h` cover construction, `setzero`, `setr`, `construct`, `set1`, `load_partial`, `to_array`, `get_element`, `set_element`, all six public comparison helpers, byte and slim movemasks for every signed, unsigned, float, and double lane family, integer extrema positions, lane-shift boundaries, 128-bit whole-register bit/byte-shift boundaries, and `SimdVector` default/array/broadcast construction. Public comparison contracts cover every operation choice reachable through the public helpers; the protected legacy `compare_each_element` dispatcher has no public caller and is not treated as a supported test seam. - -A mechanical comparison with `HEAD` confirms that the first 121 BMI assertions and first six UInt128 assertions in the dedicated sources are text-identical to the removed production-header assertions. Expanded contracts follow those preserved blocks. - -## Runtime/compiler parity - -The 128- and 256-bit runtime parity tests rebuild deterministic inputs through volatile scalars before invoking comparisons, extrema, and lane shifts. This prevents compile-time folding and compares optimized dispatch with the same shared constexpr snapshot. - -`UInt128.tests.cpp` also uses volatile operands for addition and subtraction. The optimized target has a compile-time selection check: - -- MSVC x64 must select `_addcarry_u64` and `_subborrow_u64`; -- Clang/GCC must select `__builtin_add_overflow` and `__builtin_sub_overflow`; -- portable and scalar profiles must disable compiler carry intrinsics. - -The complete MSVC Release suite passes 144/144 tests. The complete Clang coverage suite passes 147/147 tests; Clang has three additional native-`unsigned __int128` tests. Clang coverage cannot contain the preprocessor-excluded MSVC intrinsic lines, so the green MSVC optimized target and its volatile compiler-path test are the evidence for those lines rather than a Clang red-gutter defect. - -The separate `tests/consumer` project configures with MSVC 19.44, builds against `SimdLib::SimdLib`, confirms that the target remains an interface library, and passes its 1/1 CTest entry. The public-header diff contains no declaration, `requires` clause, diagnostic-message, or representation change: it removes test examples, documents retained ABI assertions, and adds constant-evaluation-only bodies. The retained UInt128 size/alignment/layout assertions, strict full builds, volatile runtime parity tests, and external consumer build jointly cover ABI and runtime compatibility. CMake 4.4.0 drove both compiler matrices; Clang validation used Clang 22.1.8. - -## Consumer compile-time and emitted-code comparison - -Measurement date: 2026-07-19. The exact pre-extraction headers came from `HEAD`; post-extraction headers came from the working tree. Both were copied to equal-length sibling paths. Each minimal translation unit included one header and defined the same `extern "C"` anchor. Clang 22.1.8 used `-std=c++20 -O2 -msse4.2 -mavx2`. Runs alternated before/after order after a discarded warm-up. The table reports the median of 15 clean object compiles. - -| Header | Before median | After median | Change | Preprocessed bytes before/after | Preprocessed lines before/after | Object bytes before/after | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| `Bmi.h` | 286.10 ms | 271.48 ms | -5.11% | 2,682,189 / 2,674,248 | 45,980 / 45,860 | 974 / 974 | -| `UInt128.h` | 514.29 ms | 509.06 ms | -1.02% | 4,278,676 / 4,271,309 | 77,446 / 77,343 | 1,194 / 1,194 | -| `SimdLib.h` | 545.92 ms | 527.17 ms | -3.44% | 4,334,803 / 4,327,402 | 78,694 / 78,591 | 1,194 / 1,194 | - -Clang `-ftime-report -fsyntax-only` front-end wall-clock medians also did not regress after stabilization: `Bmi.h` used 21 alternating runs and changed from 0.22 s to 0.21 s; seven alternating runs changed `UInt128.h` from 0.47 s to 0.44 s and `SimdLib.h` from 0.45 s to 0.44 s. The unchanged object sizes confirm that extracting compile-time assertions introduced no emitted code. \ No newline at end of file diff --git a/docs/ContainerValidation.md b/docs/ContainerValidation.md new file mode 100644 index 0000000..e1fdb89 --- /dev/null +++ b/docs/ContainerValidation.md @@ -0,0 +1,184 @@ +# Container validation + +SimdLib uses repository-owned Linux images for GCC 13, GCC 14, and GNU-like +Clang 22 validation. The same Dockerfiles, Compose definition, entrypoint, and +PowerShell runner are used locally and in GitHub Actions. Native jobs remain +authoritative for MSVC, clang-cl, Windows ABI behavior, and vector calling-convention behavior. + +## Environment contract + +| Service | Scope | Base | Compiler | +| --- | --- | --- | --- | +| `gcc13` | Core-only | Alpine 3.20.8, digest pinned | GCC/G++ 13.2.1 | +| `gcc14` | Core and Register | Alpine 3.22.5, digest pinned | GCC/G++ 14.2.0 | +| `clang22` | Core and Register | Alpine 3.24.1, digest pinned | Clang 22.1.3 | + +GCC 13 remains a qualified core-only compiler. Its cells do not claim support +for `SimdLib::Register`. GCC 14 and Clang 22 own the complete core and Register +surface. + +Each image builds the checksum-verified CMake 4.4.0 source release and contains +the exact Catch2 commit declared by its Dockerfile. Package versions, Alpine +images, and the Dockerfile frontend are pinned. The entrypoint rejects an +unexpected compiler or CMake version before configuring the project. +Building these images requires Docker Compose 2.39.0 or newer so the runner can +disable BuildKit provenance without changing the image-identity contract. + +The runtime containers: + +- run without root privileges and with all Linux capabilities dropped; +- use a read-only root filesystem and source mount; +- provide an executable temporary filesystem only at `/tmp`; +- write only below `out/pipeline`; +- use UTC and the C locale; and +- validate CPU features before executing ISA-specific tests or benchmarks. + +## Operations + +The formal cross-platform commands and fingerprint reuse contract are +documented in [Unified build and validation](BuildPipeline.md). Direct use of +the container runner remains available for Linux-cell diagnostics and CI +ownership. + +One build operation creates every Linux validation artifact. One later test +operation consumes those artifacts without configuring or compiling: + +```powershell +tools/Run-ContainerMatrix.ps1 -Action Build +tools/Run-ContainerMatrix.ps1 -Action Test +``` + +Select one compiler or configuration when diagnosing a specific cell: + +```powershell +tools/Run-ContainerMatrix.ps1 -Action Build -Compiler Gcc14 -Cell Release +tools/Run-ContainerMatrix.ps1 -Action Test -Compiler Clang22 -Cell Debug +tools/Run-ContainerMatrix.ps1 -Action Test -Compiler Clang22 -Cell AsanUbsan +``` + +Optional `-TestRegex` and `-TestLabel` filters only narrow a test operation; +they never define a build profile or alter artifact identity. + +There is no mandatory Feature build cell. AVX2, FMA, BMI, and scalar tests are +registered in the exhaustive runtime inventory, audited before execution, and +run once in each owning cell. A label filter is an optional diagnostic view of +that existing inventory, not a second compilation scenario. + +Benchmark compilation and execution are separate operations. Both own only the +existing Release cells, and building benchmarks does not rebuild validation +targets: + +```powershell +tools/Run-ContainerMatrix.ps1 -Action BuildBenchmarks +tools/Run-ContainerMatrix.ps1 -Action RunBenchmarks +``` + +Rebuild images without Docker cache, reuse existing images during a build, or +inspect only the pinned environments without compiling SimdLib: + +```powershell +tools/Run-ContainerMatrix.ps1 -Action InspectEnvironment -NoImageCache +tools/Run-ContainerMatrix.ps1 -Action Build -SkipImageBuild +tools/Run-ContainerMatrix.ps1 -Action InspectEnvironment -SkipImageBuild +``` + +Remove the selected local images and compiler artifact roots together with +abandoned `simdlib-container-*` containers and networks: + +```powershell +tools/Run-ContainerMatrix.ps1 -Action Clean +tools/Run-ContainerMatrix.ps1 -Action Clean -Compiler Clang22 +``` + +`Clean` is intentionally destructive to the selected generated state below +`out/pipeline`; it does not touch source files or artifacts owned by an +unselected compiler. Normal incremental work does not require cleaning. + +## Build cells and artifacts + +| Cell | Services | Configuration | Artifact target | +| --- | --- | --- | --- | +| `Release` | GCC 13, GCC 14, Clang 22 | optimized exhaustive validation | `ExhaustiveArtifacts` | +| `Debug` | GCC 13, GCC 14, Clang 22 | unoptimized runtime validation without Register generated-code work | `ExhaustiveArtifacts` | +| `AsanUbsan` | Clang 22 | instrumented runtime validation without Register generated-code work | `ExhaustiveArtifacts` | + +Record-only generated-code work is selected separately and never joins a +normal build receipt: + +```powershell +tools/Record-Codegen.ps1 -Scope Containers -Compiler Gcc14 -Cell Debug +tools/Record-Codegen.ps1 -Scope Containers -Compiler Clang22 -Cell Debug +tools/Record-Codegen.ps1 -Scope Containers -Compiler Clang22 -Cell AsanUbsan +``` + +The Debug operation is intended for an active compiler investigation, rather +than routine coverage across every compiler. The sanitizer operation has the +narrow purpose of exposing instrumentation-induced wrapper/raw memory, +control-flow, or ABI differences that runtime sanitizer execution cannot show. +It is not a correctness or optimized generated-code gate. + +The runner builds selected images once under the stable +`simdlib-container-images` Compose project, then executes cells with bounded +parallelism controlled by `-MaxParallel`. Each operation has a unique Compose +project and independent standard-output and standard-error logs. Stable image +build ownership prevents an invocation-only Compose label from changing image +identity. A failure in one cell does not hide failures from the remaining +cells. + +Each cell has a canonical JSON fingerprint. The full SHA-256 is stored in the +fingerprint document, while its first 16 hexadecimal characters disambiguate +the readable directory name: + +```text +out/pipeline/linux-/-/ + build/ + consumer/ + reports/ + provenance/ +``` + +Compiler image content identity, pinned base image, toolchain, configuration, +sanitizers, required flags, generator, dependencies, and CPU requirements +participate in the fingerprint. Source revision, source digest, test selection, +CI state, and parallelism do not. Build manifests separately bind a completed +artifact to its source digest and revision, so tests reject stale source inputs. +Image builds disable BuildKit source-context provenance so unrelated project +source changes cannot alter an otherwise identical toolchain image identity. +The content identity covers the filesystem layer chain and runtime image +configuration while excluding Compose's per-invocation project labels. + +Release and benchmark operations share each compiler's Release tree. Debug and +sanitizer configurations have separate fingerprints and trees. + +## Failure and cancellation checks + +The runner retains intentional-failure and cancellation controls for testing +aggregation and cleanup: + +```powershell +tools/Run-ContainerMatrix.ps1 -Action InspectEnvironment -SkipImageBuild -InjectFailure gcc14-release +tools/Run-ContainerMatrix.ps1 -Action InspectEnvironment -SkipImageBuild -InjectFailure All +tools/Run-ContainerMatrix.ps1 -Action Build -SkipImageBuild -CancelAfterSeconds 2 +``` + +These commands return nonzero. Cleanup is scoped to the unique Compose project +created for the invocation, while logs received from completed cells remain +available. + +## Refresh procedure + +Image refreshes are deliberate review changes: + +1. Select the smallest maintained Alpine release that provides the required + compiler and retrieve its immutable multi-platform manifest digest. +2. Update every exact package version, CMake checksum, and Catch2 commit. +3. Run `InspectEnvironment` with `-NoImageCache` and review the identities. +4. Run `tools/Build.ps1 -Scope Containers`, then + `tools/Run-Tests.ps1 -Scope Containers` and + `tools/Build-Benchmarks.ps1 -Scope Containers` followed by + `tools/Run-Benchmarks.ps1 -Scope Containers`. +5. Confirm the native MSVC and clang-cl configurations separately. + +The scheduled reproducibility workflow performs the no-cache environment +rebuild without compiling SimdLib. Pull requests and normal CI use the same +repository-owned definitions and runner. diff --git a/docs/ImmediateControlRuntimeNaming.md b/docs/ImmediateControlRuntimeNaming.md new file mode 100644 index 0000000..07f3450 --- /dev/null +++ b/docs/ImmediateControlRuntimeNaming.md @@ -0,0 +1,30 @@ +# Runtime controls for immediate-mode operations + +Many x86 SIMD instructions encode their control value directly in the instruction. That control must therefore be known while the caller is compiled. SimdLib reserves an unsuffixed operation name for this compile-time form and for genuinely native runtime-control instructions. + +A name ending in `_slow` is a deliberate runtime substitute for an operation whose native counterpart normally requires a compile-time immediate. The substitute preserves the operation's semantics for a runtime scalar control, but it may require dispatch, branching, or a longer synthesized instruction sequence. The suffix describes the control mechanism; it does not mean that every call is necessarily slow after inlining and constant propagation. + +## Operation inventory + +| Operation family | Unsuffixed compile-time or native runtime form | Runtime immediate substitute | Exposed layers | +| --- | --- | --- | --- | +| Lane extraction | `extract(value)`; `Register::lane()` | `extract_slow(value, index)` | `Api`, implementation; `SimdVector` uses the Api slow path internally | +| Lane insertion | `insert(value, lane)`; `Register::with_lane(lane)` | `insert_slow(value, lane, index)` | `Api`, implementation | +| Immediate blend | `blend(lhs, rhs)` | `blend_slow(lhs, rhs, control)` | `Api`, implementation, extension helper | +| Register-mask blend | `blend(lhs, rhs, mask)` | Not applicable; the mask register is a native runtime control | `Api`, implementation | +| Floating shuffle | Immediate or compile-time logical `shuffle` forms | `shuffle_slow(lhs, rhs, control)` | `Api`, implementation, extension helper | +| Byte shuffle | `shuffle(value, selector_register)` | Not applicable; the selector register is a native runtime control | `Api`, implementation | +| Low 16-bit half shuffle | `shuffle_lo(value)` | `shuffle_lo_slow(value, control)` | `Api`, implementation, extension helper | +| High 16-bit half shuffle | `shuffle_hi(value)` | `shuffle_hi_slow(value, control)` | `Api`, implementation, extension helper | +| 32-bit group shuffle | `shuffle_32(value)` | `shuffle_32_slow(value, control)` | `Api` through its implementation mapping, implementation, extension helper | +| Complete-register byte shift | `shift_bytes_left(value)`, `shift_bytes_right(value)` | `shift_bytes_left_slow(value, count)`, `shift_bytes_right_slow(value, count)` | Immediate: `Api`, `Register`, and implementation at 128/256 bits; `_slow`: the same layers at 128 bits | +| Complete-register bit shift | `shift_bits_left(value)`, `shift_bits_right(value)` | `shift_bits_left_slow(value, count)`, `shift_bits_right_slow(value, count)` | `Api`, `Register`, implementation, extension helper at 128 bits | +| Ordinary per-lane shift | `shift_left(value, count)`, `shift_right(value, count)`, and arithmetic variants | Not applicable; the runtime count uses native variable-count instructions | `Api`, `Register`, `SimdVector`, implementation | + +A complete-register byte shift treats the register as one contiguous byte sequence: it crosses element, 64-bit, and—at 256 bits—128-bit-half boundaries. A complete-register bit shift treats the supported 128-bit register as one bit string. Neither is an ordinary per-lane shift: `shift_left`, `shift_right`, and `shift_right_arithmetic` retain their lane-wise semantics and native runtime-count behavior. + +`Register` intentionally exposes compile-time lane access and immediate rearrangement, but it does not add dynamic lane extraction, dynamic lane insertion, or scalar-control blend and shuffle members. `SimdVector` likewise has no public immediate-control emulation surface; its reductions use `Api::extract_slow` internally when a lane is selected at runtime. + +## Choosing a form + +Use the unsuffixed template form whenever the control is part of the algorithm and can be expressed as a template argument. Use an unsuffixed register-control overload when the instruction family natively accepts a selector or mask register. Use `_slow` only when the control is genuinely determined at runtime and the immediate-mode operation's semantics are required. \ No newline at end of file diff --git a/docs/MethodFlagsContract.md b/docs/MethodFlagsContract.md new file mode 100644 index 0000000..2333d9f --- /dev/null +++ b/docs/MethodFlagsContract.md @@ -0,0 +1,507 @@ +# SIMD method-flag contract + +## Scope + +`SIMD_FLAGS(...)` is the public declaration macro for stating the SIMD ABI and +optimization promises of an ordinary function. It is intended for both SimdLib +and downstream code. + +The initial declaration form keeps the return type independent: + +```cpp +Result SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) transform(Input value); +``` + +Every invocation starts with exactly one SIMD boundary mode: `Neither`, `In`, +`Out`, or `InOut`. It is followed by only the modifiers that apply to that +function, in the canonical order `RegisterOnly`, `ForceInline`, then `Flatten`. +The fixed order is part of the grammar rather than a formatting preference. + +The macro records developer intent. The preprocessor can validate the flag +grammar, but it cannot inspect C++ parameter types, return types, function +bodies, template instantiations, or transitive callees. Correct flag selection +therefore remains a source-review responsibility. + +## Boundary-mode semantics + +### `Neither` + +`Neither` promises that no native SIMD value, `Register`, or `RegisterMask` +crosses the function boundary by value as either an input or result. + +- Pointers, references, spans, and arrays do not themselves violate `Neither`. +- An ordinary implicit `this` pointer does not violate `Neither`. +- A scalar input or result does not violate `Neither`. +- For dependent parameter or return types, every supported instantiation + described by the declaration must satisfy the promise. + +`Neither` emits no vector calling convention. It makes no memory-effect or +optimization promise; those properties remain explicit modifiers. + +### `In` + +`In` promises that at least one native SIMD value, `Register`, or +`RegisterMask` enters the function by value. + +- A C++23 explicit-object parameter taken by value counts as an input. +- An ordinary implicit `this` pointer does not count as a by-value SIMD input. +- A pointer, reference, span, array, or scalar does not count as a SIMD input. +- For a dependent parameter type, every supported instantiation described by + the declaration must satisfy the promise. + +`In` requests the configured vector calling convention where one is supported. +It does not promise that every input remains in a physical register after +register allocation. + +### `Out` + +`Out` promises that the function returns a native SIMD value, `Register`, or +`RegisterMask` by value. + +- Scalar, pointer, reference, span, and array returns do not satisfy `Out`. +- For a dependent or deduced return type, every supported instantiation + described by the declaration must satisfy the promise. + +`Out` requests the configured vector calling convention where one is supported. +It does not independently guarantee that a platform ABI will avoid hidden +return storage. + +### `InOut` + +`InOut` promises that the function satisfies both the `In` and `Out` contracts. +It describes one bidirectional SIMD call boundary and emits the configured +vector calling convention exactly once where one is supported. + +`In, Out` is not an alternate spelling. A declaration that satisfies both +directions uses the single `InOut` boundary mode. + +## Modifier semantics + +### `RegisterOnly` + +`RegisterOnly` promises that every runtime-evaluated path is authored as +register/scalar computation and does not intentionally write a value to +addressable memory. + +The promise allows: + +- SIMD and scalar computation; +- extraction from and insertion into SIMD registers; +- returning SIMD or scalar values; +- reading through const pointers, const references, and read-only spans; +- intrinsic loads from input memory; +- non-addressable scalar temporaries; +- calls whose relevant paths independently satisfy the same no-write contract; +- storage used exclusively by an `if consteval` branch that cannot be evaluated + at runtime. + +The promise prohibits: + +- writes through pointers, references, spans, iterators, or array parameters; +- stores to globals, static storage, thread-local storage, or volatile storage; +- runtime local arrays or other explicit addressable local buffers; +- a `memcpy`, `memmove`, memory intrinsic, or library call with a destination; +- SIMD store, scatter, streaming-store, masked-store, or similar intrinsics; +- inline assembly with a memory output, memory clobber, or unreviewed memory + side effect; +- calls that perform a prohibited write on behalf of the function; +- returning an array or another result whose authored contract requires output + storage. + +A read-only volatile access and inline assembly without a memory output require +individual review rather than automatic acceptance. + +Compiler-created spills, stack frames, unwind records, instrumentation, and +hidden ABI storage do not falsify the source-level promise. They also are not +prevented by it. ABI and generated-code tests remain responsible for detecting +those effects. + +On supported Microsoft C++ configurations, `RegisterOnly` maps to +`__declspec(safebuffers)`. That mapping suppresses the +function's `/GS` security-cookie instrumentation and is the reason the promise +must never be applied speculatively. An empty mapping on another compiler does +not weaken the semantic promise. + +### `ForceInline` + +`ForceInline` promises that optimized generated code is intended to inline the +annotated function into its caller. Its compiler mapping includes the C++ +`inline` specifier needed for a header definition. + +The flag is an optimization request, not a claim that every compiler, +configuration, recursion pattern, or invalid program shape can perform the +inlining. A function that only requires the C++ ODR meaning of `inline` uses the +language specifier directly and does not claim `ForceInline`. + +### `Flatten` + +`Flatten` promises that calls made by the annotated function are intended to be +recursively inlined where the compiler provides a flattening attribute. + +`Flatten` does not request that the annotated function itself be inlined into +its caller. A declaration that requires both behaviors specifies both +`ForceInline` and `Flatten`. + +## Grammar + +### Accepted boundary modes, modifiers, and arity + +The initial grammar accepts one boundary mode and zero to three modifiers: + +```text +SIMD_FLAGS(boundary-mode [, modifier ...]) + +boundary-mode: + Neither + In + Out + InOut + +modifier sequence: + [RegisterOnly] [ForceInline] [Flatten] + +modifier: + RegisterOnly + ForceInline + Flatten +``` + +Four is the initial maximum argument count. Modifier omission is allowed, but +the selected modifiers remain an ordered subsequence of `RegisterOnly`, +`ForceInline`, `Flatten`. + +The following rules are mandatory: + +- `SIMD_FLAGS()` is invalid. +- A modifier-only invocation is invalid; use `Neither` as the boundary mode. +- More than four arguments is invalid. +- An unknown or misspelled token is invalid. +- A boundary mode in a modifier position is invalid. +- A modifier in the boundary-mode position is invalid. +- A duplicate modifier is invalid. +- A noncanonical modifier order is invalid. +- No invalid token may be silently ignored. +- No underlying attribute or calling convention may be emitted more than once. + +Invalid input must fail at the declaration. Empty and over-arity invocations +use these stable diagnostic identifiers: + +- `SIMDLIB_FLAGS_ERROR_EMPTY` +- `SIMDLIB_FLAGS_ERROR_TOO_MANY` + +Other invalid tokens or token sequences fail through an unresolved +`SIMDLIB_DETAIL_FLAGS_BOUNDARY_...` or +`SIMDLIB_DETAIL_FLAGS_MODIFIERS_...` mapping. This deliberately avoids a +general-purpose membership parser solely to improve diagnostic spelling. + +No public object-like macros named `Neither`, `In`, `Out`, `InOut`, +`RegisterOnly`, `ForceInline`, or `Flatten` may be defined to implement the +grammar. + +No object-like macro with one of those exact names may be active at a +`SIMD_FLAGS(...)` invocation. Macro arguments are expanded before a variadic +forwarding layer can classify them, so such a collision makes the invocation +invalid. A function-like macro with the same name does not expand when passed +as a bare token and is not a collision. + +### Canonical declaration position + +`SIMD_FLAGS(...)` follows the independently specified return type and immediately +precedes the function name. The macro never selects, replaces, or deduces the +return type. + +The canonical order is: + +1. template head and any leading `requires` clause; +2. standard declaration attributes such as `[[nodiscard]]`; +3. `friend`, `static`, ordinary `inline`, and then `constexpr`, when + applicable; `consteval` declarations are rejected by the initial contract; +4. independently specified return type, including `auto` when selected by the + declaration; +5. `SIMD_FLAGS(...)`; +6. function name and parameter list; +7. member cv/ref qualifiers; +8. exception specification; +9. an independently specified trailing return type, when applicable; +10. trailing `requires` clause. + +The macro emits placement-safe optimization attributes followed by the +configured vector calling convention. This order and position are required +because MSVC accepts `__vectorcall` after the return type and immediately before +the function name, but rejects it before the return type. GNU-style compilers +accept their corresponding function attributes in the same pre-name position. +An ordinary return type, a deduced `auto` return, and `auto` with an explicit +trailing return remain normal C++ syntax outside the macro. + +`ForceInline` already supplies the header-definition `inline` specifier. +Ordinary `inline` is therefore omitted when `ForceInline` is present. +Declarations and out-of-line definitions repeat the same complete flag list. +Every overload is classified independently. + +Compiler qualification must prove this pre-name placement before the +public macro is implemented. A compiler-specific warning suppression is not a +substitute for accepted placement. + +## Canonical declaration forms + +### Free function + +```cpp +[[nodiscard]] constexpr +Result +SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) +transform(Input lhs) noexcept; +``` + +### Static member + +```cpp +[[nodiscard]] static constexpr +Register +SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) +zero() noexcept; +``` + +### Non-static member + +An implicit object does not itself satisfy `In`. + +```cpp +[[nodiscard]] constexpr +Register +SIMD_FLAGS(InOut, RegisterOnly, ForceInline) +combine(Register rhs) const noexcept; +``` + +### Explicit-object member + +A by-value explicit object satisfies `In`. + +```cpp +[[nodiscard]] constexpr +Register +SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) +combine(this Register lhs, Register rhs) noexcept; +``` + +### Operator + +Operators use the same independently specified return-type form. + +```cpp +[[nodiscard]] friend constexpr +Register +SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) +operator+(Register lhs, Register rhs) noexcept; +``` + +An explicit-object operator uses the explicit-object member form rather than +adding `friend`. + +### Function template + +```cpp +template + requires RegisterTarget +[[nodiscard]] static constexpr +Target +SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) +convert(native_type value) noexcept; +``` + +The promises apply to every supported specialization selected by the +constraints. + +### Constrained trailing-return function + +```cpp +template +[[nodiscard]] static constexpr +auto +SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) +convert(native_type value) noexcept -> Target + requires RegisterTarget; +``` + +The declaration supplies both `auto` and the resolved trailing return type. +`SIMD_FLAGS(...)` supplies neither. `Out` describes the resolved return type. + +### Friend function + +A friend definition follows the same flag rules as a namespace function. + +```cpp +[[nodiscard]] friend constexpr +Register +SIMD_FLAGS(InOut, RegisterOnly, ForceInline) +select(RegisterMask mask, Register yes, Register no) noexcept; +``` + +## Unsupported declaration categories + +The initial `SIMD_FLAGS(...)` surface deliberately excludes categories that +lack an ordinary return type before the function name or have incompatible ABI and +optimization rules: + +- constructors and destructors; +- conversion operators; +- deduction guides; +- lambdas; +- explicit function-pointer and pointer-to-member type declarations; +- virtual functions and overriding declarations; +- coroutines; +- C-style variadic functions; +- `extern "C"` declarations; +- allocation and deallocation functions; +- defaulted or deleted functions; +- immediate-only `consteval` functions. + +`constexpr` functions are supported because they can also have runtime-evaluated +paths. `consteval` functions have no runtime call boundary or generated-code +contract and therefore do not use SIMD method flags. + +The address of a supported flagged function may be taken. Code that needs an +explicit callback type derives it with `decltype(&function)` so the compiler's +calling-convention type is preserved instead of placing `SIMD_FLAGS(...)` +inside a pointer declarator. + +These categories are outside the supported contract. `SIMD_FLAGS(...)` cannot +inspect its surrounding declaration, so a compiler may accept some such uses +without a dedicated diagnostic. Compiler acceptance does not make the +declaration a supported extension. + +## Downstream declarations and definitions + +Downstream functions use the same declaration form as SimdLib. Repeat an +ABI-compatible flag list on the declaration and definition: + +```cpp +// Transform.h +/** + * @brief Applies a downstream register transformation. + * @param value Input register. + * @return Transformed register. + */ +SimdLib::Register +SIMD_FLAGS(InOut) +transform(SimdLib::Register value) noexcept; + +// Transform.cpp +SimdLib::Register +SIMD_FLAGS(InOut) +transform(const SimdLib::Register value) noexcept +{ + return value + SimdLib::Register::broadcast(1.0F); +} +``` + +All translation units that declare, define, take the address of, or call the +function must agree on the vectorcall capability and token adapter. A mismatch +is an ABI disagreement; source-level type similarity does not make it safe. +Use `decltype(&transform)` when storing the function pointer so the configured +calling convention remains part of its type where the compiler models it. + +`Out` selects the configured calling convention when one exists. It does not +independently force a value into physical return registers or override a +platform ABI that uses hidden return storage for an aggregate. + +Apply `RegisterOnly` only after reviewing the complete runtime call graph. +Downstream authors must not use it on stores, writable spans, output pointers +or references, addressable local buffers, array-backed algorithms, or +unreviewed transitive calls. Its Microsoft mapping suppresses `/GS` for the +whole function; an incorrect promise removes a security mitigation. + +## Register-only audit procedure + +Every `RegisterOnly` decision is made per function and per reachable runtime +path: + +1. Identify every runtime path, separating unreachable `if consteval` storage + from runtime storage. +2. Inspect parameters and results for writable pointers, references, spans, + arrays, iterators, aggregate return storage, and mutable proxy types. +3. Inspect locals for arrays, address-taking, explicit buffers, destination + objects, and memory-copy destinations. +4. Inspect intrinsics and inline assembly for stores, scatters, memory outputs, + memory clobbers, or undocumented side effects. +5. Inspect every call for transitive writes, including helpers hidden behind + templates, overloads, and constant/runtime dispatch. +6. Confirm that valid runtime behavior consists only of input reads, + register/scalar computation, and register/scalar return. +7. Retain generated-code and ABI review as a separate gate for compiler-created + spills, hidden storage, security cookies, and other effects source review + cannot prove. + +If review contradicts an existing register-only declaration, the declaration +requires explicit investigation rather than mechanical relaxation. A newly +identified candidate likewise requires review before `RegisterOnly` is added. + +## Semantic flags and compiler mappings + +The source contract is stable even when a compiler mapping is empty. The +initial mapping baseline is: + +| Mode or modifier | Microsoft C++ | clang-cl | GNU-like Clang | GCC | +|---|---|---|---|---| +| `Neither` | no boundary token | no boundary token | no boundary token | no boundary token | +| `In`, `Out`, or `InOut` | configured `__vectorcall` on supported Windows x86 targets | configured `__vectorcall` on supported Windows x86 targets | no vector-calling-convention token | no vector-calling-convention token | +| `RegisterOnly` | `__declspec(safebuffers)` after audit | no emitted token | no emitted token | no emitted token | +| `ForceInline` | `__forceinline` | `inline __attribute__((always_inline))` | `inline __attribute__((always_inline))` | `inline __attribute__((always_inline))` | +| `Flatten` | `[[msvc::flatten]]` | `__attribute__((flatten))` | `__attribute__((flatten))` | `__attribute__((flatten))` | + +These are adapter mappings, not definitions of the flags. A new compiler may +map the same promise differently. Changing a compiler mapping requires focused +syntax, ABI, and generated-code evidence; it does not require rewriting +correctly classified function declarations. + +The placement-safe method-flags adapters may use a different spelling from a +legacy low-level adapter with the same semantic effect. In particular, the +C++11-style force-inline attributes are not accepted after every semantic +specifier by MSVC and clang-cl, while the keyword or GNU attribute spellings +above are accepted in the canonical declaration position without warnings. + +### Compiler-adapter configuration + +Each compiler property has a caller-overridable capability and token adapter: + +| Property | Capability macro | Token adapter | +|---|---|---| +| vector calling convention | `SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL` | `SIMDLIB_METHOD_FLAGS_VECTORCALL` | +| safe-buffer suppression | `SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS` | `SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS` | +| forced inlining | `SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE` | `SIMDLIB_METHOD_FLAGS_FORCE_INLINE` | +| recursive flattening | `SIMDLIB_METHOD_FLAGS_HAS_FLATTEN` | `SIMDLIB_METHOD_FLAGS_FLATTEN` | + +A custom toolchain defines the relevant capability and token-adapter pair before +the first inclusion of `SimdLib/Config.h`. It does not redefine `SIMD_FLAGS(...)` +or any `SIMDLIB_DETAIL_...` parsing helper. A zero capability may produce an +empty adapter; `ForceInline` retains ordinary `inline` semantics when compiler +enforcement is unavailable. All translation units that exchange flagged +functions must agree on the ABI-affecting vectorcall configuration. + +## Extension rule + +A future boundary mode or modifier is admitted only after all of the following +are recorded: + +1. one precise source-level promise; +2. valid and invalid usage categories; +3. interaction with every existing boundary mode and modifier; +4. canonical placement; +5. supported and empty compiler mappings; +6. configuration and downstream override behavior; +7. compile-pass coverage and compile-failure coverage wherever the macro or + compiler can diagnose the invalid form reliably; +8. ABI or generated-code evidence when the flag can affect either. + +Adding support for another compiler or changing an adapter follows the same +qualification path: define the semantic mapping, prove the canonical +post-return-type placement, cover default and overridden configuration, verify +cross-translation-unit ABI behavior, and retain generated-code evidence for +every affected optimization or stack-protection property. An empty mapping is +valid only when the semantic flag remains meaningful to source review and the +compiler lacks an applicable attribute. + +Generic `Read` and `Write` modifiers are not part of the initial vocabulary +because they do not distinguish SIMD call direction from memory effects. `In`, +`Out`, and `InOut` describe SIMD values crossing the call boundary; +`RegisterOnly` describes the absence of authored runtime writes. diff --git a/docs/PreconditionInventory.md b/docs/PreconditionInventory.md index 0d8a80e..6bc830a 100644 --- a/docs/PreconditionInventory.md +++ b/docs/PreconditionInventory.md @@ -50,7 +50,7 @@ otherwise. | `SimdResample.h`: `ReduceBytesToBitsBy8_All` | `src.size() == dst.size() * 8`. | Caller-facing extent contract. | `SimdResample reduce all terminates for an invalid shape` uses seven source bytes and one destination byte. | | `SimdResample.h`: `ReduceBytesToBitsBy8_Parity` | `src.size() == dst.size() * 8`. | Caller-facing extent contract. | `SimdResample reduce parity terminates for an invalid shape` uses seven source bytes and one destination byte. | | `SimdResample.h`: `ExpandBitsToBytesBy8` | `dst.size() == src.size() * 8`. | Caller-facing extent contract. | `SimdResample expand terminates for an invalid shape` uses one source byte and seven destination bytes. | -| `SimdVector.h`: partial-result validation | Every inactive lane in an internally produced result is zero. | Internal implementation invariant, evaluated only at runtime for partial vectors when `SIMDLIB_ENABLE_CHECKS` is enabled. It is not a caller-supplied input contract and cannot be intentionally failed through a supported public call without first introducing a library defect. | `SimdLibTestsVectorChecks` observes three successful evaluations for partial divide, modulus, and clamp, and zero evaluations for their full-vector counterparts. | +| `SimdVector.h`: partial-result validation | Every inactive lane in an internally produced result is zero. | Internal implementation invariant, evaluated only at runtime for partial vectors when `SIMDLIB_ENABLE_CHECKS` is enabled. It is not a caller-supplied input contract and cannot be intentionally failed through a supported public call without first introducing a library defect. | `VectorChecksTests` observes the partial divide, modulus, and clamp paths and their full-vector counterparts. | No runtime `SIMDLIB_PRECONDITION` for an index, divisor, or overlap was found. Compile-time width, count, and availability restrictions remain enforced by diff --git a/docs/PublicNamespace.md b/docs/PublicNamespace.md index 6d054cb..b55b7fc 100644 --- a/docs/PublicNamespace.md +++ b/docs/PublicNamespace.md @@ -30,8 +30,11 @@ the rename preserves the complete member API rather than selecting a subset. | Automatically sized SIMD facade | `SimdLib::NativeApi` | | Register-width SIMD facade | `SimdLib::Api` | | Availability query and constraint | `SimdLib::is_api_available_v` and `SimdLib::ApiAvailable` | +| Automatically sized complete-register value | `SimdLib::NativeRegister` | +| Explicit-width complete-register value | `SimdLib::Register` | +| Complete-register predicate value | `SimdLib::RegisterMask` | | Fixed logical SIMD value | `SimdLib::SimdVector` | -| Fixed-width vector aliases | Root `SimdLib::*x*` and `SimdLib::Vector*` aliases | +| Fixed-width complete-register aliases | C++23 root `SimdLib::*x*` and `SimdLib::Vector*` aliases | | Bit manipulation | `SimdLib::Bmi` | | Unsigned wide integer | `SimdLib::uint128_t` | | Byte-mask resampling | `SimdLib::SimdResample` | @@ -43,11 +46,16 @@ in `SimdLib::SimdApi` adds length without distinguishing another public API. The short name also reads clearly in aliases such as `using u32x4_api = SimdLib::Api<128, std::uint32_t>`. -`NativeApi` is the preferred entry point when consumers do not -require a fixed register width. It selects the 256-bit facade when the compile -target enables it and otherwise selects the 128-bit facade. Explicit -`Api` remains the supported form for width-specific -algorithms and ABI contracts. +For C++23 complete-register expressions, `NativeRegister` is the +preferred entry point when consumers do not require a fixed register width. +Explicit `Register` is required when storage layout +or an ABI contract must remain stable across target configurations. + +`NativeApi` remains the preferred backend facade for C++20, +collection helpers, compatibility code, and specialized low-level operations. +It selects the 256-bit facade when the compile target enables it and otherwise +selects the 128-bit facade. Explicit `Api` remains +the supported form for width-specific backend algorithms. `SimdResample` remains unchanged. It names a cohesive, existing operation family and changing it would add churn without improving the requested type diff --git a/docs/RegisterCodegenAudit.md b/docs/RegisterCodegenAudit.md new file mode 100644 index 0000000..0ae3cb3 --- /dev/null +++ b/docs/RegisterCodegenAudit.md @@ -0,0 +1,174 @@ +# Permanent Generated-Code Suite Audit + +This document defines the ownership and retention policy for SimdLib's permanent +generated-code fixtures. The machine-readable, per-symbol decision ledger is +[`RegisterCodegenSymbolAudit.csv`](RegisterCodegenSymbolAudit.csv). + +## Contract categories + +Every retained symbol belongs to exactly one category: + +| Category | Permanent observable contract | +|---|---| +| Public abstraction parity | A public `Register` operation adds no work relative to the matching public `Api` operation. | +| ABI boundary | A non-inlined `Register`, `RegisterMask`, explicit-object mirror, or native-vector signature preserves the documented calling boundary. | +| Compiler-attribute enforcement | `SIMD_FLAGS(...)` and the legacy declaration attributes produce the same ABI and generated code, including inlining and stack restrictions. | +| Instruction-property guarantee | A feature mode or immediate form retains a required instruction property, such as fused multiply-add presence or absence. | +| Composed-expression optimization | Multiple public operations optimize as one expression without wrapper temporaries or repeated work. | +| Register-pressure behavior | Simultaneously live values and opaque calls do not introduce wrapper-specific spills or reloads. | +| Explicitly diagnostic evidence | The artifact records compiler behavior but is excluded from zero-overhead pass/fail claims. | + +The ledger has one row for each source-level fixture symbol. Force-inline and +flatten helper symbols are intentionally absent from optimized objects. Reuse of +a symbol name across width or ISA configurations is represented by its `applicability` +field. Feature-mode comparisons that deliberately compile the same symbol twice, +such as FMA enabled and disabled, identify both records in that row. + +## Retained symbol ownership + +| Owning fixture | Symbols | Category coverage | Distinct purpose | +|---|---:|---|---| +| `RegisterCodegenFixture.h` | 23 | Public parity, composition, instruction property, register pressure | Protects expression and lifetime behavior that an isolated operation cannot represent. | +| `RegisterTypeMatrixCodegenFixture.h` | 442 | Public parity | Canonical isolated operation matrix over every supported element type, register width, and ISA profile. | +| `RegisterSpecializedCodegenFixture.h` | 138 | Public parity | Covers specialized arithmetic and reduction methods that are absent from the basic type matrix. | +| `RegisterFmaCodegenFixture.h` | 2 | Instruction property | Isolates the two multiply-add symbols so FMA presence and absence cannot be satisfied by unrelated code. | +| `RegisterRearrangementCodegenFixture.h` | 181 | Public parity | Covers immediate selectors, complete-register shuffles, bit casts, numeric conversions, lower halves, and widening cells. | +| `RegisterAbi.cpp` | 12 | ABI boundary | Separates explicit-object signature mirrors from real downstream `Register` and `RegisterMask` boundaries. | +| `RegisterDefaultAbi.cpp` | 1 | Explicitly diagnostic evidence | Records the platform-default aggregate convention without treating it as a supported zero-overhead boundary. | +| `MethodFlagsFlagged.cpp` | 11 | Compiler-attribute enforcement | Compares `SIMD_FLAGS(...)` with equivalent raw compiler attributes and checks inlining and stack restrictions. | + +The total is 810 retained source-level symbols. The CSV ledger is authoritative +for individual decisions; the table above is only a fixture summary. + +## Raw-baseline policy + +Public zero-overhead fixtures compare `Register` with the narrowest equivalent +public `Api` expression. A raw translation unit must not call `Register`, an +implementation specialization, or an extension helper. Sharing the production +implementation beneath the two public layers is intentional: the independent +boundary under test is the `Register` abstraction itself. + +ABI fixtures instead compare aggregate signatures with native-vector signatures. +Method-flag fixtures compare `SIMD_FLAGS(...)` declarations with equivalent +raw compiler-attribute declarations. The platform-default ABI fixture is a paired +diagnostic recording rather than an equality gate. + +## Comparison records and owning validation + +Each record appears exactly once in its profile's generated +`all-records.txt`. The profile also writes disjoint `enforced-records.txt` and +`diagnostic-records.txt` indexes. Release validation requires every enforced +record to report `ENFORCE`; a record-only result can appear only in the +diagnostic index and cannot satisfy that gate. `RegisterExpressionCodegen` +and `RegisterConsumerAbi` are build-only orchestration targets and do +not own validation. + +| Record | Symbol selection | Wrapper input | Raw input | Owning validation | +|---|---|---|---|---| +| `primary-composition` | Memory-capable and composed primary symbols | `RegisterCodegen.cpp` | `RegisterCodegenRaw.cpp` | `RegisterCodegen.` | +| `register-only` | Register-only primary symbols | `RegisterCodegen.cpp` | `RegisterCodegenRaw.cpp` | `RegisterCodegen.` | +| `reassignment` | Ordinary reassignment arithmetic | `RegisterCodegen.cpp` | `RegisterCodegenRaw.cpp` | `RegisterCodegen.` | +| `specialized` | All FMA-independent specialized symbols | `RegisterSpecializedCodegen.cpp` | `RegisterSpecializedCodegenRaw.cpp` | `RegisterCodegen.` | +| `fma-disabled` | `multiply_add_f32` and `multiply_add_f64` | `RegisterFmaCodegen.cpp` with FMA disabled | `RegisterFmaCodegenRaw.cpp` with FMA disabled | `RegisterCodegen.` | +| `fma-enabled` | `multiply_add_f32` and `multiply_add_f64` | `RegisterFmaCodegen.cpp` with FMA enabled | `RegisterFmaCodegenRaw.cpp` with FMA enabled | AVX2 `RegisterCodegen.` | +| `rearrangement-conversion` | All applicable rearrangement symbols | `RegisterRearrangementCodegen.cpp` | `RegisterRearrangementCodegenRaw.cpp` | `RegisterCodegen.` | +| `common-type-matrix` | All applicable non-modulus type-matrix symbols | `RegisterTypeMatrixCodegen.cpp` | `RegisterTypeMatrixCodegenRaw.cpp` | `RegisterCodegen.` | +| `modulus-type-matrix` | Integer modulus symbols | `RegisterTypeMatrixCodegen.cpp` | `RegisterTypeMatrixCodegenRaw.cpp` | `RegisterCodegen.` | +| `abi` | Explicit-object ABI mirrors | `RegisterAbi.cpp` | `RegisterAbiRaw.cpp` | `RegisterCodegen.` | +| `consumer-abi` | Real downstream Register and RegisterMask boundaries | `RegisterAbi.cpp` | `RegisterAbiRaw.cpp` | `RegisterCodegen.` | +| `default-abi` | Platform-default aggregate boundary | `RegisterDefaultAbi.cpp` | `RegisterDefaultAbiRaw.cpp` | `RegisterCodegen.` | +| `method-flags` | `SIMD_FLAGS(...)` declaration fixtures | `MethodFlagsFlagged.cpp` | `MethodFlagsRaw.cpp` | `MethodFlagsCodegen` | + +SSE4.2/128 owns 11 Register records because it has no FMA-enabled record. +AVX2/128 and AVX2/256 each own 12. The method-flags comparison is owned by its +single configuration-probe validation. + +Unified native and container runners aggregate only these CMake-owned indexes. +They do not recursively discover residual JSON files in reused build trees, so +retired artifacts cannot acquire validation ownership. Ordinary Debug, +sanitizer, and coverage profiles configure no Register codegen targets or +indexes. Explicit diagnostic profiles contain only record-only codegen targets. + +## Source and build inventory + +| Fixture family | Complete source inventory | +|---|---| +| Primary | `tests/codegen/RegisterCodegen.cpp`, `RegisterCodegenRaw.cpp`, and `RegisterCodegenFixture.h` | +| Specialized | `tests/codegen/RegisterSpecializedCodegen.cpp`, `RegisterSpecializedCodegenRaw.cpp`, and `RegisterSpecializedCodegenFixture.h` | +| FMA | `tests/codegen/RegisterFmaCodegen.cpp`, `RegisterFmaCodegenRaw.cpp`, and `RegisterFmaCodegenFixture.h` | +| Rearrangement | `tests/codegen/RegisterRearrangementCodegen.cpp`, `RegisterRearrangementCodegenRaw.cpp`, and `RegisterRearrangementCodegenFixture.h` | +| Type matrix | `tests/codegen/RegisterTypeMatrixCodegen.cpp`, `RegisterTypeMatrixCodegenRaw.cpp`, and `RegisterTypeMatrixCodegenFixture.h` | +| Explicit-object and consumer ABI | `tests/codegen/RegisterAbi.cpp` and `RegisterAbiRaw.cpp` | +| Platform-default ABI | `tests/codegen/RegisterDefaultAbi.cpp` and `RegisterDefaultAbiRaw.cpp` | +| Method attributes | `tests/method_flags/codegen/MethodFlagsFlagged.cpp` and `MethodFlagsRaw.cpp` | + +`cmake/development/RegisterCodegen.cmake` owns the per-profile object targets, +records, aggregate build targets, policy-separated record indexes, and three +Register CTests. +`cmake/development/MethodFlagsCodegen.cmake` owns the method-flags pair and its +CTest. `CompareRegisterCodegen.cmake`, `RecordRegisterDefaultAbi.cmake`, +`ValidateCodegenRecords.cmake`, `ValidateRegisterCodegenProfile.cmake`, +`VerifyCodegenProfileIsolation.cmake`, `VerifyMethodFlagsCodegen.cmake`, and +`VerifyMethodFlagsCodegenRecords.cmake` are the complete comparison, diagnostic, +record-integrity, profile-isolation, and attribute-verification script inputs. + +Every `` suffix is one of `128Sse42`, `128Avx2`, or `256Avx2`: + +| Target family | Complete generated target inventory | +|---|---| +| Primary objects | `RegisterCodegenWrapper`, `RegisterCodegenRaw` | +| Default ABI objects | `RegisterDefaultAbiWrapper`, `RegisterDefaultAbiRaw` | +| Explicit-object and consumer ABI objects | `RegisterAbiWrapper`, `RegisterAbiRaw` | +| Specialized objects | `RegisterSpecializedWrapper`, `RegisterSpecializedRaw` | +| FMA-disabled objects | `RegisterFmaDisabledWrapper`, `RegisterFmaDisabledRaw` | +| FMA-enabled objects | `RegisterFmaEnabledWrapper`, `RegisterFmaEnabledRaw` for AVX2 profiles | +| Rearrangement objects | `RegisterRearrangementWrapper`, `RegisterRearrangementRaw` | +| Type-matrix objects | `RegisterTypeMatrixWrapper`, `RegisterTypeMatrixRaw` | +| Register orchestration | `RegisterExpressionCodegen`, `RegisterConsumerAbi`, `RegisterCodegen`, and `RegisterCodegen` | +| Method attributes | `MethodFlagsCodegenFlagged`, `MethodFlagsCodegenLegacy`, and `MethodFlagsCodegen` | + +The orchestration targets do not define additional contracts. The complete CTest +inventory is `RegisterCodegen.128Sse42`, `RegisterCodegen.128Avx2`, +`RegisterCodegen.256Avx2`, and `MethodFlagsCodegen`. +## Artifact and documentation inventory + +Register artifacts live below: + +- `register-codegen/sse42/128`; +- `register-codegen/avx2/128`; and +- `register-codegen/avx2/256`. + +Method-attribute artifacts live below `method-flags-codegen`. Default pipeline +publication uses Release roots. `tools/Record-Codegen.ps1` creates a separate +selected Debug or Clang sanitizer record set and dedicated provenance containing +compiler flags, stack-protector mode, disassembly tools, source identity, record +hashes, and separate compilation and comparison timings. + +Documentation references have these roles: + +| Documentation | Role | +|---|---| +| `RegisterQualification.md` | Supported compiler/profile matrix, enforcement policy, and diagnostic exception ledger. | +| `RegisterProposal.md` | Public zero-overhead and ABI requirements. | +| `RegisterImplementationMatrix.md` | Public-operation-to-generated-code traceability. | +| `MethodFlagsContract.md` | Compiler-attribute promises, compiler mappings, and extension policy. | +| `BuildPipeline.md` and `ContainerValidation.md` | Reproduction commands and execution-reporting boundaries. | +| `SimdLibDevelopment.todo`, `TestCoverageExpansion.todo`, and `project.todo` | Active planning and project backlog; not normative pass claims. | +| `README.md` and `wiki/Technical-Reference.md` | User-facing support and performance guidance. | + +## Removed redundant fixtures + +| Removed fixture or symbol family | Redundancy reason | +|---|---| +| `LogicalShuffleCodegenRaw.cpp` and `LogicalShuffleIntrinsic` | Reimplemented the intrinsic algorithm; public `Register::shuffle` versus public `Api::shuffle` is the permanent boundary. | +| Handwritten scalar remainder baselines | Duplicated the selected production algorithm; algorithm comparison belongs in execution evidence or benchmarks. | +| Direct type-matrix implementation-layer runtime extract/insert symbols | Compared `Api` with its implementation rather than testing a public `Register` contract. | +| Primary isolated unary, binary, scalar, mask, construction, transfer, arithmetic, sign-bit, and runtime per-lane shift symbols | Duplicated canonical isolated type-matrix cells. | +| Uninstantiated aggregate `evaluate` and `transfer` helpers | Emitted no permanent contract and added fixture complexity. | +| Specialized fixtures rebuilt under both FMA modes | Revalidated FMA-independent symbols; only isolated multiply-add cells require the mode split. | +| Overlapping lane and full-primary comparison records | Revalidated symbols already owned by narrower nonoverlapping records. | +| Identity-return fixtures for unavailable operation/type cells | Produced code without a supported public operation and could hide availability mistakes. | + +Temporary candidate-implementation comparisons are not permanent fixtures. +Reusable throughput or latency investigations belong in benchmarks; one-time +compiler decisions belong in execution reporting. diff --git a/docs/RegisterCodegenSymbolAudit.csv b/docs/RegisterCodegenSymbolAudit.csv new file mode 100644 index 0000000..d652218 --- /dev/null +++ b/docs/RegisterCodegenSymbolAudit.csv @@ -0,0 +1,811 @@ +"symbol","owning_fixture","applicability","contract_category","comparison_baseline","comparison_record","owning_validation","decision","rationale" +"simdlib_abi_binary","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The explicit-object aggregate mirror isolates one non-inlined SIMD_FLAGS(...) signature shape from operation semantics." +"simdlib_abi_mask","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The explicit-object aggregate mirror isolates one non-inlined SIMD_FLAGS(...) signature shape from operation semantics." +"simdlib_abi_mutate","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The explicit-object aggregate mirror isolates one non-inlined SIMD_FLAGS(...) signature shape from operation semantics." +"simdlib_abi_native","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The explicit-object aggregate mirror isolates one non-inlined SIMD_FLAGS(...) signature shape from operation semantics." +"simdlib_abi_scalar","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The explicit-object aggregate mirror isolates one non-inlined SIMD_FLAGS(...) signature shape from operation semantics." +"simdlib_abi_store","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The explicit-object aggregate mirror isolates one non-inlined SIMD_FLAGS(...) signature shape from operation semantics." +"simdlib_abi_ternary","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The explicit-object aggregate mirror isolates one non-inlined SIMD_FLAGS(...) signature shape from operation semantics." +"simdlib_abi_unary","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The explicit-object aggregate mirror isolates one non-inlined SIMD_FLAGS(...) signature shape from operation semantics." +"simdlib_consumer_abi_mask_pass","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","consumer-abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","A real public Register or RegisterMask crosses the downstream non-inlined SIMD_FLAGS(...) boundary and is compared with the native signature." +"simdlib_consumer_abi_mask_return","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","consumer-abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","A real public Register or RegisterMask crosses the downstream non-inlined SIMD_FLAGS(...) boundary and is compared with the native signature." +"simdlib_consumer_abi_register_pass","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","consumer-abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","A real public Register or RegisterMask crosses the downstream non-inlined SIMD_FLAGS(...) boundary and is compared with the native signature." +"simdlib_consumer_abi_register_return","tests/codegen/RegisterAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","ABI boundary","tests/codegen/RegisterAbiRaw.cpp matching native-vector signature","consumer-abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","A real public Register or RegisterMask crosses the downstream non-inlined SIMD_FLAGS(...) boundary and is compared with the native signature." +"simdlib_codegen_aligned_transfer","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","primary-composition","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Aligned load and aligned store must optimize as one transfer chain; isolated load/store cells do not cover the chain." +"simdlib_codegen_basic_bitwise","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Chained bitwise operators including public andnot polarity must collapse to the Api expression." +"simdlib_codegen_basic_broadcast_chain","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Multiple scalar broadcasts in an arithmetic chain must add no wrapper work." +"simdlib_codegen_basic_shift_left_immediate","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","instruction-property guarantee","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","A compile-time shift count must retain the immediate public operation code shape." +"simdlib_codegen_broadcast_reuse","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","A reused broadcast value must remain common and avoid redundant wrapper work." +"simdlib_codegen_byte_transfer","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","primary-composition","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Byte-span load and store must optimize as one transfer chain; isolated load/store cells do not cover the chain." +"simdlib_codegen_complete_byte_shift","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterCodegenRaw.cpp public Api expression","primary-composition","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit slow runtime complete-register byte shift must match the Api operation without wrapper storage." +"simdlib_codegen_complete_shift_runtime","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterCodegenRaw.cpp public Api expression","primary-composition","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit slow runtime complete-register bit shift must match the Api operation without wrapper storage." +"simdlib_codegen_complete_shift_static","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterCodegenRaw.cpp public Api expression","primary-composition","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","Static complete-register bit shifting must match the Api operation." +"simdlib_codegen_lane_last","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Nonzero constant-index lane extraction protects the highest-lane public path." +"simdlib_codegen_load_operate_store","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","primary-composition","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Load, arithmetic, and store must optimize as one memory-capable expression." +"simdlib_codegen_mask_all","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Comparison followed by all-lane reduction must compose without wrapper overhead." +"simdlib_codegen_mask_any","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Comparison followed by any-lane reduction must compose without wrapper overhead." +"simdlib_codegen_mask_bits","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Comparison followed by compact-mask extraction must compose without wrapper overhead." +"simdlib_codegen_mask_combine","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Comparison and predicate union must compose without wrapper overhead." +"simdlib_codegen_mask_select","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Comparison and selection must compose without wrapper overhead." +"simdlib_codegen_mutate","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","primary-composition","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","A caller-owned native reference updated through a local Register must not acquire wrapper overhead." +"simdlib_codegen_native","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Aggregate wrapping and native-member observation must add no instructions." +"simdlib_codegen_opaque","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","register-pressure behavior","tests/codegen/RegisterCodegenRaw.cpp public Api expression","primary-composition","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","A wrapped value kept live across an opaque call must match raw spill and reload behavior." +"simdlib_codegen_pressure","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","register-pressure behavior","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Multiple simultaneously live wrapper values must match the raw register-pressure expression." +"simdlib_codegen_reassignment_arithmetic","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","reassignment","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Successive ordinary assignments must match the equivalent nested Api arithmetic." +"simdlib_codegen_special_members","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Aggregate copy construction and assignment must add no runtime work." +"simdlib_codegen_ternary","tests/codegen/RegisterCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","composed-expression optimization","tests/codegen/RegisterCodegenRaw.cpp public Api expression","register-only","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","Operator multiply-add composition must collapse to the equivalent Api expression." +"simdlib_codegen_default","tests/codegen/RegisterDefaultAbi.cpp","SSE4.2/128; AVX2/128; AVX2/256","explicitly diagnostic evidence","tests/codegen/RegisterDefaultAbiRaw.cpp native-vector platform-default signature","default-abi","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The platform-default aggregate calling convention is recorded for diagnosis and is excluded from zero-overhead pass/fail claims." +"simdlib_fma_codegen_multiply_add_f32","tests/codegen/RegisterFmaCodegenFixture.h","SSE4.2/128 FMA-disabled; AVX2/128 and AVX2/256 FMA-disabled and FMA-enabled","instruction-property guarantee","tests/codegen/RegisterFmaCodegenRaw.cpp matching public Api::multiply_add operation","fma-disabled; fma-enabled on AVX2","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated f32 multiply-add cell enforces both fusion absence and fusion presence in the corresponding feature mode." +"simdlib_fma_codegen_multiply_add_f64","tests/codegen/RegisterFmaCodegenFixture.h","SSE4.2/128 FMA-disabled; AVX2/128 and AVX2/256 FMA-disabled and FMA-enabled","instruction-property guarantee","tests/codegen/RegisterFmaCodegenRaw.cpp matching public Api::multiply_add operation","fma-disabled; fma-enabled on AVX2","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated f64 multiply-add cell enforces both fusion absence and fusion presence in the corresponding feature mode." +"simdlib_rearrangement_codegen_bit_cast_f32_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f32_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f32_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f32_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f32_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f32_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f32_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f32_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f32_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f32_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_f64_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f64-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i16_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i16-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i32_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i64_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i64-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_i8_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i8-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u16_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u16-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u32_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u64_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u64-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-f32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-f64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-i16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-i32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-i64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-i8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-u16 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-u32 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-u64 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_bit_cast_u8_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u8-to-u8 full-width bit-cast cell independently protects that public template instantiation." +"simdlib_rearrangement_codegen_blend_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate blend rearrangement for f32 must match its public Api operation." +"simdlib_rearrangement_codegen_blend_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate blend rearrangement for f64 must match its public Api operation." +"simdlib_rearrangement_codegen_blend_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate blend rearrangement for i16 must match its public Api operation." +"simdlib_rearrangement_codegen_blend_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate blend rearrangement for i32 must match its public Api operation." +"simdlib_rearrangement_codegen_blend_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate blend rearrangement for u16 must match its public Api operation." +"simdlib_rearrangement_codegen_blend_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate blend rearrangement for u32 must match its public Api operation." +"simdlib_rearrangement_codegen_byte_shuffle_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The 128-bit complete byte reversal must match the public Api shuffle." +"simdlib_rearrangement_codegen_byte_shuffle_i32_cross","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The 256-bit i32_cross selector pattern protects the corresponding local or cross-half public byte-shuffle path." +"simdlib_rearrangement_codegen_byte_shuffle_i32_local","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The 256-bit i32_local selector pattern protects the corresponding local or cross-half public byte-shuffle path." +"simdlib_rearrangement_codegen_byte_shuffle_i32_mixed","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The 256-bit i32_mixed selector pattern protects the corresponding local or cross-half public byte-shuffle path." +"simdlib_rearrangement_codegen_convert_f32_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The f32-to-i32 numeric conversion must match the public Api operation." +"simdlib_rearrangement_codegen_convert_i32_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The i32-to-f32 numeric conversion must match the public Api operation." +"simdlib_rearrangement_codegen_convert_u32_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The u32-to-f32 numeric conversion must match the public Api operation." +"simdlib_rearrangement_codegen_logical_shuffle_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for f32 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_logical_shuffle_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for f64 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_logical_shuffle_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for i16 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_logical_shuffle_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for i32 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_logical_shuffle_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for i64 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_logical_shuffle_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for i8 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_logical_shuffle_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for u16 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_logical_shuffle_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for u32 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_logical_shuffle_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for u64 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_logical_shuffle_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The width-specific logical shuffle for u8 protects complete-register selector lowering, including cross-128-bit movement at 256 bits." +"simdlib_rearrangement_codegen_lower_half_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The f32 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_lower_half_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The f64 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_lower_half_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The i16 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_lower_half_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The i32 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_lower_half_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The i64 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_lower_half_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The i8 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_lower_half_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The u16 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_lower_half_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The u32 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_lower_half_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The u64 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_lower_half_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.256Avx2","retain","The u8 lower-half conversion changes register width and must match the Api boundary." +"simdlib_rearrangement_codegen_shuffle_high_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate shuffle_high rearrangement for i16 must match its public Api operation." +"simdlib_rearrangement_codegen_shuffle_high_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate shuffle_high rearrangement for u16 must match its public Api operation." +"simdlib_rearrangement_codegen_shuffle_low_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate shuffle_low rearrangement for i16 must match its public Api operation." +"simdlib_rearrangement_codegen_shuffle_low_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The immediate shuffle_low rearrangement for u16 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for f32 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for f64 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for i16 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for i32 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for i64 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for i8 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for u16 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for u32 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for u64 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_high_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_high rearrangement for u8 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_f32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for f32 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_f64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for f64 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_i16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for i16 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_i32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for i32 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_i64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for i64 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_i8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for i8 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_u16","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for u16 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_u32","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for u32 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_u64","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for u64 must match its public Api operation." +"simdlib_rearrangement_codegen_unpack_low_u8","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated unpack_low rearrangement for u8 must match its public Api operation." +"simdlib_rearrangement_codegen_widen_i16_i32_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane i16-to-i32 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i16_i32_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane i16-to-i32 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i16_i64_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane i16-to-i64 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i16_i64_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane i16-to-i64 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i32_i64_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane i32-to-i64 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i32_i64_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane i32-to-i64 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i8_i16_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane i8-to-i16 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i8_i16_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane i8-to-i16 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i8_i32_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane i8-to-i32 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i8_i32_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane i8-to-i32 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i8_i64_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane i8-to-i64 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_i8_i64_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane i8-to-i64 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u16_u32_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane u16-to-u32 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u16_u32_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane u16-to-u32 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u16_u64_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane u16-to-u64 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u16_u64_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane u16-to-u64 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u32_u64_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane u32-to-u64 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u32_u64_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane u32-to-u64 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u8_u16_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane u8-to-u16 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u8_u16_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane u8-to-u16 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u8_u32_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane u8-to-u32 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u8_u32_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane u8-to-u32 256-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u8_u64_128","tests/codegen/RegisterRearrangementCodegenFixture.h","SSE4.2/128; AVX2/128","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2","retain","The explicit low-lane u8-to-u64 128-bit widening result must match the public Api operation." +"simdlib_rearrangement_codegen_widen_u8_u64_256","tests/codegen/RegisterRearrangementCodegenFixture.h","AVX2/128 source to 256-bit result","public abstraction parity","tests/codegen/RegisterRearrangementCodegenRaw.cpp matching public Api operation","rearrangement-conversion","RegisterCodegen.128Avx2","retain","The explicit low-lane u8-to-u64 256-bit widening result must match the public Api operation." +"simdlib_specialized_codegen_absolute_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_absolute_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_absolute_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_absolute_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_absolute_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_absolute_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_absolute_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_absolute_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_absolute_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_absolute_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::absolute specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_add_saturated_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated add_saturated specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_add_saturated_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated add_saturated specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_add_saturated_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated add_saturated specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_add_saturated_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated add_saturated specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_add_subtract_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated add_subtract specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_add_subtract_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated add_subtract specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_average_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated average specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_average_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated average specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_byte_multiply_add_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated byte_multiply_add specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_byte_multiply_add_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated byte_multiply_add specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_byte_multiply_add_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated byte_multiply_add specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_byte_multiply_add_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated byte_multiply_add specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_byte_multiply_add_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated byte_multiply_add specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_byte_multiply_add_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated byte_multiply_add specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_byte_multiply_add_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated byte_multiply_add specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_byte_multiply_add_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated byte_multiply_add specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_dot_product_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated dot_product specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_dot_product_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated dot_product specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_add_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_add specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_add_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_add specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_add_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_add specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_add_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_add specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_add_saturated_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_add_saturated specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_add_saturated_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_add_saturated specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_add_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_add specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_add_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_add specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_subtract_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_subtract specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_subtract_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_subtract specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_subtract_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_subtract specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_subtract_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_subtract specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_subtract_saturated_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_subtract_saturated specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_subtract_saturated_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_subtract_saturated specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_subtract_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_subtract specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_horizontal_subtract_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated horizontal_subtract specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_checked_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated checked-magnitude specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_checked_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated checked-magnitude specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_checked_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated checked-magnitude specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_checked_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated checked-magnitude specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_checked_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated checked-magnitude specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_checked_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated checked-magnitude specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_checked_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated checked-magnitude specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_checked_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated checked-magnitude specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_magnitude_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::magnitude specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_max_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_max_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_max_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_max_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_max_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_max_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_max_position_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated max_position specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_max_position_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated max_position specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_max_position_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated max_position specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_max_position_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated max_position specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_max_position_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated max_position specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_max_position_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated max_position specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_max_position_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated max_position specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_max_position_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated max_position specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_max_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_max_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_max_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_max_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::max specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_min_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_min_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_min_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_min_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_min_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_min_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_min_position_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated min_position specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_min_position_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated min_position specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_min_position_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated min_position specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_min_position_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated min_position specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_min_position_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated min_position specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_min_position_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated min_position specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_min_position_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated min_position specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_min_position_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated min_position specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_min_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_min_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_min_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_min_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::min specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_multi_sad_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multi_sad specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_multi_sad_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multi_sad specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_multi_sad_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multi_sad specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_multi_sad_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multi_sad specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_multi_sad_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multi_sad specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_multi_sad_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multi_sad specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_multi_sad_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multi_sad specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_multi_sad_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multi_sad specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_multiply_add_adjacent_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multiply_add_adjacent specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_multiply_add_adjacent_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multiply_add_adjacent specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_multiply_add_adjacent_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multiply_add_adjacent specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_multiply_add_adjacent_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multiply_add_adjacent specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_multiply_add_adjacent_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multiply_add_adjacent specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_multiply_add_adjacent_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multiply_add_adjacent specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_multiply_add_adjacent_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multiply_add_adjacent specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_multiply_add_adjacent_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated multiply_add_adjacent specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_normalize_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated normalize specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_normalize_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated normalize specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_f32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for f32 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_f64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for f64 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_sqrt_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated Register::sqrt specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_subtract_saturated_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated subtract_saturated specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_subtract_saturated_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated subtract_saturated specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_subtract_saturated_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated subtract_saturated specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_subtract_saturated_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated subtract_saturated specialization for u8 must match its public Api operation." +"simdlib_specialized_codegen_sum_absolute_byte_differences_i16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated sum_absolute_byte_differences specialization for i16 must match its public Api operation." +"simdlib_specialized_codegen_sum_absolute_byte_differences_i32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated sum_absolute_byte_differences specialization for i32 must match its public Api operation." +"simdlib_specialized_codegen_sum_absolute_byte_differences_i64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated sum_absolute_byte_differences specialization for i64 must match its public Api operation." +"simdlib_specialized_codegen_sum_absolute_byte_differences_i8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated sum_absolute_byte_differences specialization for i8 must match its public Api operation." +"simdlib_specialized_codegen_sum_absolute_byte_differences_u16","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated sum_absolute_byte_differences specialization for u16 must match its public Api operation." +"simdlib_specialized_codegen_sum_absolute_byte_differences_u32","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated sum_absolute_byte_differences specialization for u32 must match its public Api operation." +"simdlib_specialized_codegen_sum_absolute_byte_differences_u64","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated sum_absolute_byte_differences specialization for u64 must match its public Api operation." +"simdlib_specialized_codegen_sum_absolute_byte_differences_u8","tests/codegen/RegisterSpecializedCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterSpecializedCodegenRaw.cpp matching public Api operation","specialized","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated sum_absolute_byte_differences specialization for u8 must match its public Api operation." +"simdlib_type_matrix_add_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for f32 protects that public Register specialization." +"simdlib_type_matrix_add_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for f64 protects that public Register specialization." +"simdlib_type_matrix_add_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for i16 protects that public Register specialization." +"simdlib_type_matrix_add_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for i32 protects that public Register specialization." +"simdlib_type_matrix_add_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for i64 protects that public Register specialization." +"simdlib_type_matrix_add_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for i8 protects that public Register specialization." +"simdlib_type_matrix_add_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for u16 protects that public Register specialization." +"simdlib_type_matrix_add_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for u32 protects that public Register specialization." +"simdlib_type_matrix_add_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for u64 protects that public Register specialization." +"simdlib_type_matrix_add_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated add cell for u8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for f32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for f64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for i16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for i32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for i64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for i8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for u16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for u32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for u64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_and_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_and cell for u8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for f32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for f64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for i16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for i32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for i64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for i8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for u16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for u32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for u64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_andnot_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_andnot cell for u8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for f32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for f64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for i16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for i32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for i64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for i8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for u16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for u32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for u64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_not_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_not cell for u8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for f32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for f64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for i16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for i32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for i64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for i8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for u16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for u32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for u64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_or_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_or cell for u8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for f32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for f64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for i16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for i32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for i64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for i8 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for u16 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for u32 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for u64 protects that public Register specialization." +"simdlib_type_matrix_bitwise_xor_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated bitwise_xor cell for u8 protects that public Register specialization." +"simdlib_type_matrix_broadcast_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for f32 protects that public Register specialization." +"simdlib_type_matrix_broadcast_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for f64 protects that public Register specialization." +"simdlib_type_matrix_broadcast_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for i16 protects that public Register specialization." +"simdlib_type_matrix_broadcast_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for i32 protects that public Register specialization." +"simdlib_type_matrix_broadcast_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for i64 protects that public Register specialization." +"simdlib_type_matrix_broadcast_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for i8 protects that public Register specialization." +"simdlib_type_matrix_broadcast_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for u16 protects that public Register specialization." +"simdlib_type_matrix_broadcast_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for u32 protects that public Register specialization." +"simdlib_type_matrix_broadcast_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for u64 protects that public Register specialization." +"simdlib_type_matrix_broadcast_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated broadcast cell for u8 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for f32 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for f64 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for i16 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for i32 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for i64 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for i8 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for u16 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for u32 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for u64 protects that public Register specialization." +"simdlib_type_matrix_compare_equal_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_equal cell for u8 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for f32 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for f64 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for i16 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for i32 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for i64 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for i8 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for u16 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for u32 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for u64 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_equal_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater_equal cell for u8 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for f32 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for f64 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for i16 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for i32 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for i64 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for i8 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for u16 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for u32 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for u64 protects that public Register specialization." +"simdlib_type_matrix_compare_greater_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_greater cell for u8 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for f32 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for f64 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for i16 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for i32 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for i64 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for i8 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for u16 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for u32 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for u64 protects that public Register specialization." +"simdlib_type_matrix_compare_less_equal_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less_equal cell for u8 protects that public Register specialization." +"simdlib_type_matrix_compare_less_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for f32 protects that public Register specialization." +"simdlib_type_matrix_compare_less_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for f64 protects that public Register specialization." +"simdlib_type_matrix_compare_less_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for i16 protects that public Register specialization." +"simdlib_type_matrix_compare_less_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for i32 protects that public Register specialization." +"simdlib_type_matrix_compare_less_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for i64 protects that public Register specialization." +"simdlib_type_matrix_compare_less_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for i8 protects that public Register specialization." +"simdlib_type_matrix_compare_less_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for u16 protects that public Register specialization." +"simdlib_type_matrix_compare_less_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for u32 protects that public Register specialization." +"simdlib_type_matrix_compare_less_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for u64 protects that public Register specialization." +"simdlib_type_matrix_compare_less_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated compare_less cell for u8 protects that public Register specialization." +"simdlib_type_matrix_construct_array_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for f32 protects that public Register specialization." +"simdlib_type_matrix_construct_array_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for f64 protects that public Register specialization." +"simdlib_type_matrix_construct_array_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for i16 protects that public Register specialization." +"simdlib_type_matrix_construct_array_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for i32 protects that public Register specialization." +"simdlib_type_matrix_construct_array_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for i64 protects that public Register specialization." +"simdlib_type_matrix_construct_array_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for i8 protects that public Register specialization." +"simdlib_type_matrix_construct_array_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for u16 protects that public Register specialization." +"simdlib_type_matrix_construct_array_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for u32 protects that public Register specialization." +"simdlib_type_matrix_construct_array_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for u64 protects that public Register specialization." +"simdlib_type_matrix_construct_array_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_array transfer cell for u8 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for f32 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for f64 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for i16 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for i32 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for i64 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for i8 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for u16 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for u32 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for u64 protects that public Register specialization." +"simdlib_type_matrix_construct_lanes_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated construct_lanes transfer cell for u8 protects that public Register specialization." +"simdlib_type_matrix_divide_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for f32 protects that public Register specialization." +"simdlib_type_matrix_divide_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for f64 protects that public Register specialization." +"simdlib_type_matrix_divide_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for i16 protects that public Register specialization." +"simdlib_type_matrix_divide_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for i32 protects that public Register specialization." +"simdlib_type_matrix_divide_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for i64 protects that public Register specialization." +"simdlib_type_matrix_divide_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for i8 protects that public Register specialization." +"simdlib_type_matrix_divide_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for u16 protects that public Register specialization." +"simdlib_type_matrix_divide_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for u32 protects that public Register specialization." +"simdlib_type_matrix_divide_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for u64 protects that public Register specialization." +"simdlib_type_matrix_divide_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated divide cell for u8 protects that public Register specialization." +"simdlib_type_matrix_equal_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for f32 protects that public Register specialization." +"simdlib_type_matrix_equal_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for f64 protects that public Register specialization." +"simdlib_type_matrix_equal_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for i16 protects that public Register specialization." +"simdlib_type_matrix_equal_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for i32 protects that public Register specialization." +"simdlib_type_matrix_equal_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for i64 protects that public Register specialization." +"simdlib_type_matrix_equal_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for i8 protects that public Register specialization." +"simdlib_type_matrix_equal_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for u16 protects that public Register specialization." +"simdlib_type_matrix_equal_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for u32 protects that public Register specialization." +"simdlib_type_matrix_equal_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for u64 protects that public Register specialization." +"simdlib_type_matrix_equal_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result equal cell for u8 protects that public Register specialization." +"simdlib_type_matrix_extract_first_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for f32 protects that public Register specialization." +"simdlib_type_matrix_extract_first_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for f64 protects that public Register specialization." +"simdlib_type_matrix_extract_first_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for i16 protects that public Register specialization." +"simdlib_type_matrix_extract_first_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for i32 protects that public Register specialization." +"simdlib_type_matrix_extract_first_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for i64 protects that public Register specialization." +"simdlib_type_matrix_extract_first_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for i8 protects that public Register specialization." +"simdlib_type_matrix_extract_first_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for u16 protects that public Register specialization." +"simdlib_type_matrix_extract_first_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for u32 protects that public Register specialization." +"simdlib_type_matrix_extract_first_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for u64 protects that public Register specialization." +"simdlib_type_matrix_extract_first_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result extract_first cell for u8 protects that public Register specialization." +"simdlib_type_matrix_insert_last_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for f32 protects that public Register specialization." +"simdlib_type_matrix_insert_last_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for f64 protects that public Register specialization." +"simdlib_type_matrix_insert_last_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for i16 protects that public Register specialization." +"simdlib_type_matrix_insert_last_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for i32 protects that public Register specialization." +"simdlib_type_matrix_insert_last_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for i64 protects that public Register specialization." +"simdlib_type_matrix_insert_last_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for i8 protects that public Register specialization." +"simdlib_type_matrix_insert_last_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for u16 protects that public Register specialization." +"simdlib_type_matrix_insert_last_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for u32 protects that public Register specialization." +"simdlib_type_matrix_insert_last_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for u64 protects that public Register specialization." +"simdlib_type_matrix_insert_last_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated insert_last cell for u8 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for f32 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for f64 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for i16 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for i32 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for i64 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for i8 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for u16 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for u32 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for u64 protects that public Register specialization." +"simdlib_type_matrix_lane_sign_bits_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result lane_sign_bits cell for u8 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for f32 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for f64 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for i16 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for i32 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for i64 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for i8 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for u16 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for u32 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for u64 protects that public Register specialization." +"simdlib_type_matrix_load_aligned_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_aligned transfer cell for u8 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for f32 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for f64 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for i16 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for i32 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for i64 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for i8 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for u16 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for u32 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for u64 protects that public Register specialization." +"simdlib_type_matrix_load_bytes_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load_bytes transfer cell for u8 protects that public Register specialization." +"simdlib_type_matrix_load_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for f32 protects that public Register specialization." +"simdlib_type_matrix_load_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for f64 protects that public Register specialization." +"simdlib_type_matrix_load_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for i16 protects that public Register specialization." +"simdlib_type_matrix_load_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for i32 protects that public Register specialization." +"simdlib_type_matrix_load_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for i64 protects that public Register specialization." +"simdlib_type_matrix_load_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for i8 protects that public Register specialization." +"simdlib_type_matrix_load_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for u16 protects that public Register specialization." +"simdlib_type_matrix_load_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for u32 protects that public Register specialization." +"simdlib_type_matrix_load_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for u64 protects that public Register specialization." +"simdlib_type_matrix_load_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated load transfer cell for u8 protects that public Register specialization." +"simdlib_type_matrix_logical_shift_right_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count logical_shift_right cell for i16 protects that public Register specialization." +"simdlib_type_matrix_logical_shift_right_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count logical_shift_right cell for i32 protects that public Register specialization." +"simdlib_type_matrix_logical_shift_right_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count logical_shift_right cell for i64 protects that public Register specialization." +"simdlib_type_matrix_logical_shift_right_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count logical_shift_right cell for i8 protects that public Register specialization." +"simdlib_type_matrix_logical_shift_right_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count logical_shift_right cell for u16 protects that public Register specialization." +"simdlib_type_matrix_logical_shift_right_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count logical_shift_right cell for u32 protects that public Register specialization." +"simdlib_type_matrix_logical_shift_right_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count logical_shift_right cell for u64 protects that public Register specialization." +"simdlib_type_matrix_logical_shift_right_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count logical_shift_right cell for u8 protects that public Register specialization." +"simdlib_type_matrix_mask_all_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for f32 protects that public Register specialization." +"simdlib_type_matrix_mask_all_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for f64 protects that public Register specialization." +"simdlib_type_matrix_mask_all_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for i16 protects that public Register specialization." +"simdlib_type_matrix_mask_all_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for i32 protects that public Register specialization." +"simdlib_type_matrix_mask_all_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for i64 protects that public Register specialization." +"simdlib_type_matrix_mask_all_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for i8 protects that public Register specialization." +"simdlib_type_matrix_mask_all_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for u16 protects that public Register specialization." +"simdlib_type_matrix_mask_all_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for u32 protects that public Register specialization." +"simdlib_type_matrix_mask_all_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for u64 protects that public Register specialization." +"simdlib_type_matrix_mask_all_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_all cell for u8 protects that public Register specialization." +"simdlib_type_matrix_mask_and_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for f32 protects that public Register specialization." +"simdlib_type_matrix_mask_and_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for f64 protects that public Register specialization." +"simdlib_type_matrix_mask_and_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for i16 protects that public Register specialization." +"simdlib_type_matrix_mask_and_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for i32 protects that public Register specialization." +"simdlib_type_matrix_mask_and_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for i64 protects that public Register specialization." +"simdlib_type_matrix_mask_and_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for i8 protects that public Register specialization." +"simdlib_type_matrix_mask_and_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for u16 protects that public Register specialization." +"simdlib_type_matrix_mask_and_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for u32 protects that public Register specialization." +"simdlib_type_matrix_mask_and_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for u64 protects that public Register specialization." +"simdlib_type_matrix_mask_and_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_and cell for u8 protects that public Register specialization." +"simdlib_type_matrix_mask_any_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for f32 protects that public Register specialization." +"simdlib_type_matrix_mask_any_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for f64 protects that public Register specialization." +"simdlib_type_matrix_mask_any_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for i16 protects that public Register specialization." +"simdlib_type_matrix_mask_any_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for i32 protects that public Register specialization." +"simdlib_type_matrix_mask_any_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for i64 protects that public Register specialization." +"simdlib_type_matrix_mask_any_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for i8 protects that public Register specialization." +"simdlib_type_matrix_mask_any_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for u16 protects that public Register specialization." +"simdlib_type_matrix_mask_any_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for u32 protects that public Register specialization." +"simdlib_type_matrix_mask_any_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for u64 protects that public Register specialization." +"simdlib_type_matrix_mask_any_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_any cell for u8 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for f32 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for f64 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for i16 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for i32 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for i64 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for i8 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for u16 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for u32 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for u64 protects that public Register specialization." +"simdlib_type_matrix_mask_bits_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_bits cell for u8 protects that public Register specialization." +"simdlib_type_matrix_mask_none_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for f32 protects that public Register specialization." +"simdlib_type_matrix_mask_none_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for f64 protects that public Register specialization." +"simdlib_type_matrix_mask_none_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for i16 protects that public Register specialization." +"simdlib_type_matrix_mask_none_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for i32 protects that public Register specialization." +"simdlib_type_matrix_mask_none_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for i64 protects that public Register specialization." +"simdlib_type_matrix_mask_none_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for i8 protects that public Register specialization." +"simdlib_type_matrix_mask_none_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for u16 protects that public Register specialization." +"simdlib_type_matrix_mask_none_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for u32 protects that public Register specialization." +"simdlib_type_matrix_mask_none_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for u64 protects that public Register specialization." +"simdlib_type_matrix_mask_none_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result mask_none cell for u8 protects that public Register specialization." +"simdlib_type_matrix_mask_not_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for f32 protects that public Register specialization." +"simdlib_type_matrix_mask_not_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for f64 protects that public Register specialization." +"simdlib_type_matrix_mask_not_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for i16 protects that public Register specialization." +"simdlib_type_matrix_mask_not_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for i32 protects that public Register specialization." +"simdlib_type_matrix_mask_not_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for i64 protects that public Register specialization." +"simdlib_type_matrix_mask_not_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for i8 protects that public Register specialization." +"simdlib_type_matrix_mask_not_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for u16 protects that public Register specialization." +"simdlib_type_matrix_mask_not_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for u32 protects that public Register specialization." +"simdlib_type_matrix_mask_not_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for u64 protects that public Register specialization." +"simdlib_type_matrix_mask_not_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_not cell for u8 protects that public Register specialization." +"simdlib_type_matrix_mask_or_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for f32 protects that public Register specialization." +"simdlib_type_matrix_mask_or_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for f64 protects that public Register specialization." +"simdlib_type_matrix_mask_or_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for i16 protects that public Register specialization." +"simdlib_type_matrix_mask_or_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for i32 protects that public Register specialization." +"simdlib_type_matrix_mask_or_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for i64 protects that public Register specialization." +"simdlib_type_matrix_mask_or_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for i8 protects that public Register specialization." +"simdlib_type_matrix_mask_or_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for u16 protects that public Register specialization." +"simdlib_type_matrix_mask_or_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for u32 protects that public Register specialization." +"simdlib_type_matrix_mask_or_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for u64 protects that public Register specialization." +"simdlib_type_matrix_mask_or_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_or cell for u8 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for f32 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for f64 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for i16 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for i32 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for i64 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for i8 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for u16 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for u32 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for u64 protects that public Register specialization." +"simdlib_type_matrix_mask_xor_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated mask_xor cell for u8 protects that public Register specialization." +"simdlib_type_matrix_modulus_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","modulus-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated integer modulus cell for i16 is separated so compiler scheduling diagnostics cannot weaken other operations." +"simdlib_type_matrix_modulus_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","modulus-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated integer modulus cell for i32 is separated so compiler scheduling diagnostics cannot weaken other operations." +"simdlib_type_matrix_modulus_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","modulus-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated integer modulus cell for i64 is separated so compiler scheduling diagnostics cannot weaken other operations." +"simdlib_type_matrix_modulus_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","modulus-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated integer modulus cell for i8 is separated so compiler scheduling diagnostics cannot weaken other operations." +"simdlib_type_matrix_modulus_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","modulus-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated integer modulus cell for u16 is separated so compiler scheduling diagnostics cannot weaken other operations." +"simdlib_type_matrix_modulus_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","modulus-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated integer modulus cell for u32 is separated so compiler scheduling diagnostics cannot weaken other operations." +"simdlib_type_matrix_modulus_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","modulus-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated integer modulus cell for u64 is separated so compiler scheduling diagnostics cannot weaken other operations." +"simdlib_type_matrix_modulus_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","modulus-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The isolated integer modulus cell for u8 is separated so compiler scheduling diagnostics cannot weaken other operations." +"simdlib_type_matrix_movemask_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for f32 protects that public Register specialization." +"simdlib_type_matrix_movemask_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for f64 protects that public Register specialization." +"simdlib_type_matrix_movemask_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for i16 protects that public Register specialization." +"simdlib_type_matrix_movemask_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for i32 protects that public Register specialization." +"simdlib_type_matrix_movemask_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for i64 protects that public Register specialization." +"simdlib_type_matrix_movemask_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for i8 protects that public Register specialization." +"simdlib_type_matrix_movemask_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for u16 protects that public Register specialization." +"simdlib_type_matrix_movemask_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for u32 protects that public Register specialization." +"simdlib_type_matrix_movemask_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for u64 protects that public Register specialization." +"simdlib_type_matrix_movemask_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result movemask cell for u8 protects that public Register specialization." +"simdlib_type_matrix_multiply_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for f32 protects that public Register specialization." +"simdlib_type_matrix_multiply_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for f64 protects that public Register specialization." +"simdlib_type_matrix_multiply_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for i16 protects that public Register specialization." +"simdlib_type_matrix_multiply_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for i32 protects that public Register specialization." +"simdlib_type_matrix_multiply_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for i64 protects that public Register specialization." +"simdlib_type_matrix_multiply_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for i8 protects that public Register specialization." +"simdlib_type_matrix_multiply_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for u16 protects that public Register specialization." +"simdlib_type_matrix_multiply_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for u32 protects that public Register specialization." +"simdlib_type_matrix_multiply_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for u64 protects that public Register specialization." +"simdlib_type_matrix_multiply_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated multiply cell for u8 protects that public Register specialization." +"simdlib_type_matrix_negate_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for f32 protects that public Register specialization." +"simdlib_type_matrix_negate_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for f64 protects that public Register specialization." +"simdlib_type_matrix_negate_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for i16 protects that public Register specialization." +"simdlib_type_matrix_negate_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for i32 protects that public Register specialization." +"simdlib_type_matrix_negate_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for i64 protects that public Register specialization." +"simdlib_type_matrix_negate_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for i8 protects that public Register specialization." +"simdlib_type_matrix_negate_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for u16 protects that public Register specialization." +"simdlib_type_matrix_negate_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for u32 protects that public Register specialization." +"simdlib_type_matrix_negate_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for u64 protects that public Register specialization." +"simdlib_type_matrix_negate_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated negate cell for u8 protects that public Register specialization." +"simdlib_type_matrix_not_equal_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for f32 protects that public Register specialization." +"simdlib_type_matrix_not_equal_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for f64 protects that public Register specialization." +"simdlib_type_matrix_not_equal_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for i16 protects that public Register specialization." +"simdlib_type_matrix_not_equal_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for i32 protects that public Register specialization." +"simdlib_type_matrix_not_equal_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for i64 protects that public Register specialization." +"simdlib_type_matrix_not_equal_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for i8 protects that public Register specialization." +"simdlib_type_matrix_not_equal_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for u16 protects that public Register specialization." +"simdlib_type_matrix_not_equal_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for u32 protects that public Register specialization." +"simdlib_type_matrix_not_equal_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for u64 protects that public Register specialization." +"simdlib_type_matrix_not_equal_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated scalar-result not_equal cell for u8 protects that public Register specialization." +"simdlib_type_matrix_observe_array_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for f32 protects that public Register specialization." +"simdlib_type_matrix_observe_array_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for f64 protects that public Register specialization." +"simdlib_type_matrix_observe_array_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for i16 protects that public Register specialization." +"simdlib_type_matrix_observe_array_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for i32 protects that public Register specialization." +"simdlib_type_matrix_observe_array_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for i64 protects that public Register specialization." +"simdlib_type_matrix_observe_array_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for i8 protects that public Register specialization." +"simdlib_type_matrix_observe_array_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for u16 protects that public Register specialization." +"simdlib_type_matrix_observe_array_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for u32 protects that public Register specialization." +"simdlib_type_matrix_observe_array_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for u64 protects that public Register specialization." +"simdlib_type_matrix_observe_array_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated observe_array transfer cell for u8 protects that public Register specialization." +"simdlib_type_matrix_select_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for f32 protects that public Register specialization." +"simdlib_type_matrix_select_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for f64 protects that public Register specialization." +"simdlib_type_matrix_select_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for i16 protects that public Register specialization." +"simdlib_type_matrix_select_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for i32 protects that public Register specialization." +"simdlib_type_matrix_select_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for i64 protects that public Register specialization." +"simdlib_type_matrix_select_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for i8 protects that public Register specialization." +"simdlib_type_matrix_select_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for u16 protects that public Register specialization." +"simdlib_type_matrix_select_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for u32 protects that public Register specialization." +"simdlib_type_matrix_select_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for u64 protects that public Register specialization." +"simdlib_type_matrix_select_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated select cell for u8 protects that public Register specialization." +"simdlib_type_matrix_shift_left_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_left cell for i16 protects that public Register specialization." +"simdlib_type_matrix_shift_left_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_left cell for i32 protects that public Register specialization." +"simdlib_type_matrix_shift_left_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_left cell for i64 protects that public Register specialization." +"simdlib_type_matrix_shift_left_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_left cell for i8 protects that public Register specialization." +"simdlib_type_matrix_shift_left_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_left cell for u16 protects that public Register specialization." +"simdlib_type_matrix_shift_left_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_left cell for u32 protects that public Register specialization." +"simdlib_type_matrix_shift_left_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_left cell for u64 protects that public Register specialization." +"simdlib_type_matrix_shift_left_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_left cell for u8 protects that public Register specialization." +"simdlib_type_matrix_shift_right_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_right cell for i16 protects that public Register specialization." +"simdlib_type_matrix_shift_right_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_right cell for i32 protects that public Register specialization." +"simdlib_type_matrix_shift_right_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_right cell for i64 protects that public Register specialization." +"simdlib_type_matrix_shift_right_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_right cell for i8 protects that public Register specialization." +"simdlib_type_matrix_shift_right_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_right cell for u16 protects that public Register specialization." +"simdlib_type_matrix_shift_right_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_right cell for u32 protects that public Register specialization." +"simdlib_type_matrix_shift_right_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_right cell for u64 protects that public Register specialization." +"simdlib_type_matrix_shift_right_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated runtime-count shift_right cell for u8 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for f32 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for f64 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for i16 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for i32 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for i64 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for i8 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for u16 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for u32 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for u64 protects that public Register specialization." +"simdlib_type_matrix_store_aligned_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_aligned transfer cell for u8 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for f32 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for f64 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for i16 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for i32 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for i64 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for i8 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for u16 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for u32 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for u64 protects that public Register specialization." +"simdlib_type_matrix_store_bytes_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store_bytes transfer cell for u8 protects that public Register specialization." +"simdlib_type_matrix_store_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for f32 protects that public Register specialization." +"simdlib_type_matrix_store_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for f64 protects that public Register specialization." +"simdlib_type_matrix_store_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for i16 protects that public Register specialization." +"simdlib_type_matrix_store_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for i32 protects that public Register specialization." +"simdlib_type_matrix_store_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for i64 protects that public Register specialization." +"simdlib_type_matrix_store_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for i8 protects that public Register specialization." +"simdlib_type_matrix_store_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for u16 protects that public Register specialization." +"simdlib_type_matrix_store_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for u32 protects that public Register specialization." +"simdlib_type_matrix_store_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for u64 protects that public Register specialization." +"simdlib_type_matrix_store_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated store transfer cell for u8 protects that public Register specialization." +"simdlib_type_matrix_subtract_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for f32 protects that public Register specialization." +"simdlib_type_matrix_subtract_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for f64 protects that public Register specialization." +"simdlib_type_matrix_subtract_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for i16 protects that public Register specialization." +"simdlib_type_matrix_subtract_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for i32 protects that public Register specialization." +"simdlib_type_matrix_subtract_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for i64 protects that public Register specialization." +"simdlib_type_matrix_subtract_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for i8 protects that public Register specialization." +"simdlib_type_matrix_subtract_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for u16 protects that public Register specialization." +"simdlib_type_matrix_subtract_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for u32 protects that public Register specialization." +"simdlib_type_matrix_subtract_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for u64 protects that public Register specialization." +"simdlib_type_matrix_subtract_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated subtract cell for u8 protects that public Register specialization." +"simdlib_type_matrix_zero_f32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for f32 protects that public Register specialization." +"simdlib_type_matrix_zero_f64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for f64 protects that public Register specialization." +"simdlib_type_matrix_zero_i16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for i16 protects that public Register specialization." +"simdlib_type_matrix_zero_i32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for i32 protects that public Register specialization." +"simdlib_type_matrix_zero_i64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for i64 protects that public Register specialization." +"simdlib_type_matrix_zero_i8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for i8 protects that public Register specialization." +"simdlib_type_matrix_zero_u16","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for u16 protects that public Register specialization." +"simdlib_type_matrix_zero_u32","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for u32 protects that public Register specialization." +"simdlib_type_matrix_zero_u64","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for u64 protects that public Register specialization." +"simdlib_type_matrix_zero_u8","tests/codegen/RegisterTypeMatrixCodegenFixture.h","SSE4.2/128; AVX2/128; AVX2/256","public abstraction parity","tests/codegen/RegisterTypeMatrixCodegenRaw.cpp matching public Api operation","common-type-matrix","RegisterCodegen.128Sse42; RegisterCodegen.128Avx2; RegisterCodegen.256Avx2","retain","The canonical isolated zero cell for u8 protects that public Register specialization." +"simdlib_method_flags_codegen_binary","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","tests/method_flags/codegen/MethodFlagsRaw.cpp equivalent raw compiler-attribute declaration","method-flags","MethodFlagsCodegen","retain","The SIMD_FLAGS declaration for binary must preserve the raw compiler-attribute ABI/code shape and its stack contract." +"simdlib_method_flags_codegen_flatten","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","tests/method_flags/codegen/MethodFlagsRaw.cpp equivalent raw compiler-attribute declaration","method-flags","MethodFlagsCodegen","retain","The SIMD_FLAGS declaration for flatten must preserve the raw compiler-attribute ABI/code shape and its stack contract." +"simdlib_method_flags_codegen_forceinline","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","tests/method_flags/codegen/MethodFlagsRaw.cpp equivalent raw compiler-attribute declaration","method-flags","MethodFlagsCodegen","retain","The SIMD_FLAGS declaration for forceinline must preserve the raw compiler-attribute ABI/code shape and its stack contract." +"simdlib_method_flags_codegen_load","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","tests/method_flags/codegen/MethodFlagsRaw.cpp equivalent raw compiler-attribute declaration","method-flags","MethodFlagsCodegen","retain","The SIMD_FLAGS declaration for load must preserve the raw compiler-attribute ABI/code shape and its stack contract." +"simdlib_method_flags_codegen_register_result","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","tests/method_flags/codegen/MethodFlagsRaw.cpp equivalent raw compiler-attribute declaration","method-flags","MethodFlagsCodegen","retain","The SIMD_FLAGS declaration for register_result must preserve the raw compiler-attribute ABI/code shape and its stack contract." +"simdlib_method_flags_codegen_scalar_result","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","tests/method_flags/codegen/MethodFlagsRaw.cpp equivalent raw compiler-attribute declaration","method-flags","MethodFlagsCodegen","retain","The SIMD_FLAGS declaration for scalar_result must preserve the raw compiler-attribute ABI/code shape and its stack contract." +"simdlib_method_flags_codegen_store","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","tests/method_flags/codegen/MethodFlagsRaw.cpp equivalent raw compiler-attribute declaration","method-flags","MethodFlagsCodegen","retain","The SIMD_FLAGS declaration for store must preserve the raw compiler-attribute ABI/code shape and its stack contract." +"simdlib_method_flags_codegen_ternary","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","tests/method_flags/codegen/MethodFlagsRaw.cpp equivalent raw compiler-attribute declaration","method-flags","MethodFlagsCodegen","retain","The SIMD_FLAGS declaration for ternary must preserve the raw compiler-attribute ABI/code shape and its stack contract." +"simdlib_method_flags_codegen_unary","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","tests/method_flags/codegen/MethodFlagsRaw.cpp equivalent raw compiler-attribute declaration","method-flags","MethodFlagsCodegen","retain","The SIMD_FLAGS declaration for unary must preserve the raw compiler-attribute ABI/code shape and its stack contract." +"simdlib_method_flags_flatten_leaf","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","paired legacy Flatten helper declaration","method-flags helper-call inspection","MethodFlagsCodegen","retain","The helper must disappear from the flatten caller; the validation rejects any remaining call." +"simdlib_method_flags_force_leaf","tests/method_flags/codegen/MethodFlagsFlagged.cpp","x64 configuration-probe builds; SSE4.2","compiler-attribute enforcement","paired legacy ForceInline helper declaration","method-flags helper-call inspection","MethodFlagsCodegen","retain","The helper must disappear from the forceinline caller; the validation rejects any remaining call." diff --git a/docs/RegisterImplementationMatrix.md b/docs/RegisterImplementationMatrix.md new file mode 100644 index 0000000..9a5f32c --- /dev/null +++ b/docs/RegisterImplementationMatrix.md @@ -0,0 +1,418 @@ +# Register Implementation Matrix + +This document makes the accepted design in `RegisterProposal.md` executable and +traceable. The proposal controls semantics; `ApiOperationMatrix.md` controls the +current backend availability matrix; and this matrix records the implemented +operation coverage. A disagreement is resolved by correcting the controlling +semantic or availability document before implementing the affected operation. + +The supported compiler, configuration, generated-code, ABI, and exception +boundaries are defined by `RegisterQualification.md`. + +## Contract identity + +| Field | Value | +| --- | --- | +| Register widths | 128-bit SSE4.2 and AVX2; 256-bit AVX2 | +| Element types | `int8_t`, `uint8_t`, `int16_t`, `uint16_t`, `int32_t`, `uint32_t`, `int64_t`, `uint64_t`, `float`, `double` | +| Existing language baseline | C++20 through `SimdLib::SimdLib` | +| Register language baseline | C++23 explicit object parameters through the opt-in `SimdLib::Register` target | + +### Portability requirements + +The cross-platform compiler boundary requires these C++20 portability rules: + +- `tests/Api128.tests.cpp` now passes the fixed-extent output of + `Api::transform_pack<1>()` as an explicit `std::span`. The count + also appears in the declared span extent and cannot be deduced portably through + an implicit `std::array` conversion. +- `Api::transform_pack()` compiles its native-word store only when the output can + contain a complete native word. This states the existing size invariant and + prevents GCC from diagnosing an unreachable eight-byte store into a smaller + result object. +- The transform tail tests retain their immediate canaries but give the backing + objects at least one full-register access of physical capacity. This prevents + GCC's inliner from diagnosing the unreachable full-register path against a + three-element allocation while preserving the logical one-element span. +- `SimdVector` default construction delegates unconditionally to the existing + constexpr `Api::setzero()` path. This preserves intrinsic runtime zeroing and + avoids assigning `{}` directly to GCC's native vector extension type. + +These portability rules do not change a public declaration. + +## Contract traceability + +| Contract | Accepted implementation requirement | Owning task | Required evidence | +| --- | --- | ---: | --- | +| Template identity | All new public templates, concepts, aliases, and examples use ``; only internal delegation uses `Api` | 3, 9 | Compile probes and public-source audit | +| Availability | `SIMDLIB_REGISTER_INTERFACE_AVAILABLE` is computed from the standard explicit-object feature macro or the documented MSVC 19.44 fallback and cannot be overridden | 1 | Positive and negative configuration probes | +| Build boundary | `SimdLib::SimdLib` remains C++20; `SimdLib::Register` requests C++23, requires Register availability, and selects `/std:c++latest` for Microsoft C++ | 1 | CMake consumer probes and generated command inspection | +| Reproducible toolchains | GCC and GNU-like Clang container environments are pinned, locally and CI reusable, aggregate failures reliably, and remain explicitly separate from native Windows ABI evidence | 2 | Dockerfile provenance, Compose/orchestrator comparison, clean/failing matrix demonstrations | +| Supported geometry | A specialization owns one complete 128-bit or 256-bit native register and has no logical active count | 3 | Availability, size, alignment, and lane-count assertions | +| Representation | Register and RegisterMask are aggregates with one public native vector member each; neither type has bases, metadata, allocation, proxies, or address-dependent state | 3 | Aggregate/layout traits and ABI inspection | +| Special members | Register and RegisterMask use implicit trivial copy/move construction, assignment, and destruction; their member initializers explicitly use the intrinsic-backed zero operation | 3, 4 | Type traits and zero-construction code generation | +| All-active invariant | Every lane participates in transfer, arithmetic, comparison, rearrangement, and reduction behavior | 4-9 | Distinctive highest-lane runtime and constexpr tests | +| Transfer extent | Element and byte loads/stores use fixed extents equal to `lane_count` or `byte_count`; partial and unsafe forms do not exist | 4 | Compile rejection, canaries, and sanitizers | +| Alignment | Aligned loads/stores require `byte_count` alignment and follow the existing SimdLib precondition configuration | 4, 10 | Checks-enabled failures and release code generation | +| Scalar operands | Arithmetic and bitwise operations initially accept only the same Register type; scalar use requires explicit `broadcast()` | 4, 6 | Compile rejection and broadcast code generation | +| Integer division | Because x86 has no packed integer divide instruction, the named `_ext128_div_{epi,epu}{8,16,32,64}` methods explicitly extract, divide, and reinsert every lane with constant-index intrinsics; the matching `_ext256_` methods divide two 128-bit halves and reassemble them without a fold helper, runtime selector, or addressable array | 6, 10 | Scalar-oracle correctness and register-only wrapper-versus-raw generated-code parity for every integer type and width | +| Native interoperation | Register and RegisterMask support explicit aggregate-brace initialization from one complete native value and expose their representation through the public `native` member; direct mask initialization requires canonical predicate lanes | 4, 5 | Aggregate/constructibility assertions and native-result ABI probes | +| Explicit object parameters | Active non-static members take the explicit object by value; compound assignment is intentionally disabled and its implementations remain preserved in source comments | 3-9 | Declaration audit, constraint rejection, and reassignment code-generation probes | +| Calling convention | Register-shaped members use the appropriate `SIMD_FLAGS(...)` boundary mode; consumer-defined non-inlined boundaries must opt in separately | 3, 10 | Vector/default convention wrapper-versus-raw mirrors | +| Mask invariant | Comparisons and mask operations produce all-zero/all-one predicate lanes; direct aggregate initialization has the same canonical-lane precondition | 5 | Constraint tests, predicate-bit tests, and documented aggregate precondition | +| Compact mask bits | `bits_type` is normalized from lane count, is `uint32_t` for initial widths, maps bit `i` to lane `i`, and clears unused bits | 5 | Static assertions and mask-pattern tests | +| Comparison semantics | Named comparisons reproduce the selected intrinsic, including signedness, NaNs, signed zero, ordered/unordered predicates, and lane bit patterns | 5 | Runtime, portable, emulated, and constexpr parity | +| Whole equality | `operator==` means all lanes compare equal; `operator!=` is its Boolean negation; relational operators are absent | 5 | Boolean and compile-rejection tests | +| Shift counts | Per-lane negative counts are invalid; logical overshifts zero, arithmetic overshifts sign-fill, and byte/whole-register shifts follow the proposal boundary table | 6 | Boundary, precondition, constexpr, and codegen tests | +| Immediate controls | Every `imm8` is constrained to `0..255`; logical element and byte shuffles require exactly one selector per output lane or byte, permit repeated selectors, and reject selectors outside the complete source register | 7, 8 | Compile-success/failure boundaries | +| Rearrangement order | `lower_half()`, unpacking, and shuffling use logical low-to-high lanes or bytes. The 256-bit logical element and byte shuffles may select from the complete source register across the 128-bit boundary; lane-group restrictions remain only on operations whose names or intrinsic contracts specify them | 8 | Independent lane and byte oracles, cross-half selectors, highest-position sentinels, and exact code-generation parity | +| Type-changing results | Public operations name the exact constrained namespace-level result alias and never expose a raw intrinsic result | 7 | Type assertions and unsupported-combination rejection | +| Conversion split | `bit_cast()` preserves bits; `convert()` changes numeric values; `widen_low()` explicitly consumes only low source lanes | 8 | Independent bit/numeric/lane-consumption tests | +| Zero overhead | No supported register-only wrapper expression or call boundary adds instructions, moves, spills, reloads, stack traffic, temporaries, return buffers, branches, or indirection relative to the identical raw baseline | 3, 10 | Mandatory exact-parity generated-code and ABI gates with provenance | +| MSVC `/GS` boundary | Register-only, reassignment, common non-modulus type-matrix, specialized, FMA, rearrangement, and ABI records retain strict wrapper-versus-raw gates. The sole comparator-accepted Release exception is the exact 128-bit `Register::from_array` cookie sequence; all remaining instructions in that record must match. Integer modulus remains isolated and exact except for the MSVC AVX2/256 scheduling diagnostic, where the same scalar lane operations are ordered differently after the aggregate operator boundary. The primary composition/memory record is also diagnostic on MSVC because store, transfer, mutation, and opaque-call paths intentionally retain `/GS` | 3, 10 | Nonoverlapping register-only, reassignment, composition/memory, common type-matrix, modulus type-matrix, specialized, isolated-FMA, rearrangement, and ABI records; one owning `RegisterCodegen.` validation; named diagnostic reasons; comparison result; provenance; and the `RegisterQualification.md` exception ledger | +| Compatibility | `Api` remains supported; collection transforms and compatibility-only operations do not migrate | 9, 11 | Final ledger audit and unchanged C++20 matrix | +| Public exposure | `SimdLib.h` conditionally includes `Register.h` when `SIMDLIB_REGISTER_INTERFACE_AVAILABLE` is nonzero; C++20 translation units retain the existing umbrella surface | 1, 11 | C++20 exclusion, C++23 umbrella, isolated-header, ODR, and external-consumer gates | + +## Explicit exclusions + +| Excluded surface | Classification | Reason | +| --- | --- | --- | +| Partial load/store or lane construction | Higher-level responsibility | Register has no inactive lanes or fill policy | +| Dynamic-extent `load_unsafe` | `Api` compatibility-only | Its precondition is unsuitable for the restrictive value type | +| Native-order `set` | `Api` compatibility-only | Public lane order is logical low-to-high | +| Implicit scalar broadcast | Excluded | Broadcast cost and intent remain explicit | +| Implicit native conversion or mutable native reference | Excluded | Native access is an explicit by-value boundary | +| Scalar mask construction or `from_bits()` | Excluded | Native aggregate interoperation stays explicit and scalar expansion policy remains deferred | +| Runtime `extract_slow` | Api-only slow path | Register deliberately exposes only compile-time lane access | +| Register-selector `shuffle(value, selector)` | Api-only native control | Register exposes portable logical and byte shuffles instead of the backend register signature | +| `expand` and `compress` | Compatibility-only | Result width, lane consumption, and saturation are ambiguous | +| Multi-register widening/narrowing | Separate future design | One Register operation produces one complete result Register | +| Scalar arithmetic overloads | Deferred additive API | Real call sites and code generation must first justify them | +| Compound assignment overloads | Excluded | Reassignment is equally expressive, while mutable wrapper references cause a redundant 32-byte stack-alignment frame for 256-bit values under MSVC 19.44 | +| `RegisterMask::from_bits()` | Deferred additive API | Scalar-to-vector expansion cost and demand are not established | +| 512-bit registers and AVX-512 predicate registers | Future extension | Initial storage and mask contract is limited to 128/256-bit vectors | +| Span transforms and `transform_pack` | Collection-owned | Iteration and tail policy remain outside Register | +| `FinishIntegerMagnitudeFromPairSums`, `TransformForMaxPosition`, `compare_each_element` | Internal-only | These remain backend or compatibility helpers | + +## Type-changing result matrix + +| Public alias | Exact result | Availability rule | +| --- | --- | --- | +| `multiply_add_adjacent_result_t` | Same signedness at twice the lane width through 64 bits; 64-bit lanes remain 64-bit | Alias and method exist only for backend-supported source types/widths | +| `byte_multiply_add_result_t` | `Register` | Supported signed/unsigned byte input combinations only | +| `sad_result_t` | `Register` | Backend-supported SAD combinations only | +| `multi_sad_result_t` | `Register` | Backend-supported multi-SAD combinations only | + +## Public operation migration matrix + +The disposition column records whether the preferred Register surface implements +the operation or intentionally leaves it in a compatibility or collection layer. + +| Current public `Api` operation | Register result | Disposition | +| --- | --- | --- | +| `load` | `Register::load(fixed_span)` | Implemented | +| `load_aligned` | `Register::load_aligned(fixed_span)` | Implemented | +| `load_unaligned` | Canonicalized to `Register::load(fixed_span)` | Implemented | +| `load_partial` | No Register operation | Compatibility | +| `load_unsafe` | No Register operation | Compatibility | +| Element `store` | `value.store(fixed_span)` | Implemented | +| `store_aligned` | `value.store_aligned(fixed_span)` | Implemented | +| `store_unaligned` | Canonicalized to `value.store(fixed_span)` | Implemented | +| Fixed-byte `store` | `value.store_bytes(fixed_byte_span)` | Implemented | +| Dynamic-byte `store` | No Register operation | Compatibility | +| Fixed-byte `load` | `Register::load_bytes(fixed_byte_span)` | Implemented | +| `construct(array)` | `Register::from_array(array)` | Implemented | +| `to_array` | `value.to_array()` | Implemented | +| `setzero` | Default construction and `Register::zero()` | Implemented | +| `set1` | `Register::broadcast(value)` | Implemented | +| `setr` | `Register::from_lanes(...)` | Implemented | +| `set`, `set_partial`, `setr_partial` | No Register operation | Compatibility | +| `add` | `lhs + rhs` | Implemented | +| `subtract` | `lhs - rhs` | Implemented | +| `multiply` | `lhs * rhs` | Implemented | +| `divide` | `lhs / rhs` | Implemented | +| `modulus` | `lhs % rhs` | Implemented | +| `negate` | `-value` | Implemented | +| `min` | `lhs.min(rhs)` | Implemented | +| `max` | `lhs.max(rhs)` | Implemented | +| `multiply_add` | `lhs.multiply_add(rhs, addend)` | Implemented | +| `widen` | `value.widen_low()` | Implemented | +| `absolute` | `value.absolute()` | Implemented | +| `sqrt` | `value.sqrt()` | Implemented | +| `magnitude` | `value.magnitude()` | Implemented | +| `magnitude_checked` | `value.magnitude_checked()` | Implemented | +| `normalize` | `value.normalize()` | Implemented | +| `avg` | `lhs.average(rhs)` | Implemented | +| `add_horizontal` | `lhs.horizontal_add(rhs)` | Implemented | +| `subtract_horizontal` | `lhs.horizontal_subtract(rhs)` | Implemented | +| `multiply_add_adjacent` | `lhs.multiply_add_adjacent(rhs)` with named result alias | Implemented | +| `multiply_add_unsigned_signed_bytes` | Same named member with byte-multiply-add result alias | Implemented | +| `sum_absolute_byte_differences` | Same named member with SAD result alias | Implemented | +| `multi_sum_absolute_byte_differences` | Same named immediate member with multi-SAD result alias | Implemented | +| `min_position` | `value.min_position()` | Implemented | +| `max_position` | `value.max_position()` | Implemented | +| `add_saturated` | `lhs.add_saturated(rhs)` | Implemented | +| `subtract_saturated` | `lhs.subtract_saturated(rhs)` | Implemented | +| `hadd_saturated` | `lhs.horizontal_add_saturated(rhs)` | Implemented | +| `hsubtract_saturated` | `lhs.horizontal_subtract_saturated(rhs)` | Implemented | +| `add_subtract` | `lhs.add_subtract(rhs)` | Implemented | +| `dot_product` | `lhs.dot_product(rhs)` | Implemented | +| `bitwise_and` | `lhs & rhs` | Implemented | +| `bitwise_or` | `lhs \| rhs` | Implemented | +| `bitwise_xor` | `lhs ^ rhs` | Implemented | +| `bitwise_not` | `~value` | Implemented | +| `bitwise_andnot` | `lhs.andnot(rhs)` with preserved polarity | Implemented | +| `select` | `mask.select(when_true, when_false)` | Implemented | +| `movemask` | `value.movemask()` with intrinsic-native granularity | Implemented | +| `movemask_slim` | `value.lane_sign_bits()` with one bit per lane | Implemented | +| `compare_equal`, `compare_greater`, `compare_greater_equal`, `compare_less`, `compare_less_equal` | Corresponding named comparison | Implemented | +| `cmp_eq_mask`, `cmp_gt_mask`, `cmp_ge_mask`, `cmp_lt_mask`, `cmp_le_mask` | No compact-mask Register counterpart | Compatibility | +| `cmp_eq_slim`, `cmp_gt_slim`, `cmp_ge_slim`, `cmp_lt_slim`, `cmp_le_slim` | Corresponding named comparison followed by `.bits()` | Implemented | +| Deprecated `cmp_eq`, `cmp_gt`, `cmp_ge`, `cmp_lt`, `cmp_le` | Corresponding explicitly named `cmp_*_mask` method | Compatibility | +| `expand`, `compress` | No Register operation | Compatibility | +| `extract` | `value.lane()` | Implemented | +| Runtime `extract_slow` | No Register operation | Explicit Api slow path | +| `lower_half` | `value.lower_half()` | Implemented | +| `insert` | `value.with_lane(lane)` | Implemented | +| `unpack_lo` | `lhs.unpack_low(rhs)` | Implemented | +| `unpack_hi` | `lhs.unpack_high(rhs)` | Implemented | +| `shuffle` | `value.shuffle()` | Implemented for every arithmetic element type at 128 and 256 bits | +| `Api::shuffle` | `value.shuffle_bytes()` | Implemented for every arithmetic element type at 128 and 256 bits; result retains its element type | +| Register-selector `shuffle(value, selector)` | No generic Register operation | Native Api control; Register exposes `shuffle_bytes()` | +| `shuffle_lo`; `shuffle_lo_slow` | `value.shuffle_low()` | Compile-time form implemented; scalar runtime control remains Api-only | +| `shuffle_hi`; `shuffle_hi_slow` | `value.shuffle_high()` | Compile-time form implemented; scalar runtime control remains Api-only | +| `blend`; register-mask `blend`; `blend_slow` | `lhs.blend(rhs)`; predicate selection uses `mask.select()` | Compile-time and native-register controls mapped; scalar runtime control remains Api-only | +| `shift_left` | `value << count` | Implemented | +| `shift_right` | `value.logical_shift_right(count)`; unsigned `operator>>` | Implemented | +| `shift_right_arithmetic` | Signed `value >> count` | Implemented | +| Runtime `shift_bytes_left_slow` | `value.shift_bytes_left_slow(count)` | Implemented for integral 128-bit registers | +| Compile-time `shift_bytes_left` | `value.shift_bytes_left()` | Implemented for integral 128- and 256-bit registers | +| Runtime `shift_bytes_right_slow` | `value.shift_bytes_right_slow(count)` | Implemented for integral 128-bit registers | +| Compile-time `shift_bytes_right` | `value.shift_bytes_right()` | Implemented for integral 128- and 256-bit registers | +| Runtime `shift_bits_left_slow` | `value.shift_bits_left_slow(count)` | Implemented for integral 128-bit registers | +| Compile-time `shift_bits_left` | `value.shift_bits_left()` | Implemented for integral 128-bit registers | +| Runtime `shift_bits_right_slow` | `value.shift_bits_right_slow(count)` | Implemented for integral 128-bit registers | +| Compile-time `shift_bits_right` | `value.shift_bits_right()` | Implemented for integral 128-bit registers | +| `bit_cast` | `value.bit_cast()` | Implemented | +| `convert_to_float` | `value.convert()` | Implemented | +| `convert_to_int` | `value.convert()` | Implemented | +| Explicit-target `convert` | `value.convert()` | Implemented | +| Inferred-target `convert` | No Register operation | Compatibility | +| `transform_pack` | No Register operation | Collection | +| Unary and binary span `transform` overloads | No Register operation | Collection | +| `TransformForMaxPosition` | No Register operation | Internal | +| `compare_each_element` | Internal comparison fallback only | Internal | + +### Inventory audit + +A Clang AST declaration audit of `include/SimdLib/Api.h` identifies 92 unique +public static-operation names after excluding compiler-generated lambda call +helpers. The six additional operations exposed through inherited +`using impl::...` declarations—`add`, `divide`, `max`, `min`, `multiply`, and +`subtract`—produce 98 unique public operation names. Every name is classified +above. Overloaded `load`, `store`, `extract`, `insert`, `shuffle`, +`shuffle_lo`, `shuffle_hi`, `blend`, `shift_bytes_*`, `shift_bits_*`, `convert`, and span +`transform` families are split whenever their Register dispositions differ. +The protected `TransformForMaxPosition` and `compare_each_element` helpers are +classified separately as internal operations. + +### Register evidence matrix + +The supported-cell oracle is executable rather than hand-maintained: +`tests/RegisterOperationMatrix.tests.cpp` instantiates all ten element types at +128 and 256 bits, compares every conditional `IRegister` concept against its +`IApi` counterpart, verifies every unconditional `IRegister` and `IRegisterMask` +declaration, and audits every source/target cell for `bit_cast`, `convert`, and +`widen_low`. A supported cell is therefore exactly a cell accepted by that +compile-time audit; no prose-only availability list can drift independently. + +| Public family | Runtime semantics | Constexpr semantics | Constraints and exclusions | Generated code | ABI | +| --- | --- | --- | --- | --- | --- | +| Construction, observation, and full-width transfer | [`Register.tests.cpp`](../tests/Register.tests.cpp) | [`RegisterConstexpr.tests.cpp`](../tests/constexpr/RegisterConstexpr.tests.cpp) | [`RegisterOperationMatrix.tests.cpp`](../tests/RegisterOperationMatrix.tests.cpp), [`RegisterDynamicTransfer.cpp`](../tests/compile_fail/register/RegisterDynamicTransfer.cpp), and the lane-list/native/scalar/uninitialized probes in [`tests/compile_fail/register`](../tests/compile_fail/register) | [`RegisterCodegenFixture.h`](../tests/codegen/RegisterCodegenFixture.h) and [`RegisterTypeMatrixCodegenFixture.h`](../tests/codegen/RegisterTypeMatrixCodegenFixture.h) | [`RegisterAbi.cpp`](../tests/codegen/RegisterAbi.cpp), [`RegisterAbiRaw.cpp`](../tests/codegen/RegisterAbiRaw.cpp), [`RegisterDefaultAbi.cpp`](../tests/codegen/RegisterDefaultAbi.cpp), and [`RegisterDefaultAbiRaw.cpp`](../tests/codegen/RegisterDefaultAbiRaw.cpp) | +| RegisterMask, comparisons, reductions, and predicate selection | [`Register.tests.cpp`](../tests/Register.tests.cpp) | [`RegisterConstexpr.tests.cpp`](../tests/constexpr/RegisterConstexpr.tests.cpp) | [`RegisterOperationMatrix.tests.cpp`](../tests/RegisterOperationMatrix.tests.cpp) | [`RegisterCodegenFixture.h`](../tests/codegen/RegisterCodegenFixture.h) and [`RegisterTypeMatrixCodegenFixture.h`](../tests/codegen/RegisterTypeMatrixCodegenFixture.h) | Register and mask signatures in the paired ABI fixtures above | +| Basic arithmetic, bitwise operations, compact masks, and shifts | [`RegisterBasicOperations.tests.cpp`](../tests/RegisterBasicOperations.tests.cpp) and [`RegisterPreconditionFailure.tests.cpp`](../tests/RegisterPreconditionFailure.tests.cpp) | [`RegisterConstexpr.tests.cpp`](../tests/constexpr/RegisterConstexpr.tests.cpp) for the Api-constexpr subset | [`RegisterOperationMatrix.tests.cpp`](../tests/RegisterOperationMatrix.tests.cpp) and [`RegisterPreconditionFailure.tests.cpp`](../tests/RegisterPreconditionFailure.tests.cpp) | [`RegisterCodegenFixture.h`](../tests/codegen/RegisterCodegenFixture.h) and [`RegisterTypeMatrixCodegenFixture.h`](../tests/codegen/RegisterTypeMatrixCodegenFixture.h) | Paired Register/native unary, binary, scalar-result, and mutating-signature ABI fixtures above | +| Specialized arithmetic and reductions | [`RegisterSpecializedOperations.tests.cpp`](../tests/RegisterSpecializedOperations.tests.cpp) | Not a constant-evaluated `Api` surface unless a method is separately covered by the constexpr fixture | [`RegisterOperationMatrix.tests.cpp`](../tests/RegisterOperationMatrix.tests.cpp) | [`RegisterSpecializedCodegenFixture.h`](../tests/codegen/RegisterSpecializedCodegenFixture.h) | Type-changing and scalar-result signatures in the paired ABI fixtures above | +| Rearrangement, immediate controls, and lower-half extraction | [`RegisterRearrangementConversion.tests.cpp`](../tests/RegisterRearrangementConversion.tests.cpp) and [`LogicalShuffleRegister.tests.cpp`](../tests/LogicalShuffleRegister.tests.cpp) | [`RegisterConstexpr.tests.cpp`](../tests/constexpr/RegisterConstexpr.tests.cpp) | [`RegisterOperationMatrix.tests.cpp`](../tests/RegisterOperationMatrix.tests.cpp), [`RegisterInvalidByteShuffleSelector.cpp`](../tests/compile_fail/register/RegisterInvalidByteShuffleSelector.cpp), [`RegisterWrongByteShuffleSelectorCount.cpp`](../tests/compile_fail/register/RegisterWrongByteShuffleSelectorCount.cpp), and the other selector/immediate/compatibility probes in [`tests/compile_fail/register`](../tests/compile_fail/register) | [`RegisterRearrangementCodegenFixture.h`](../tests/codegen/RegisterRearrangementCodegenFixture.h) | Register/native return signatures in the paired ABI fixtures above | +| Bit reinterpretation, numeric conversion, and explicit low-lane widening | [`RegisterRearrangementConversion.tests.cpp`](../tests/RegisterRearrangementConversion.tests.cpp) | [`RegisterConstexpr.tests.cpp`](../tests/constexpr/RegisterConstexpr.tests.cpp) | All source/target cells in [`RegisterOperationMatrix.tests.cpp`](../tests/RegisterOperationMatrix.tests.cpp), plus unsupported-target and unavailable-width probes in [`tests/compile_fail/register`](../tests/compile_fail/register) | [`RegisterRearrangementCodegenFixture.h`](../tests/codegen/RegisterRearrangementCodegenFixture.h) | Type-changing Register/native return signatures in the paired ABI fixtures above | +| Compatibility-only partial, unsafe, scalar, native-order, runtime-selector, inferred-target, generic-selector, and collection operations | Not part of Register | Not part of Register | Dedicated compile-failure probes in [`tests/compile_fail/register`](../tests/compile_fail/register), including [`RegisterCollectionOperations.cpp`](../tests/compile_fail/register/RegisterCollectionOperations.cpp) | Not part of Register | Not part of Register | + +### Public-surface invariants + +- `Register` and `RegisterMask` are constrained at the class + boundary by `RegisterAvailable`. Operations available for every + valid specialization inherit that constraint; conditional operations add an + `IApi` concept or an immediate/index/width constraint before the body. +- Register-facing traits, concepts, aliases, examples, diagnostics, and result + types use `` order. Only internal delegation uses `Api`. +- Public operation results are Register, RegisterMask, or documented scalar + types. Register has no base class, inherited backend members, public + implementation selector, or public `SimdLib::Detail` dependency. +- All ordinary operations consume every active input lane. `widen_low()` names + and documents its consumed source prefix; `lower_half()` explicitly names its + lower-half result; sparse integer magnitude layouts document every defined + result lane and still consume every input lane. +- Every production class and active method in `Register.h`, `RegisterMask.h`, + and `RegisterFwd.h` has a Doxygen contract. Conditional methods document + availability, selectors and lane-moving methods document logical order, and + preconditioned methods document their valid domains. +- Partial and dynamic transfer, implicit scalar/native construction, + native-order construction, runtime extraction, generic implementation + selectors, inferred conversion targets, and collection algorithms are + rejected by the registered compile-failure sources under + `tests/compile_fail/register`. + +## Precondition and selector matrix + +| Surface | Contract | Failure evidence | +| --- | --- | --- | +| Full element transfer | Fixed extent equals `lane_count` | Compile rejection | +| Raw-byte transfer | Fixed extent equals `byte_count` | Compile rejection and canaries | +| Aligned transfer | Address is aligned to `byte_count` | Checks-enabled negative test | +| Lane access/replacement | `index < lane_count` | Constraint rejection | +| Logical shuffle | Exactly one selector per output lane; repeated selectors permitted; every selector names a lane in the complete source register; no zero-fill sentinel | Count/range constraint rejection and positive cross-half coverage | +| Logical byte shuffle | Exactly `byte_count` selectors; repeated selectors permitted; every selector is less than `byte_count`; no zero-fill sentinel | Count/range constraint rejection and positive cross-half coverage | +| Immediate operations | `0 <= imm8 <= 255` | Constraint rejection at `-1` and `256` | +| Per-lane logical/left shift | Runtime count is nonnegative; count at least lane width yields zero | Negative precondition and boundary tests | +| Per-lane arithmetic shift | Runtime count is nonnegative; oversized count clamps to `lane_width - 1` | Negative precondition and sign-fill tests | +| 128-bit byte shift | Count at most zero is identity; count at least 16 is zero | Runtime and constexpr boundaries | +| Runtime 128-bit whole-register shift | Count at most zero is identity; count at least 128 is zero | Runtime and constexpr boundaries | +| Compile-time whole-register shift | Negative rejected; count at least 128 is zero | Compile rejection and constexpr test | +| Unsupported operation/type/width | Removed from overload resolution | Requires-expression and compile-failure probes | + +## Compiler and configuration matrix + +| Surface | Compiler | Architecture/configuration | Requirement | +| --- | --- | --- | --- | +| C++20 core | MSVC 19.44 | Windows x64; Debug and Release | Existing full public matrix remains supported | +| C++20 core | clang-cl 20.1.8 | Windows x64; Debug and Release | Existing full public matrix remains supported | +| C++20 core | Clang 22.1.8 | Linux x64; Debug and Release | Existing full public matrix remains supported | +| C++20 core | GCC 13.2 | Linux x64; Debug and Release | Existing full public matrix remains supported; Register unavailable | +| C++20 core sanitizer | Clang 22.1.8 | Linux x64 Debug, `-O1`, ASan/UBSan, frame pointers | No sanitizer diagnostics | +| Register | MSVC 19.44 | Windows x64, `/std:c++latest`; supported ISA profiles | SSE4.2 diagnostics and strict AVX2 gates; memory-writing fixtures retain `/GS` and the exact documented exception | +| Register | clang-cl 20.1.8 | Windows x64, C++23; supported ISA profiles | SSE4.2 diagnostics and strict AVX2 correctness, ABI, and generated-code gates | +| Register | Clang 22.1.8 | Linux x64, C++23; supported ISA profiles | SSE4.2 diagnostics and strict AVX2 correctness, ABI, and generated-code gates | +| Register | GCC 14 or newer | Linux x64, C++23; supported ISA profiles | SSE4.2 diagnostics and strict AVX2 correctness, ABI, and generated-code gates | + +Linux x64 GCC 13.2 remains the required unavailable-interface probe; it is not +a Register compiler. A Register compiler floor is lowered or expanded only after +the complete correctness, layout, ABI, and generated-code gates pass. + +## Test and evidence ownership + +| Evidence family | Source owner | CMake/CTest owner | +| --- | --- | --- | +| Runtime Register correctness | `tests/Register.tests.cpp` | `RegisterSse42Tests`, `RegisterAvx2Tests` | +| Runtime mask/comparison correctness | `tests/Register.tests.cpp` | `RegisterSse42Tests`, `RegisterAvx2Tests` | +| Complete public-surface and availability audit | `tests/RegisterOperationMatrix.tests.cpp` | `RegisterSse42Tests`, `RegisterAvx2Tests` | +| Shared independent scalar oracles | Focused helpers in each Register runtime test source | Included only by public Register tests | +| Constexpr contracts | `tests/constexpr/RegisterConstexpr.tests.cpp` | `RegisterConstexpr128Probe`, `RegisterConstexpr256Probe` | +| Availability and language modes | `tests/availability/Register*.cpp` | Compile-only Register availability targets | +| Configuration fallback/exclusion | `tests/config/Register*.cpp` | Compile-only Register configuration targets | +| First-and-only headers | `tests/headers/RegisterHeaderProbe.cpp`, `tests/headers/RegisterMaskHeaderProbe.cpp`, and `tests/headers/SimdLibRegisterHeaderProbe.cpp` | `HeaderRegisterProbe`, `HeaderRegisterMaskProbe`, `HeaderSimdLibRegisterProbe` | +| Invalid declarations | `tests/compile_fail/register/*.cpp` | CMake `try_compile`/CTest compile-failure driver | +| ODR and multi-TU use | `tests/register_odr/main.cpp`, `tests/register_odr/second_translation_unit.cpp` | `RegisterOdr` | +| External consumer | `tests/consumer/register.cpp` and consumer CMake target | Existing consumer CTest project linked through `SimdLib::Register` | +| Forced-inline code generation | `tests/codegen/RegisterCodegen.cpp` and `RegisterCodegenFixture.h` | `RegisterCodegen` plus compiler-specific extraction scripts | +| Raw code-generation baselines | `tests/codegen/RegisterCodegenRaw.cpp` and `RegisterCodegenFixture.h` | Paired with `RegisterCodegen` under identical flags | +| Non-inlined ABI mirrors | `tests/codegen/RegisterAbi.cpp`, `tests/codegen/RegisterAbiRaw.cpp` | ABI records owned by `RegisterCodegen128Sse42`, `RegisterCodegen128Avx2`, and `RegisterCodegen256Avx2` | +| Register pressure and opaque calls | `tests/codegen/RegisterCodegenFixture.h` | Register code-generation gate | +| Code-generation comparison | `cmake/CompareRegisterCodegen.cmake` and checked-in allowlisted normalization rules | CTest mandatory performance gate | +| Permanent generated-code ownership audit | `docs/RegisterCodegenSymbolAudit.csv` and `docs/RegisterCodegenAudit.md` | Per-symbol fixture, baseline, record, validation, and retention traceability | +| Checks-enabled preconditions | `tests/RegisterPreconditionFailure.tests.cpp` | Existing precondition death-test infrastructure | +| Sanitizers | Runtime Register and mask sources | Fresh Clang ASan/UBSan configuration | +| Supplemental benchmarks | `benchmarks/Register.benchmarks.cpp` | `Benchmarks`; never a correctness/codegen substitute | +| Per-run evidence | Generated build receipts, JUnit reports, provenance files, and logs | Runtime artifacts rather than enduring documentation | + +Every production class and method has Doxygen documentation. Test +and generated-code sources use only public SimdLib declarations except the +proposal-approved narrow internal comparison adapter tests. + +## Validation ownership + +Compiler commands and result files are runtime artifacts rather than durable +documentation. CMake presets, CI workflows, and `ContainerValidation.md` own +the reproducible invocation contract; generated build trees, JUnit reports, +provenance files, and logs own individual outcomes. + +## Language and build-integration design + +This work introduces only the language boundary. `Register.h` deliberately +contains no Register or RegisterMask declaration until the representation work +begins. It also remains absent from `SimdLib.h`. + +| Requirement | Contract | +| --- | --- | +| Computed availability | `Config.h` computes `SIMDLIB_REGISTER_INTERFACE_AVAILABLE` from `__cpp_explicit_this_parameter >= 202110L`, or from non-clang Microsoft C++ 19.44 with `_MSVC_LANG > 202002L` | +| Non-overridable result | Defining the availability macro is rejected with `SIMDLIB_REGISTER_INTERFACE_AVAILABILITY_IS_COMPUTED` | +| Requirement signal | `SIMDLIB_REQUIRE_REGISTER_INTERFACE` defaults to zero and diagnoses unavailable required use without changing availability | +| Core target | `SimdLib::SimdLib` retains only `cxx_std_20` | +| Opt-in target | `SimdLib::Register` links the core target, requests `cxx_std_23`, and publishes `SIMDLIB_REQUIRE_REGISTER_INTERFACE=1` | +| Microsoft language selection | Only Microsoft C++ receives `/std:c++latest`; clang-cl and GNU-like Clang use their CMake-selected C++23 modes | +| Focused header | Direct unsupported inclusion of `Register.h` emits `SIMDLIB_REGISTER_HEADER_REQUIRES_CXX23` | +| Positive syntax | The enabled probe compiles named, arithmetic, comparison, and reference-mutating explicit-object members using `SIMD_FLAGS(...)` | +| Reproducible negative probes | The compile-failure inputs and public headers are configure dependencies; every fresh or affected configuration reruns each `try_compile` and records its compiler output | +| External consumers | The core consumer explicitly remains C++20; the separate Register consumer receives C++23 only by linking `SimdLib::Register` | + +## Container-environment design + +The container environment uses Alpine Linux for both GNU-like compiler services. The complete +Release, feature-labelled, sanitizer, constexpr, configuration, header, +consumer, and C++23 availability gates are required to remain on Alpine/musl. +A larger distribution is considered only after a concrete incompatibility is +recorded and the next-smallest maintained option is evaluated. + +| Environment | Immutable base | Toolchain contract | +| --- | --- | --- | +| `gcc14` | Alpine 3.22.5 manifest digest `sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce` | GCC 14.2.0, CMake 4.4.0, Ninja 1.12.1, musl 1.2.5 | +| `clang22` | Alpine 3.24.1 manifest digest `sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b` | Clang 22.1.3, libc++, LLD, compiler-rt, libunwind, binutils 2.45.1, CMake 4.4.0, Ninja 1.13.2, musl 1.2.6 | + +Both multi-stage Dockerfiles verify the CMake 4.4.0 source checksum and bake in +the peeled Catch2 v3.8.1 commit. Exact runtime package versions are pinned. The +BuildKit Dockerfile frontend is pinned to the digest used by the no-cache proof. +The containers run as a non-root user with a read-only root and source mount, +dropped capabilities, an executable temporary filesystem, and explicit +writable outputs. The Clang image intentionally omits the GCC compiler after +the CMake builder stage. A build operation configures each fingerprint-owned +tree once and CI applies CMake's fresh-toolchain behavior during that configure +step. Test and benchmark-execution operations validate the completed manifest +and never configure, clear, or rebuild the tree. + +The exhaustive build and test operations collectively cover the complete +Linux-supported C++20/C++23 suite, not a platform-independent subset. Portable +header repairs guard the Windows-only `` boundary, include x86 +intrinsics only on x86, use an empty vectorcall adapter for GNU-like Linux Clang, and +value-initialize the temporary used by `register_set_constexpr`. Native Windows jobs +remain authoritative for MSVC, clang-cl, Windows ABI, and calling-convention +evidence. + +### Compose and orchestration decision + +`compose.yml` is the single declarative environment used locally and in CI. It +uses a shared service anchor and one compiler-service profile. +`tools/Run-ContainerMatrix.ps1` owns the build-cell matrix: it builds selected +images once, starts compiler cells with bounded concurrency, waits for every +exit, retains separate logs, and removes only that invocation's unique Compose +project. + +A separate Compose healthcheck is intentionally absent: these are one-shot +`compose run` jobs, for which Compose does not wait on the service's own health +state. The canonical entrypoint instead performs synchronous compiler, CMake, +CPU-feature, and argument preflight before any configure or test work. +Generated-code comparisons are ordinary artifacts of each owning build cell. + +Direct parallel `docker compose up` interleaves logs, retains stopped service +containers, and cannot provide deterministic all-service failure attribution. +Direct `compose run` provides isolation but requires repeated arguments. The +wrapper therefore remains the smallest interface satisfying aggregate status, +cancellation, and artifact requirements while Compose remains the environment +definition. Its intentional-failure and cancellation switches exercise +one-service failure, multi-service failure, partial-log retention, and +unique-project cleanup. + +The scheduled reproducibility workflow runs the environment inspection action +with Docker caching disabled and records image inspection output. Normal CI +first builds every Linux cell and then runs a compile-free test operation +through the same wrapper and Dockerfiles. There is no CI-only Linux dependency +installation path. Exact local commands, artifact conventions, +refresh/security procedure, and project-owned cleanup are recorded in +`ContainerValidation.md`. diff --git a/docs/RegisterProposal.md b/docs/RegisterProposal.md new file mode 100644 index 0000000..9e5217b --- /dev/null +++ b/docs/RegisterProposal.md @@ -0,0 +1,1521 @@ +# Register Class Proposal + +Status: implemented and qualified public interface. Supported cells and explicit +exceptions are controlled by `RegisterQualification.md`. + +## Summary + +`SimdLib::Register` is the recommended value-like +interface for operations on one complete SIMD register when the translation +unit supports the required C++23 explicit-object feature. Unlike +`SimdVector`, a `Register` has no logical element +count that can be smaller than its hardware lane count. Every lane always +participates in loads, stores, arithmetic, comparisons, rearrangements, and +reductions. + +`Register` composes the existing `Api` facade instead +of inheriting from it. Operations delegate to that supported surface, including +the native-predicate comparison and selection operations added for Register. +This preserves the established implementation and feature-routing behavior +without a redundant Register backend, while presenting an interface that +supports natural expression chaining and keeps `Detail` types out of ordinary +call sites. + +The existing `Api` remains the C++20 interface and a supported compatibility and +backend-facing surface alongside `Register`. Span-wide +algorithms and partial-register handling remain outside `Register`. + +## Decision status + +| Status | Decisions | +| --- | --- | +| Controlling requirement | Template order is ``; every hardware lane is active; default construction uses the native zero-register operation; comparison behavior matches the selected hardware intrinsic; the abstraction has zero runtime overhead in supported configurations. | +| Implemented public design | Explicit register width with `NativeRegister` for target-selected width; C++23 explicit-object members for register-consuming operations; explicit scalar broadcast; `RegisterMask` predicates; fixed-extent element and byte transfers; operation names and results defined by the migration ledger. | +| Intentionally excluded | Partial and unsafe loads, automatic lane filling, collection transforms, native-order construction, ambiguous `expand`/`compress`, implementation-specific runtime rearrangements, and multi-register widening results. | +| Qualification contract | The supported compiler, ISA, type, width, generated-code, and non-inlined calling-boundary cells are defined in `docs/RegisterQualification.md`; individual outcomes are emitted as build receipts, reports, provenance files, and logs. | + +## Motivation + +The current `Api` facade exposes register operations as static functions: + +```cpp +using FloatApi = SimdLib::NativeApi; + +const auto scale = FloatApi::set1(0.02F); +const auto offset = FloatApi::set1(64.0F); +const auto input = FloatApi::load(source); +const auto output = FloatApi::add(FloatApi::multiply(input, scale), offset); +FloatApi::store(output, destination); +``` + +This is precise but verbose. Intermediate values have compiler intrinsic types, +so the type carrying the element and register-width contract is separate from +the value being manipulated. `SimdVector` provides a friendlier value-like +interface, but it also owns a logical element-count contract and must preserve +zero-filled inactive lanes. That behavior is valuable for fixed logical +vectors, but it is unnecessary and sometimes actively undesirable in +register-oriented code. + +The implemented interface keeps the low-level, complete-register semantics while +making the value itself carry the contract: + +```cpp +using FloatRegister = SimdLib::NativeRegister; + +const auto scale = FloatRegister::broadcast(0.02F); +const auto offset = FloatRegister::broadcast(64.0F); +const auto output = FloatRegister::load(source) * scale + offset; +output.store(destination); +``` + +## Goals + +- Represent exactly one 128-bit or 256-bit SIMD register. +- Treat every hardware lane as active at all times. +- Provide a value-like, chainable interface over the operations currently + curated by `Api`. +- Preserve the existing type, width, feature, fallback, and `constexpr` + contracts wherever the corresponding `Api` operation already defines them. +- Remain a zero-overhead abstraction with one native register data member, no + allocation, and no runtime metadata. +- Make broadcasts, native-register interoperation, numeric conversion, and bit + reinterpretation explicit. +- Constrain unsupported operations away instead of accepting a call that fails + inside an implementation body. +- Establish a credible migration path that does not prematurely remove or + deprecate `Api`. + +## Non-goals + +- Representing a logical vector whose element count is smaller than a hardware + register. +- Loading, storing, or constructing partial registers. +- Automatically filling lanes with zero, one, or any other neutral value. +- Iterating across arbitrary spans or handling a final partial batch. +- Owning dynamic storage, shapes, strides, or multidimensional data. +- Replacing `SimdVector`, `SimdAlgo`, `SimdResample`, or the future `Tensor` + abstraction. +- Hiding whether a width-changing operation consumes or produces more than one + register. + +## Responsibility boundaries + +| Surface | Responsibility | Partial data | +| --- | --- | --- | +| `Register` | One complete hardware register | Rejected | +| `SimdVector` | One fixed logical value | Inactive lanes are managed by the type | +| `SimdAlgo` and future `Tensor` operations | Collections and batches | Tail policy belongs to the algorithm | +| `Api` | Compatibility facade and implementation routing | Existing behavior remains supported | + +`Register` deliberately has no equivalent to `Api::load_partial`, +`Api::set_partial`, or `Api::setr_partial`. A caller with fewer than +`lane_count` elements must use a higher-level abstraction or explicitly stage +a complete register with a fill policy chosen by that caller. + +### Replacement boundary + +Where `SIMDLIB_REGISTER_INTERFACE_AVAILABLE` is nonzero, `Register` supersedes +`Api` as the recommended interface for operations whose inputs and outputs are +one or more complete registers. Supersession means that applicable new +documentation, examples, and register-local call sites use `Register`; it does +not mean that every static member currently located on `Api` becomes a +`Register` member. C++20 consumers continue to use `Api`. + +`Api` retains these supported responsibilities: + +- Span-wide `transform` and `transform_pack` collection algorithms. +- Partial-register staging used internally to implement collection tails. +- `load_unsafe`, whose dynamic-extent precondition is unsuitable for the + restrictive `Register` interface. +- Native-order construction and implementation-specific overloads retained for + compatibility. +- Existing callers that have not yet migrated. + +The operation ledger below classifies every current public `Api` operation. An +operation marked compatibility-only is intentionally outside the preferred +`Register` surface and therefore does not block register-local replacement. + +## Language and interface availability + +The existing SimdLib target remains a C++20 interface. `Register` is an +optional C++23 public surface because its member-call syntax depends on explicit +object parameters. Availability is detected from the standardized feature-test +macro when the compiler advertises it, with a version-and-language-mode fallback +for Microsoft C++. MSVC has supported explicit object parameters since Visual +Studio 2022 version 17.2, but the tested MSVC 19.36, 19.38, and 19.44 toolsets do +not define `__cpp_explicit_this_parameter` even when `/std:c++latest` is active. +Syntax support alone is not the support contract: the initial Microsoft C++ +floor is the MSVC 19.44 toolset on which the complete declaration forms and +initial ABI probes have been validated: + +```cpp +#if defined(__cpp_explicit_this_parameter) && \ + __cpp_explicit_this_parameter >= 202110L +#define SIMDLIB_REGISTER_INTERFACE_AVAILABLE 1 +#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1944 && \ + defined(_MSVC_LANG) && _MSVC_LANG > 202002L +#define SIMDLIB_REGISTER_INTERFACE_AVAILABLE 1 +#else +#define SIMDLIB_REGISTER_INTERFACE_AVAILABLE 0 +#endif +``` + +`SIMDLIB_REGISTER_INTERFACE_AVAILABLE` means that the current translation unit +can parse and use the `Register` interface. It does not mean that every element +type and register width is available; `is_register_available_v` retains +that per-specialization responsibility. + +The MSVC fallback deliberately combines the compiler version with +`_MSVC_LANG`; neither value alone proves that the required syntax is enabled. +The `!defined(__clang__)` condition prevents clang-cl from entering the MSVC +fallback merely because it also defines `_MSC_VER`. clang-cl follows the +standard feature-test-macro path. The implementation must compile-probe every +explicit-object declaration form used by `Register` at the supported MSVC +floor. The floor may be lowered below 19.44 only after that toolset passes the +complete correctness, layout, ABI, and generated-code gates. Documented syntax +support is not sufficient by itself. + +`_HAS_CXX23` is not used. It is an internal Microsoft runtime-library mode macro, +becomes visible only after a Microsoft header defines it, and is not specific to +explicit object parameters. No Microsoft header should be required merely to +determine whether SimdLib can expose `Register`. + +The umbrella header includes +`Register.h` only when `SIMDLIB_REGISTER_INTERFACE_AVAILABLE` is nonzero. +Directly including `Register.h` without the required feature produces a focused +preprocessing diagnostic. C++20 consumers can therefore continue using every +existing SimdLib surface without enabling C++23, while a translation unit +compiled with a supporting C++23 compiler gains `Register`. + +No normalized SimdLib language-version macro is introduced. The standardized +explicit-object feature-test macro remains the primary capability check. The +MSVC fallback is a documented exception for a compiler that implements the +required syntax without defining that macro. The resulting availability macro +is computed by SimdLib and is never caller-overridable. The initial interface +provides no opt-out macro. `Config.h` must document this availability macro as +an exception to its current rule that all configuration macros are +caller-overridable. + +No namespace-scope `inline constexpr` availability variable is added. Its value +could differ between C++20 and C++23 translation units and create an avoidable +ODR hazard. The preprocessor macro is the only public availability query. + +### Supported compiler matrix + +The core target retains its existing C++20 compiler matrix. Register support is +a narrower, separately validated matrix: + +| Compiler family | Initial Register floor | Platform | Language mode | Availability path | +| --- | --- | --- | --- | --- | +| Microsoft C++ | MSVC 19.44 | Windows x64 | `/std:c++latest` | `_MSC_VER` and `_MSVC_LANG` fallback | +| clang-cl | 20 | Windows x64 | C++23 | Standard feature-test macro | +| Clang | 22 | Linux x64 | C++23 | Standard feature-test macro | +| GCC | 14 | Linux x64 | C++23 | Standard feature-test macro | + +Linux x64 GCC 13.2 remains in the core C++20 matrix and must compile the umbrella +header with `SIMDLIB_REGISTER_INTERFACE_AVAILABLE == 0`. A compiler is added to the +Register matrix only after all correctness and zero-overhead gates pass for the +supported architecture, ISA profile, type, and width combinations. + +### CMake opt-in target + +The base `SimdLib::SimdLib` target remains C++20. A separate +`SimdLib::Register` interface target links the base target, requests +`cxx_std_23`, and publishes `SIMDLIB_REQUIRE_REGISTER_INTERFACE=1`. The focused +header reports an error when that requirement is present but +`SIMDLIB_REGISTER_INTERFACE_AVAILABLE` is zero: + +```cmake +add_library(SimdLibRegister INTERFACE) +add_library(SimdLib::Register ALIAS SimdLibRegister) +target_link_libraries(SimdLibRegister INTERFACE SimdLib::SimdLib) +target_compile_features(SimdLibRegister INTERFACE cxx_std_23) +target_compile_options( + SimdLibRegister + INTERFACE $<$:/std:c++latest>) +target_compile_definitions( + SimdLibRegister + INTERFACE SIMDLIB_REQUIRE_REGISTER_INTERFACE=1) +``` + +```cpp +#if defined(SIMDLIB_REQUIRE_REGISTER_INTERFACE) && \ + !SIMDLIB_REGISTER_INTERFACE_AVAILABLE +#error "SimdLib::Register requires supported C++23 explicit object parameters." +#endif +``` + +The CMake target requests C++23 generally and explicitly selects +`/std:c++latest` for Microsoft C++, which is the language mode used to validate +the MSVC fallback. Configuration probes must inspect the generated compiler +command and `_MSVC_LANG` so a future CMake or compiler change cannot silently +select a mode that lacks the required explicit-object syntax. The target does +not define or override the computed availability result. A consumer that only +links `SimdLib::SimdLib` does not inherit a C++23 requirement. + +Translation units may use different language modes provided no C++20 unit names +or exchanges a `Register` type. All translation units that exchange `Register` +or `RegisterMask` values across a function boundary must use compatible ISA, +ABI-affecting `SIMD_FLAGS(...)` adapter configuration, compiler ABI, and SimdLib settings. + +## Type shape and specialization availability + +The primary template puts the element type first, matching +`SimdVector`, and keeps the register width explicit. This ordering is the +canonical SimdLib order for new value types. The existing +`Api` order is a legacy design mistake and must not +be copied into `Register` or its associated traits: + +```cpp +namespace SimdLib +{ + +/** + * @brief Reports whether a complete SIMD register is available for an element + * type and register width. + */ +template +inline constexpr bool is_register_available_v = + SimdLib::is_api_available_v; + +/** + * @brief Constrains a type and width to a supported complete SIMD register. + */ +template +concept RegisterAvailable = + is_register_available_v; + +/** + * @brief Owns one complete SIMD register whose lanes are all active. + * @tparam element_t Scalar interpretation of each register lane. + * @tparam register_width Width of the native register in bits. + */ +template + requires RegisterAvailable +class Register final; + +/** + * @brief Selects the widest register available for an element type. + * @tparam element_t Scalar interpretation of each register lane. + */ +template + requires RegisterAvailable +using NativeRegister = Register< + element_t, + is_register_available_v ? 256 : 128>; + +} // namespace SimdLib +``` + +The primary template should not default `register_width`. `NativeRegister` +makes target-selected width visible at the call site, while +`Register` remains suitable for stable storage, interfaces, and ABI +contracts. Consumers should avoid placing `NativeRegister` in an ABI that +must remain identical across different compiler feature configurations. + +`is_register_available_v` may delegate to the existing +`is_api_available_v` implementation, but that delegation is an +internal compatibility detail. All new Register-facing templates, concepts, +aliases, documentation, and examples use the `` order. + +## Complete-register invariant + +For `Register`: + +- `lane_count == Bits / (sizeof(T) * 8)`. +- `byte_count == Bits / 8`. +- The object contains exactly one `Api::vector_t` value. +- No active-lane count or active-lane mask is stored or computed. +- Every operation observes and produces all `lane_count` lanes. +- Whole-register equality and reductions include the highest lane. +- A full load requires a fixed-extent span of exactly `lane_count` elements. +- A full store writes exactly `lane_count` elements. +- Construction from lane values requires exactly `lane_count` arguments. +- Default construction invokes the appropriate `Api` or implementation + zero-register operation and produces a fully initialized intrinsic zero + register. + +Zero-initialized default construction gives `Register{}` ordinary value-type +semantics. It does not represent inactive-lane filling: every resulting zero +lane is active. The runtime path must use the native zero-register operation, +preferably through `api_type::setzero()`, so the compiler can emit the target's +ordinary register-zeroing instruction. A constant-evaluation path, when +required by the compiler representation, must produce the same all-zero bits. +There is no public or private uninitialized `Register` construction path. + +## Core interface sketch + +The following declaration-only sketch is internally complete for construction, +transfer, native interoperation, representative arithmetic, and comparison. +The operation ledger defines the remaining operation names. + +```cpp +/** + * @brief Stores one Boolean predicate for every lane in a complete register. + * @tparam element_t Scalar geometry associated with each predicate lane. + * @tparam bits Width of the associated register in bits. + */ +template + requires RegisterAvailable +class RegisterMask; + +/** + * @brief Owns one complete SIMD register whose lanes are all active. + * @tparam element_t Scalar interpretation of each register lane. + * @tparam bits Width of the native register in bits. + */ +template + requires RegisterAvailable +class Register final +{ + public: + using element_type = element_t; + using api_type = Api; + using native_type = typename api_type::vector_t; + using mask_type = RegisterMask; + + constexpr static inline std::size_t register_width = bits; + constexpr static inline std::size_t byte_count = api_type::byte_count; + constexpr static inline std::size_t lane_count = api_type::element_count; + + /** @brief Owns the complete native register value represented by this aggregate. */ + native_type native = api_type::setzero(); + + /** + * @brief Returns a register with every active lane set to zero. + * @return Fully initialized zero register. + */ + [[nodiscard]] static constexpr Register SIMD_FLAGS(Out, ForceInline) zero() noexcept; + + /** + * @brief Broadcasts one scalar value to every active lane. + * @param value Scalar value to broadcast. + * @return Register containing `value` in every lane. + */ + [[nodiscard]] static constexpr Register SIMD_FLAGS(Out, ForceInline) broadcast( + element_type value) noexcept; + + /** + * @brief Constructs a register from exactly one complete logical lane list. + * @param lanes Values in low-to-high logical lane order. + * @return Register containing all supplied lane values. + */ + template ... lane_types> + requires(sizeof...(lane_types) == lane_count) + [[nodiscard]] static constexpr Register SIMD_FLAGS(Out, ForceInline) from_lanes( + lane_types &&...lanes) noexcept; + + /** + * @brief Constructs a register from one complete fixed-size lane array. + * @param source Source containing every active lane in logical order. + * @return Register containing all source lane values. + */ + [[nodiscard]] static constexpr Register SIMD_FLAGS(Out, ForceInline) from_array( + const std::array &source) noexcept; + + /** + * @brief Loads a complete register from potentially unaligned storage. + * @param source Source containing exactly one register of elements. + * @return Register loaded from `source`. + */ + [[nodiscard]] static Register SIMD_FLAGS(Out, ForceInline) load( + std::span source) noexcept; + + /** + * @brief Loads a complete register from register-aligned storage. + * @param source Aligned source containing exactly one register of elements. + * @return Register loaded from `source`. + */ + [[nodiscard]] static Register SIMD_FLAGS(Out, ForceInline) load_aligned( + std::span source) noexcept; + + /** + * @brief Loads one complete register bit pattern from raw bytes. + * @param source Source containing exactly one register of bytes. + * @return Register containing the source bit pattern. + */ + [[nodiscard]] static Register SIMD_FLAGS(Out, ForceInline) load_bytes( + std::span source) noexcept; + + /** + * @brief Stores every active lane to potentially unaligned storage. + * @param value Register to store. + * @param destination Destination for exactly one register of elements. + */ + void SIMD_FLAGS(In, ForceInline) store( + this Register value, + std::span destination) noexcept; + + /** + * @brief Stores every active lane to register-aligned storage. + * @param value Register to store. + * @param destination Aligned destination for one complete register. + */ + void SIMD_FLAGS(In, ForceInline) store_aligned( + this Register value, + std::span destination) noexcept; + + /** + * @brief Stores the complete register bit pattern to raw bytes. + * @param value Register to store. + * @param destination Destination containing exactly one register of bytes. + */ + void SIMD_FLAGS(In, ForceInline) store_bytes( + this Register value, + std::span destination) noexcept; + + /** + * @brief Copies every active lane into a fixed-size array. + * @param value Register to copy. + * @return Array containing all lanes in low-to-high logical order. + */ + [[nodiscard]] constexpr + std::array SIMD_FLAGS(In, ForceInline) to_array( + this Register value) noexcept; + + /** + * @brief Returns one compile-time-selected lane. + * @tparam index Logical lane index. + * @param value Register containing the selected lane. + * @return Copy of the selected lane. + */ + template + requires(index < lane_count) + [[nodiscard]] constexpr element_type SIMD_FLAGS(In, ForceInline) lane( + this Register value) noexcept; + + /** + * @brief Returns the wrapped native register for intrinsic interoperation. + * @param value Register to unwrap. + * @return Complete native register value. + */ + [[nodiscard]] constexpr native_type SIMD_FLAGS(InOut, ForceInline) native( + this Register value) noexcept; + + /** + * @brief Adds corresponding lanes. + * @param lhs Left-hand register. + * @param rhs Right-hand register. + * @return Per-lane sum. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, ForceInline) operator+( + this Register lhs, + Register rhs) noexcept; + + /** + * @brief Subtracts corresponding lanes. + * @param lhs Left-hand register. + * @param rhs Right-hand register. + * @return Per-lane difference. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, ForceInline) operator-( + this Register lhs, + Register rhs) noexcept; + + /** + * @brief Multiplies corresponding lanes. + * @param lhs Left-hand register. + * @param rhs Right-hand register. + * @return Per-lane product. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, ForceInline) operator*( + this Register lhs, + Register rhs) noexcept; + + /** + * @brief Compares corresponding lanes for equality. + * @param lhs Left-hand register. + * @param rhs Right-hand register. + * @return Register-shaped lane predicate. + */ + [[nodiscard]] constexpr mask_type SIMD_FLAGS(InOut, ForceInline) compare_equal( + this Register lhs, + Register rhs) noexcept; + + /** + * @brief Tests whether every corresponding lane compares equal. + * @param lhs Left-hand register. + * @param rhs Right-hand register. + * @return `true` when all lanes compare equal. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, ForceInline) operator==( + this Register lhs, + Register rhs) noexcept; + + /** + * @brief Tests whether any corresponding lane compares unequal. + * @param lhs Left-hand register. + * @param rhs Right-hand register. + * @return `true` when at least one lane compares unequal. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, ForceInline) operator!=( + this Register lhs, + Register rhs) noexcept; + +}; +``` + +The wrapper must not expose an implicit conversion to `native_type`, an +implicit scalar-broadcast constructor, or mutable span conversions. Its public +`native` member is the explicit native-representation interoperation point; reading +it by value copies the native register, and assigning it replaces the representation. +A complete intrinsic result is wrapped explicitly +with aggregate-brace initialization, such as `Register{native_value}`. + +Explicit-object members preserve ordinary value-like syntax such as +`value.absolute()`, `value.store(output)`, and `mask.bits()`. The object argument +is nevertheless declared by value, so there is no implicit `this` pointer and a +surviving call can use the same vector calling convention as a by-value free +function. + +`Register::load()` and `value.store()` are the canonical potentially +unaligned operations; there are no redundant `load_unaligned()` or +`store_unaligned()` members. +`Register::load_bytes()` and `value.store_bytes(destination)` preserve the +complete register bit pattern without changing the `element_type` +interpretation. All transfer operations use fixed extents. `Register` +deliberately provides no dynamic-extent unsafe load. + +## Scalar operands + +Arithmetic and bitwise operators should initially accept only another +`Register` of the same type. A scalar operation requires an explicit broadcast: + +```cpp +const auto adjusted = values * FloatRegister::broadcast(scale) + + FloatRegister::broadcast(offset); +``` + +This is intentionally more restrictive than `SimdVector`. It makes broadcast +cost and intent visible, avoids overload ambiguities, and encourages callers to +hoist loop-invariant broadcasts. Named convenience overloads can be considered +later if benchmarks and real call sites demonstrate that they improve clarity +without hiding meaningful work. + +## Integer division + +x86 provides no packed integer division instruction for the supported lane widths. +Integral `operator/` therefore delegates to the named width-prefixed extension +suite `_ext{128,256}_div_{epi,epu}{8,16,32,64}`. Each extension body explicitly +names its width and signedness. The 128-bit extensions extract every lane with a +compile-time constant index, perform the corresponding scalar signed or unsigned +division, and insert each quotient through the matching intrinsic. The 256-bit +extensions divide their low and high halves through the corresponding 128-bit +extension, then reassemble those halves with intrinsic operations. Neither path +may use a fold-based unrolling helper, materialize a lane array, or use a runtime +lane selector. This path remains register-only even though register pressure may +require ordinary compiler spills. + +## Comparison and mask semantics + +A low-level register interface needs a register-shaped comparison result. +Returning only the current scalar `Api::mask_t` would force a register-to-scalar +transition even when the next operation is a lane selection. +Returning `Register` would allow arbitrary numeric registers to be +mistaken for valid predicates. + +Introduce `RegisterMask` in the same focused header. It is an aggregate +containing exactly one native register. Boolean mask operations require each +lane to be either all-zero or all-one. Consumers normally name it through +`Register::mask_type`, and comparisons and mask bitwise operations +produce canonical values. Direct native aggregate initialization is an +explicit unchecked interoperation boundary whose caller must supply canonical +predicate lanes. + +```cpp +/** + * @brief Stores one Boolean predicate for every lane in a complete register. + * @tparam element_t Scalar geometry associated with each predicate lane. + * @tparam bits Width of the associated register in bits. + */ +template + requires RegisterAvailable +class RegisterMask final +{ + public: + using register_type = Register; + using api_type = typename register_type::api_type; + using native_type = typename register_type::native_type; + using bits_type = std::conditional_t< + (register_type::lane_count <= 32), + std::uint32_t, + std::uint64_t>; + + constexpr static inline std::size_t register_width = bits; + constexpr static inline std::size_t lane_count = register_type::lane_count; + + /** + * @brief Owns the complete native predicate value represented by this aggregate. + * @pre Every logical lane is either all-zero or all-one when initialized directly. + */ + native_type native = api_type::setzero(); + + /** + * @brief Tests whether any predicate lane is set. + * @param value Predicate register to test. + * @return `true` when at least one lane is true. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, ForceInline) any( + this RegisterMask value) noexcept; + + /** + * @brief Tests whether every predicate lane is set. + * @param value Predicate register to test. + * @return `true` when every lane is true. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, ForceInline) all( + this RegisterMask value) noexcept; + + /** + * @brief Tests whether no predicate lane is set. + * @param value Predicate register to test. + * @return `true` when every lane is false. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, ForceInline) none( + this RegisterMask value) noexcept; + + /** + * @brief Returns one compact bit per logical predicate lane. + * @param value Predicate register to reduce. + * @return Bit `i` set exactly when lane `i` is true. + */ + [[nodiscard]] constexpr bits_type SIMD_FLAGS(In, ForceInline) bits( + this RegisterMask value) noexcept; + + /** + * @brief Returns the wrapped native predicate register for intrinsic + * interoperation. + * @param value Predicate register to unwrap. + * @return Complete native predicate register value. + */ + [[nodiscard]] constexpr native_type SIMD_FLAGS(InOut, ForceInline) native( + this RegisterMask value) noexcept; + + /** + * @brief Selects lanes from two registers according to a predicate. + * @param condition Predicate controlling each selected lane. + * @param when_true Values selected for true predicate lanes. + * @param when_false Values selected for false predicate lanes. + * @return Register containing the selected values. + */ + [[nodiscard]] register_type SIMD_FLAGS(InOut, ForceInline) select( + this RegisterMask condition, + register_type when_true, + register_type when_false) noexcept; + + /** + * @brief Computes the intersection of two predicate registers. + * @param lhs Left-hand predicate register. + * @param rhs Right-hand predicate register. + * @return Predicate that is true where both inputs are true. + */ + [[nodiscard]] constexpr RegisterMask SIMD_FLAGS(InOut, ForceInline) operator&( + this RegisterMask lhs, + RegisterMask rhs) noexcept; + + /** + * @brief Computes the union of two predicate registers. + * @param lhs Left-hand predicate register. + * @param rhs Right-hand predicate register. + * @return Predicate that is true where either input is true. + */ + [[nodiscard]] constexpr RegisterMask SIMD_FLAGS(InOut, ForceInline) operator|( + this RegisterMask lhs, + RegisterMask rhs) noexcept; + + /** + * @brief Computes the exclusive union of two predicate registers. + * @param lhs Left-hand predicate register. + * @param rhs Right-hand predicate register. + * @return Predicate that is true where exactly one input is true. + */ + [[nodiscard]] constexpr RegisterMask SIMD_FLAGS(InOut, ForceInline) operator^( + this RegisterMask lhs, + RegisterMask rhs) noexcept; + + /** + * @brief Inverts every predicate lane. + * @param value Predicate register to invert. + * @return Predicate containing the inverse of every input lane. + */ + [[nodiscard]] constexpr RegisterMask SIMD_FLAGS(InOut, ForceInline) operator~( + this RegisterMask value) noexcept; + + /* + * Disabled compound assignment operators: their convenience does not justify + * the mutable-reference API surface, and MSVC 19.44 emits a redundant 32-byte + * stack-alignment frame for 256-bit wrapper mutation through references. + * Prefer lhs = lhs & rhs, lhs = lhs | rhs, or lhs = lhs ^ rhs. + * + /// @brief Intersects this predicate with another predicate. + /// @param lhs Predicate register to update. + /// @param rhs Right-hand predicate register. + /// @return Reference to the updated predicate. + constexpr auto SIMD_FLAGS(In, ForceInline) operator&=( + this RegisterMask &lhs, + RegisterMask rhs) noexcept -> RegisterMask &; + + /// @brief Unites this predicate with another predicate. + /// @param lhs Predicate register to update. + /// @param rhs Right-hand predicate register. + /// @return Reference to the updated predicate. + constexpr auto SIMD_FLAGS(In, ForceInline) operator|=( + this RegisterMask &lhs, + RegisterMask rhs) noexcept -> RegisterMask &; + + /// @brief Exclusively combines this predicate with another predicate. + /// @param lhs Predicate register to update. + /// @param rhs Right-hand predicate register. + /// @return Reference to the updated predicate. + constexpr auto SIMD_FLAGS(In, ForceInline) operator^=( + this RegisterMask &lhs, + RegisterMask rhs) noexcept -> RegisterMask &; + */ +}; +``` + +The default member initializer invokes the same native zero-register operation +as `Register`, so value/default initialization creates an all-false mask. +`bits_type` is a normalized public unsigned type selected from `lane_count`; it +does not inherit the legacy backend `Api::mask_t` type. The initial 128-bit and +256-bit specializations have at most 32 lanes and therefore use +`std::uint32_t`. The 64-bit alternative keeps the alias well-defined if a +future supported width has between 33 and 64 lanes. +`mask.bits()` uses the element-granular movemask operation and guarantees that +bits at indices greater than or equal to `lane_count` are zero. +`mask.select(when_true, when_false)` chooses `when_true` for all-one predicate +lanes and `when_false` for all-zero predicate lanes. It delegates to +`Api::select`, whose runtime path uses the implementation layer's variable-blend +intrinsic and whose constant-evaluated path reproduces the same polarity with +register bitwise operations. + +`mask.native` is the public native-representation interoperation point. Reading it +by value copies the predicate register. Complete native predicates +can also be wrapped explicitly with `RegisterMask{native_predicate}`. That +aggregate initialization is unchecked: every logical lane must already be +all-zero or all-one. Comparisons and mask operators satisfy this precondition; +arbitrary native data does not. Numeric Registers and scalar bit fields still +cannot construct a mask, and there is no initial `from_bits()` factory. + +`RegisterMask` must not provide an implicit conversion to `bool`; control-flow +decisions must spell `mask.any()`, `mask.all()`, or `mask.none()`. + +### Direct comparison implementation + +The curated `Api` exposes native `compare_*` functions that return canonical +register-shaped predicates without reducing them. The legacy `cmp_*` functions +remain scalar-mask operations and reduce the corresponding native comparison +with `movemask`. + +The direct implementation returns complete native predicate registers for +equality, greater-than, and any other comparison supported by the selected +backend. Derived predicates such as greater-than-or-equal combine the resulting +`RegisterMask` values. Each comparison member wraps its canonical native result +with explicit aggregate-brace initialization. + +Portable and constant-evaluated comparison paths remain private `Api` +implementation methods and construct the same all-zero or all-one lane patterns +as the runtime intrinsic. `Register` wraps those native predicates directly in +`RegisterMask`, keeping one implementation of comparison semantics. + +`Register::operator==` and `operator!=` should follow conventional value-type +semantics and return a whole-register Boolean. Lane-wise comparisons use named +explicit-object members such as `lhs.compare_equal(rhs)`, +`lhs.compare_greater(rhs)`, and `lhs.compare_less(rhs)`. +Relational operators should not mean an implicit all-lanes reduction. + +Every comparison follows the semantics of the underlying hardware intrinsic +selected for that operation. The wrapper must not replace intrinsic behavior +with a different C++ interpretation. This includes floating-point ordered or +unordered behavior, NaN results, signed-zero behavior, signed versus unsigned +integer ordering, and the all-zero or all-one bit pattern produced for each +predicate lane. Where a portable, constant-evaluated, or emulated path is +needed, it must reproduce the selected runtime intrinsic's observable result. +The operation documentation must identify the intrinsic comparison predicate +whose semantics it exposes. + +## Operation surface + +The following ledger classifies every current public `Api` operation. Operation +availability continues to follow `docs/ApiOperationMatrix.md` and the selected +backend constraints. + +Every non-static operation uses a C++23 explicit object parameter and takes that +parameter by value, preserving ordinary member-call syntax without an implicit +`this` pointer. Compound assignment is intentionally absent: its convenience +does not justify a mutable-reference surface that causes MSVC 19.44 to emit a +redundant 32-byte stack-alignment frame for 256-bit wrapper mutation. Callers +use explicit reassignment such as `lhs = lhs + rhs`. All register-shaped parameters and results use the appropriate +`SIMD_FLAGS(...)` boundary mode. + +Aggregate initialization, implicit compiler-generated special members, and +static factories have no explicit object parameter. They are covered alongside +the explicit-object surface by generated-code and ABI tests. + +### Construction and transfer ledger + +| Current `Api` operation | Preferred `Register` form | Decision | +| --- | --- | --- | +| `load` | `Register::load(fixed_span)` | Canonical potentially unaligned full load | +| `load_aligned` | `Register::load_aligned(fixed_span)` | Retained with alignment precondition | +| `load_unaligned` | `Register::load(fixed_span)` | Redundant spelling omitted | +| `load_partial` | None | Partial data belongs to higher-level types | +| `load_unsafe` | None | Dynamic-extent unsafe load remains on `Api` | +| `store` to element span | `value.store(fixed_span)` | Canonical potentially unaligned full store | +| `store_aligned` | `value.store_aligned(fixed_span)` | Retained with alignment precondition | +| `store_unaligned` | `value.store(fixed_span)` | Redundant spelling omitted | +| `store` to fixed byte span | `value.store_bytes(fixed_byte_span)` | Renamed to make bit-pattern transfer explicit | +| `store` to dynamic byte span | None | Dynamic-extent transfer remains compatibility-only on `Api` | +| Fixed-byte `load` | `Register::load_bytes(fixed_byte_span)` | Symmetric bit-pattern transfer | +| `construct(array)` | `Register::from_array(array)` | Static factory; no ambiguous storage constructor | +| `to_array` | `value.to_array()` | Retained as a value conversion | +| `setzero` | Default construction and `Register::zero()` | Uses intrinsic-backed zero construction | +| `set1` | `Register::broadcast(value)` | Explicit scalar broadcast | +| `setr` | `Register::from_lanes(...)` | Requires exactly `lane_count` logical-order values | +| `set` | None | Native intrinsic argument order remains compatibility-only | +| `set_partial`, `setr_partial` | None | No partial or automatically filled lanes | + +### Arithmetic and reduction ledger + +| Current `Api` operation | Preferred `Register` form | Result | +| --- | --- | --- | +| `add` | `lhs + rhs` | Same register type | +| `subtract` | `lhs - rhs` | Same register type | +| `multiply` | `lhs * rhs` | Same register type | +| `divide` | `lhs / rhs` | Same register type where supported | +| `modulus` | `lhs % rhs` | Same integral register type | +| `negate` | `-value` | Same register type | +| `min` | `lhs.min(rhs)` | Same register type | +| `max` | `lhs.max(rhs)` | Same register type | +| `multiply_add` | `lhs.multiply_add(rhs, addend)` | Same register type | +| `widen` | `value.widen_low()` | Explicit target `Register`; consumed lanes documented | +| `absolute` | `value.absolute()` | Same register type and intrinsic edge behavior | +| `sqrt` | `value.sqrt()` | Same register type where supported | +| `magnitude` | `value.magnitude()` | Floating groups broadcast; integer groups store an unchecked result only in their leading lane | +| `magnitude_checked` | `value.magnitude_checked()` | Integral groups store a saturated result followed by a canonical overflow mask | +| `normalize` | `value.normalize()` | Same floating register type | +| `avg` | `lhs.average(rhs)` | Same register type | +| `add_horizontal` | `lhs.horizontal_add(rhs)` | Same register type | +| `subtract_horizontal` | `lhs.horizontal_subtract(rhs)` | Same register type | +| `multiply_add_adjacent` | `lhs.multiply_add_adjacent(rhs)` | Explicit operation-result Register alias | +| `multiply_add_unsigned_signed_bytes` | `lhs.multiply_add_unsigned_signed_bytes(rhs)` | Explicit signed promoted-result Register alias | +| `sum_absolute_byte_differences` | `lhs.sum_absolute_byte_differences(rhs)` | Explicit unsigned-result Register alias | +| `multi_sum_absolute_byte_differences` | `lhs.multi_sum_absolute_byte_differences(rhs)` | Explicit unsigned-result Register alias | +| `min_position` | `value.min_position()` | `std::size_t` | +| `max_position` | `value.max_position()` | `std::size_t` | +| `add_saturated` | `lhs.add_saturated(rhs)` | Same register type | +| `subtract_saturated` | `lhs.subtract_saturated(rhs)` | Same register type | +| `hadd_saturated` | `lhs.horizontal_add_saturated(rhs)` | Same register type | +| `hsubtract_saturated` | `lhs.horizontal_subtract_saturated(rhs)` | Same register type | +| `add_subtract` | `lhs.add_subtract(rhs)` | Same floating register type | +| `dot_product` | `lhs.dot_product(rhs)` | Same register type with intrinsic-selected output lanes | + +Operations whose intrinsic changes the lane type use constrained namespace-level +alias templates. Keeping these aliases outside `Register` avoids conditional +member declarations or helper-base storage that could complicate the exact +one-native-member representation: + +| Alias | Exact result mapping | +| --- | --- | +| `multiply_add_adjacent_result_t` | `Register` for `int8_t`, `Register` for `uint8_t`, then the corresponding signedness at twice the lane width through 64 bits; 64-bit lanes remain 64-bit | +| `byte_multiply_add_result_t` | `Register` for supported signed/unsigned byte inputs | +| `sad_result_t` | `Register` | +| `multi_sad_result_t` | `Register` | + +The aliases are declared only when the corresponding backend operation is +available. Each public operation names its exact alias as the return type rather +than using an undifferentiated `auto` or exposing a raw intrinsic type. Alias +availability and mapping are tested for every supported source type and width; +unsupported combinations remain absent even when a result element type could be +formed mechanically. + +### Bitwise and comparison ledger + +| Current `Api` operation | Preferred `Register` form | Result | +| --- | --- | --- | +| `bitwise_and` | `lhs & rhs` | Same register type | +| `bitwise_or` | `lhs \| rhs` | Same register type | +| `bitwise_xor` | `lhs ^ rhs` | Same register type | +| `bitwise_not` | `~value` | Same register type | +| `bitwise_andnot` | `lhs.andnot(rhs)` | Same register type with existing operand polarity | +| `select` | `mask.select(when_true, when_false)` | Same Register type; canonical predicate remains Register-shaped | +| `movemask` | `value.movemask()` | Scalar mask with the selected intrinsic's native granularity | +| `movemask_slim` | `value.lane_sign_bits()` | Scalar mask with one bit per lane | +| `compare_equal`, `compare_greater`, `compare_greater_equal`, `compare_less`, `compare_less_equal` | Corresponding named comparison | `RegisterMask` preserving native predicates | +| `cmp_eq_mask`, `cmp_gt_mask`, `cmp_ge_mask`, `cmp_lt_mask`, `cmp_le_mask` | No compact-mask Register counterpart | Byte-granular legacy-compatible scalar mask | +| `cmp_eq_slim`, `cmp_gt_slim`, `cmp_ge_slim`, `cmp_lt_slim`, `cmp_le_slim` | Corresponding named comparison followed by `.bits()` | One compact bit per lane | +| Deprecated `cmp_eq`, `cmp_gt`, `cmp_ge`, `cmp_lt`, `cmp_le` | Corresponding explicitly named `cmp_*_mask` method | Byte-granular compatibility spelling | + +The legacy scalar comparison-mask layout is not uniform across integral and +floating backends. `mask.bits()` deliberately normalizes it to one bit +per logical lane. Callers requiring the exact legacy scalar representation +continue to use the corresponding `Api::cmp_*` function. + +`Register::operator==` is equivalent to `lhs.compare_equal(rhs).all()`. +`operator!=` is equivalent to `lhs.compare_equal(rhs).all() == false`; this +preserves whole-value inequality and does not mean that every lane must differ. +For floating registers these operators retain the selected intrinsic's ordered +equality behavior: a NaN lane is not equal, while positive and negative zero are +equal. They are not bitwise-equality operators; exact bit-pattern comparison +requires an explicit integer reinterpretation followed by integer comparison. + +### Rearrangement ledger + +| Current `Api` operation | Preferred `Register` form | Decision | +| --- | --- | --- | +| `expand` | None | Ambiguous legacy widening alias remains compatibility-only | +| `compress` | None | Ambiguous legacy narrowing alias remains compatibility-only | +| `extract` | `value.lane()` | Compile-time logical lane extraction | +| Runtime `extract_slow` | None | Explicit Api slow path; Register retains compile-time lane access | +| `lower_half` | `value.lower_half()` | Returns `Register` from a 256-bit source | +| `insert` | `value.with_lane(lane)` | Compile-time logical lane replacement | +| `unpack_lo` | `lhs.unpack_low(rhs)` | Wrapped backend result | +| `unpack_hi` | `lhs.unpack_high(rhs)` | Wrapped backend result | +| `shuffle` | `value.shuffle()` | One compile-time logical source-lane selector per output lane | +| `Api::shuffle` | `value.shuffle_bytes()` | One compile-time logical source-byte selector per output byte; result retains `T` | +| Register-selector `shuffle(value, selector)` | None | Native Api runtime control; Register exposes portable logical and byte shuffle forms | +| `shuffle_lo`; `shuffle_lo_slow` | `value.shuffle_low()` | Compile-time immediate form; scalar runtime control remains Api-only | +| `shuffle_hi`; `shuffle_hi_slow` | `value.shuffle_high()` | Compile-time immediate form; scalar runtime control remains Api-only | +| `blend`; register-mask `blend`; `blend_slow` | `lhs.blend(rhs)` | Immediate blend maps directly; predicate selection uses `mask.select(lhs, rhs)`; scalar runtime control remains Api-only | + +Logical shuffle selectors use low-to-high lane numbering for the element type. +The selector count must equal the register lane count, repeated selectors are +permitted, and every selector must name a lane in the complete source register. +A 256-bit shuffle may therefore move a lane across the 128-bit boundary. +Floating-point lanes preserve their object representations, including NaN +payloads and signed zero. There is no out-of-range zero-fill sentinel; the +unsuffixed register-selector `Api::shuffle(value, selector)` overload retains +control-mask behavior defined by its native backend. + +Byte shuffle selectors view the complete register as `byte_count` bytes numbered +from low to high. The selector count must equal `byte_count`, repeated selectors +are permitted, and every selector must be less than `byte_count`. There is no +zero-fill sentinel. A 256-bit byte shuffle may move bytes across the 128-bit +boundary, and output bytes may cross the element boundaries of `T`; the result +nevertheless remains `Register`. + +### Shift and conversion ledger + +| Current `Api` operation | Preferred `Register` form | Result | +| --- | --- | --- | +| `shift_left` | `value << count` | Per-lane integral shift | +| `shift_right` | `value.logical_shift_right(count)` | Per-lane logical shift for signed or unsigned lanes | +| `shift_right_arithmetic` | `value >> count` | Per-lane arithmetic shift for signed lanes | +| Runtime `shift_bytes_left_slow` | `value.shift_bytes_left_slow(count)` | Complete integral 128-bit register byte shift | +| Compile-time `shift_bytes_left` | `value.shift_bytes_left()` | Complete integral 128- or 256-bit register byte shift | +| Runtime `shift_bytes_right_slow` | `value.shift_bytes_right_slow(count)` | Complete integral 128-bit register byte shift | +| Compile-time `shift_bytes_right` | `value.shift_bytes_right()` | Complete integral 128- or 256-bit register byte shift | +| Runtime `shift_bits_left_slow` | `value.shift_bits_left_slow(count)` | Complete integral 128-bit bit-string shift | +| Compile-time `shift_bits_left` | `value.shift_bits_left()` | Complete integral 128-bit bit-string shift | +| Runtime `shift_bits_right_slow` | `value.shift_bits_right_slow(count)` | Complete integral 128-bit bit-string shift | +| Compile-time shift_bits_right | alue.shift_bits_right() | Complete integral 128-bit bit-string shift | +| it_cast | alue.bit_cast() | Full-width bit-preserving reinterpretation | +| `convert_to_float` | `value.convert()` | `Register` from supported 32-bit integer lanes | +| `convert_to_int` | `value.convert()` | `Register` from float lanes | +| Explicit-target `convert` | `value.convert()` | Explicit target type | +| Inferred-target `convert` | None | Complementary-type inference remains compatibility-only on `Api` | + +`operator>>` is available only when it has one unambiguous hardware meaning. +Unsigned lanes use the logical shift. Signed lanes use the arithmetic shift. +`logical_shift_right()` remains available for signed lanes that intentionally +request zero fill. + +Shift-count behavior is part of the public contract and matches the existing +backend operation rather than C++ scalar-shift rules: + +| Shift family | Count contract | +| --- | --- | +| Per-lane left or logical right | Runtime count must be nonnegative; counts at least the lane width produce zero lanes | +| Per-lane arithmetic right | Runtime count must be nonnegative; counts at least the lane width clamp to `lane_width - 1` and therefore sign-fill | +| 128-bit byte shifts | Counts at most zero return the input; counts at least 16 return zero | +| Runtime 128-bit whole-register bit shifts | Counts at most zero return the input; counts at least 128 return zero | +| Compile-time 128-bit whole-register bit shifts | Negative counts are rejected; counts at least 128 produce zero | + +The implementation must not introduce release-only undefined behavior for a +documented count. Negative per-lane shift counts are invalid runtime inputs and +follow the SimdLib precondition policy; tests cover the boundary values `0`, +`width - 1`, `width`, and `width + 1`. + +### Collection and internal ledger + +| Current `Api` operation | `Register` decision | +| --- | --- | +| `transform_pack` | Remains a collection algorithm on `Api` or its future algorithm owner | +| Unary in-place `transform` | Remains a collection algorithm | +| Unary separate-output `transform` | Remains a collection algorithm | +| Binary `transform` | Remains a collection algorithm | +| `TransformForMaxPosition` | Internal helper; no public `Register` counterpart | +| `compare_each_element` | Internal fallback helper used by the comparison adapter | + +All preferred register-local operations return `Register`, `RegisterMask`, or +an explicitly documented scalar. No preferred operation exposes a raw intrinsic +result. Availability is expressed with `requires` clauses that mirror the +corresponding supported backend operation. + +## Conversion and width-changing operations + +Numeric conversion and bit reinterpretation are distinct operations: + +- `bit_cast()` preserves every register bit and requires a supported + target lane interpretation at the same register width. The target lane count + may differ because this operation reinterprets the complete bit pattern. +- `convert()` performs numeric conversion and is initially available + only where the existing API has a defined conversion into one complete + target register. +- `widen_low()` is the preferred spelling for the + existing `Api::widen` behavior. It explicitly converts only the lowest + source lanes needed to populate one complete target register. +- No `widen_all` member is included in the initial preferred surface. Producing + multiple registers is a separate algorithm contract rather than a value + operation on one result register. +- No generic narrowing or packing member is included until its + saturation/truncation policy and required source-register count have a + dedicated design. + +The existing generic `expand` and `compress` names remain supported only on +`Api`. They are not promoted to `Register`. Their lane consumption, result +type, signedness, and saturation behavior are too specialization-specific for +the preferred interface. `widen_low` makes discarded high source lanes +explicit; no other preferred operation may silently discard active lanes. + +## Rearrangement policy + +Compile-time selectors should be preferred when an instruction requires an +immediate. Examples include `value.shuffle()`, +`value.shuffle_bytes()`, `lhs.blend(rhs)`, `value.lane()`, and +`value.with_lane(lane_value)`. Runtime-selector overloads should exist +only where the current implementation supports them without misrepresenting an +immediate-only instruction as a cheap dynamic operation. + +Every `imm8` template control is constrained to the inclusive range `0..255`; +operation-specific unused bits retain the underlying intrinsic behavior. Lane +selectors require `index < lane_count`. Logical `shuffle` overloads +require exactly `lane_count` selectors and reject every index outside +`[0, lane_count)`. Logical `shuffle_bytes` overloads require exactly +`byte_count` selectors and reject every index outside `[0, byte_count)`. These +requirements participate in overload constraints instead of relying on a late +intrinsic diagnostic. + +Lane order at the public boundary is always logical low-to-high order. Native +intrinsic argument order remains available only through explicit native +interoperation or compatibility `Api` calls. + +## Layout and zero-overhead contract + +Each supported specialization should satisfy the following where the compiler +permits the corresponding type trait: + +```cpp +static_assert(sizeof(Register) == sizeof(__m128)); +static_assert(alignof(Register) == alignof(__m128)); +static_assert(std::is_standard_layout_v>); +static_assert(std::is_trivially_copyable_v>); +static_assert(std::is_trivially_copy_constructible_v>); +static_assert(std::is_trivially_move_constructible_v>); +static_assert(std::is_trivially_copy_assignable_v>); +static_assert(std::is_trivially_move_assignable_v>); +static_assert(std::is_trivially_destructible_v>); +static_assert(sizeof(RegisterMask) == sizeof(__m128)); +static_assert(alignof(RegisterMask) == alignof(__m128)); +static_assert(std::is_standard_layout_v>); +static_assert(std::is_trivially_copyable_v>); +static_assert(std::is_trivially_destructible_v>); +``` + +`Register` is required to have zero runtime performance overhead relative to +the equivalent supported `Api` or direct-intrinsic expression. This guarantee +applies to storage, alignment, argument passing, return values, construction, +loads, stores, arithmetic, comparisons, masks, selection, rearrangement, and +destruction. It is not limited to expressions that happen to be inlined. + +Zero overhead is a relative guarantee. Neither C++ nor raw SIMD intrinsics can +guarantee that a value never leaves a physical SIMD register. Finite register +capacity, register pressure, opaque calls, disabled optimization, diagnostic +instrumentation, or an explicit address escape can cause the compiler to spill +a native intrinsic value. Such a spill is not caused by `Register` when the +equivalent raw-intrinsic implementation spills in the same context. It is a +`Register` defect when the wrapper introduces a move, spill, reload, temporary, +or indirection that the equivalent raw implementation does not require. + +### Register-residency strategy + +The preferred implementation uses these mechanisms together: + +- Every `Register` and `RegisterMask` contains exactly one native vector and + remains trivially copyable and destructible. +- Small operations are defined in the focused header and use the `ForceInline` + modifier so an optimized chain becomes one vector expression in the + compiler's intermediate representation. +- Every non-mutating operation that consumes an existing wrapper is an + explicit-object member taking that object by value. It uses the appropriate + `SIMD_FLAGS(...)` boundary mode and returns register-shaped results by value. + This includes + named operations as well as overloaded operators. If a call survives + optimization, its operands and result can use the platform's vector or + homogeneous-vector-aggregate calling convention without an implicit `this` + pointer. +- Aggregate initialization, implicit compiler-generated special members, and + static factories consume no existing wrapper. Compound assignment operators + remain disabled; explicit reassignment composes the by-value binary + operations without adding a mutable-reference boundary. +- Deliberately out-of-line register operations, if any are later justified, + retain their explicit-object parameter and appropriate `SIMD_FLAGS(...)` + boundary mode so their ABI does not silently regress to an implicit `this` + boundary. +- No operation returns a mutable native reference, mutable span, proxy tied to + object storage, or other value that requires the wrapper to acquire a stable + memory address. + +The `In`, `Out`, and `InOut` boundary modes select the configured calling +convention for a surviving function call; they do not pin a value to a physical +register and have no effect after a function is inlined. The current adapter +emits `__vectorcall` for Microsoft C++ and clang-cl on x64 and is empty for GCC +and GNU-like Clang. The public aggregate representations of Register and +RegisterMask allow clang-cl to classify flagged vector-convention boundaries +like the corresponding native vector. The platform-default clang-cl convention remains a +separately recorded boundary and may use hidden return storage. GCC uses its +target ABI and is validated against the same raw-vector baseline. + +The calling convention on Register members does not propagate into an ordinary +consumer-defined function. A non-inlined consumer function that passes or +returns `Register` or `RegisterMask` must declare the appropriate +`SIMD_FLAGS(...)` boundary mode to participate in the vector-calling-convention +guarantee where that convention is supported: + +```cpp +using FloatRegister = SimdLib::Register; + +/** + * @brief Applies a consumer-defined complete-register transformation. + * @param value Input register. + * @return Transformed register. + */ +FloatRegister SIMD_FLAGS(InOut) transform_register(FloatRegister value) noexcept; +``` + +Consumer functions using the platform's default convention receive no stronger +call-boundary guarantee than equivalent raw native-vector functions under that +same convention. The validation suite compares wrapper and raw signatures under +both the supported vector convention and the platform default. Any wrapper-only +default-convention overhead is documented explicitly; it cannot be attributed +to Register member chaining or hidden by a flagged vector-convention result. + +Ordinary non-static member functions carry an implicit `this` pointer. If such +a function is not inlined, the left operand may need an addressable object even +when a by-value operation could receive it in a vector register. Focused +clang-cl 22.1.8 Windows x64 probes demonstrated this distinction for a +non-inlined 128-bit floating-point addition: the ordinary const member form +used addressable left-operand and result storage, while both the hidden-friend +and explicit-object member forms received their values in vector registers and +returned the result in a vector register. The explicit-object body was one +`vaddps`, and its caller emitted a tail call while retaining `lhs.add(rhs)` +syntax. A separate explicit-object `operator+` probe produced the same ABI and +single-instruction body. Later MSVC 19.44 probes showed that reference-taking +compound assignment on a 256-bit wrapper introduces a redundant 32-byte +stack-alignment frame even when its arithmetic remains register-only. This +evidence motivates both the explicit-object by-value default and the exclusion +of compound assignment, but the complete supported compiler, type, and width +matrix remains an acceptance test rather than an assumed ABI guarantee. + +The implementation must: + +- Store only the public `native_type native` representation in each `Register` and `RegisterMask`. +- Add no virtual functions, allocator state, active-lane metadata, or hidden + heap allocation. +- Preserve `ForceInline`, the appropriate `SIMD_FLAGS(...)` boundary mode, `noexcept`, and `constexpr` + where the delegated `Api` operation supports them. +- Use the native zero-register operation for default construction without + introducing a memory clear, temporary array, or store/reload sequence. +- Avoid a store/reload round trip for ordinary arithmetic, bitwise, + comparison, selection, and rearrangement chains. +- Pass and return `Register` and `RegisterMask` values without extra stack + traffic, hidden copies, branches, register moves, or indirection compared + with the corresponding native register type under the supported calling + convention. +- Preserve the existing scalar fallback behavior when that behavior is part of + the documented `Api` contract. +- Use mandatory generated-code checks to demonstrate that every public + operation family, overload shape, supported element type, and register width + produces code equivalent to the corresponding direct `Api` or intrinsic + expression. Benchmarks are supplemental evidence only and cannot replace a + missing generated-code comparison. + +A compiler can theoretically classify a class containing an intrinsic vector +differently from the intrinsic type itself at a non-inlined function boundary. +That possibility is not an accepted exception. The supported compiler and +calling-convention matrix must be tested with both inlined expressions and +separately compiled, non-inlined functions. If any wrapper specialization is +passed, returned, spilled, copied, or otherwise handled less efficiently than +the native register, the difference must be identified and discussed before +the design can be accepted. The implementation or public calling convention +must then be adjusted, or that compiler/type/width combination must be +explicitly excluded from the zero-overhead support claim. + +The zero-overhead support claim is configuration-specific. Each accepted result +records the compiler and version, target architecture, ISA switches, SimdLib +configuration, optimization mode, and calling convention used for both wrapper +and raw baselines. Optimized Release builds are the mandatory machine-code +gate. The representative ordinary Debug and ASan+UBSan cells run their assigned +correctness contracts but do not build generated-code fixtures. Explicitly +selected Debug or sanitizer diagnostics can record wrapper-versus-raw +differences under identical flags; they are not claimed to have optimized +Release assembly and cannot satisfy the mandatory Release gate. + +## Error and precondition policy + +Fixed-extent spans enforce full-register load and store sizes at compile time. +Aligned operations retain the existing runtime precondition that the pointer +meets `byte_count` alignment. Compile-time lane selectors are constrained to +valid indices. Unsupported type, width, and operation combinations are removed +from overload resolution with concepts or `requires` clauses. + +`Register` adds no exception-based error handling. It follows the current +SimdLib precondition configuration for invalid runtime inputs such as alignment +or shift counts. + +## Migration and compatibility + +`Register` is the recommended interface for supported C++23 complete-register +work. `Api` remains an authoritative supported C++20, compatibility, +backend-facing, and collection-oriented interface. + +- `` is included by the umbrella header conditionally when + `SIMDLIB_REGISTER_INTERFACE_AVAILABLE` is nonzero and before headers that + consume it. +- README complete-register examples use `NativeRegister` or explicit + `Register`. +- `Api` remains documented for compatibility, specialized low-level access, and + existing collection helpers. +- C++23 examples, header probes, ODR fixtures, and external-consumer gates use + `Register` without introducing circular header dependencies. +- `Api` has no deprecation attribute merely because `Register` is now + recommended. Any removal or warning policy requires a separate compatibility + decision and versioning plan. + +No production C++20 header is an appropriate Register migration candidate: +`SimdVector` can own fewer logical lanes than its backing register, `SimdAlgo` +and `SimdResample` own collection and tail policy, and `uint128_t` is a scalar +abstraction. Moving those implementations to the C++23 interface would either +raise the core language requirement or violate the ownership boundary above. + +Representative migration: + +```cpp +// Existing interface. +using U32Api = SimdLib::Api<128, std::uint32_t>; +const auto old_result = U32Api::bitwise_or( + U32Api::add(lhs, rhs), + U32Api::set1(1)); + +// Register interface. +using U32Register = SimdLib::Register; +const auto new_result = + (U32Register{lhs} + U32Register{rhs}) | + U32Register::broadcast(1); +``` + +## Validation strategy + +The implementation requires evidence in each of these areas: + +- C++20 compile probes proving that + `SIMDLIB_REGISTER_INTERFACE_AVAILABLE == 0`, the umbrella header remains + usable, and existing public surfaces retain their current language baseline. +- Supporting C++23 compile probes proving that + `SIMDLIB_REGISTER_INTERFACE_AVAILABLE == 1`, `Register.h` is exposed, and the + explicit-object declarations compile. A negative direct-header probe verifies + the focused diagnostic when the feature is unavailable. +- CMake consumer probes proving that `SimdLib::SimdLib` retains its C++20 + requirement, `SimdLib::Register` requests C++23 and the requirement macro, + Microsoft C++ receives `/std:c++latest`, and an unsupported compiler receives + the focused diagnostic. The Microsoft probe verifies the generated compiler + command, `_MSVC_LANG > 202002L`, and the required explicit-object syntax. +- Dedicated availability probes for both detection paths: the standardized + `__cpp_explicit_this_parameter >= 202110L` path on clang-cl, Clang, and GCC, + and the `_MSC_VER >= 1944` plus `_MSVC_LANG > 202002L` fallback on Microsoft + C++. MSVC probes cover named methods and overloaded arithmetic and comparison + operators, and constraint probes verify that compound assignment remains + unavailable. The same MSVC toolset is also compiled in C++20 mode to prove + that the fallback remains disabled. +- A configuration probe proving that clang-cl cannot enter the Microsoft C++ + fallback through its compatibility definition of `_MSC_VER`. +- Compile-time availability checks for every supported element type at 128 and + 256 bits under the existing feature profiles. +- Compile-time rejection of partial lane lists and wrong-extent spans. +- Compile-time result-alias checks for every supported type-changing operation, + plus rejection of aliases and operations for unsupported backend + combinations. +- Compile-time rejection of out-of-range immediates and selectors, plus runtime + and constant-evaluation tests at every documented shift-count boundary. +- Compile-only validation that the declaration sketch, forward declarations, + constraints, aggregate construction contracts, and focused-header include + boundary are self-contained. +- Layout and trivial-copy checks for integer, float, and double register + families on each supported compiler. +- Runtime construction, load, store, and operation tests that use distinctive + values in every lane, especially the highest lane. +- Element and raw-byte transfer tests proving exact full-register bit + preservation and the absence of partial or dynamic-extent unsafe overloads. +- Direct parity tests against the public `Api` contract for every migrated + operation and supported type/width combination. +- Mask tests covering all-false, all-true, alternating, first-lane-only, and + highest-lane-only predicates, compact lane bits, bitwise composition, + selection polarity, and cleared unused scalar bits. Static assertions verify + that `bits_type` is the documented unsigned type for every supported width + and lane geometry. +- Mask-native interoperation tests proving that the public `native` member contains + the complete predicate bits without a store/reload round trip, that direct + native aggregate initialization requires canonical predicate lanes, and that + scalar bit fields and numeric Registers cannot construct a `RegisterMask`. +- Backend-adapter tests proving that runtime, portable, emulated, and + constant-evaluated comparisons produce the same intrinsic-defined predicate + lanes. +- Conversion tests that prove numeric conversion and bit reinterpretation do + not overlap semantically. +- Rearrangement tests that document lane order and selector behavior. +- `constexpr` probes for every operation whose `Api` counterpart supports + constant evaluation. +- Representative Debug-contract and sanitizer runs that confirm full-register + access does not read beyond caller storage. +- Separate validation of the core C++20 matrix and the narrower Register matrix: + Windows x64 uses MSVC 19.44 and clang-cl 20 or newer. + Linux x64 uses Clang 22 and GCC 14 or newer; GCC 13.2 is a required + unavailable-interface probe for the core matrix. +- Mandatory generated-code comparisons retain composed arithmetic, comparison + followed by mask composition, selection, or reduction, broadcast reuse, + nonzero-index extraction, immediate and complete shifts, load/operate/store, + aligned and byte transfers, special members, reassignment, mutation, register + pressure, and opaque calls. Benchmarks may supplement these comparisons but + never replace them. +- The type matrix is the canonical isolated-operation suite. It emits an + individual no-inline symbol only when the matching `IRegister` concept is + available, covers all supported element types, widths, and ISA profiles, and + compares `Register` with the equivalent public `Api` expression. Dynamic + extract and insert operations are excluded because they are not Register APIs. + Common non-modulus symbols and integer-modulus symbols use separate records so + a narrowly documented compiler scheduling diagnostic cannot weaken unrelated + exact comparisons. +- The FMA-independent specialized-operation matrix is compiled once per width + and ISA profile. A separate fixture containing only `multiply_add_f32` and + `multiply_add_f64` is compiled with FMA enabled and disabled so an unrelated + fused instruction cannot satisfy the instruction-property check. +- Handwritten intrinsic and scalar codegen mirrors are temporary + algorithm-evaluation tools unless a documented instruction-property contract + cannot be expressed through the public `Api` baseline. Selected-algorithm + copies do not remain in permanent codegen fixtures. +- Forced-inline probes retain aggregate initialization, implicit + compiler-generated special members, static factories, and reassignment + expressions. The supported performance gate fails if a wrapper is + unnecessarily materialized when the equivalent direct operation remains in + registers. +- Test-only, separately compiled, non-inlined ABI mirrors cover the + explicit-object signature families: unary, binary, ternary, scalar-result, + mask-result, native-result, store, and mutating-reference operations. These + compare `Register`, `RegisterMask`, `Api::vector_t`, and raw-vector calling + conventions for every supported compiler, element type, and register width. +- Paired consumer-defined function probes use `SIMD_FLAGS(...)` and the platform + default convention. The vector-convention gate rejects any wrapper-only ABI + overhead. Default-convention differences are recorded explicitly and remain + outside the supported call-boundary guarantee unless that compiler and + signature also pass the raw-vector comparison. +- Record symbol groups are nonoverlapping. Expression and consumer-ABI + aggregates remain build conveniences, while one `RegisterCodegen.` + CTest owns every record in its profile exactly once. +- Configuration-provenance records accompany every code-generation and ABI + artifact, including compiler version, architecture, ISA switches, SimdLib + configuration, optimization mode, calling convention, stack-protector mode, + exact symbol filter, and raw baseline. Explicit Debug and sanitizer diagnostic + results are reported separately from optimized Release evidence. + +Tests use the current `Api` as the permanent generated-code parity baseline. +Independent scalar references remain necessary in behavioral tests and +benchmarks so both public surfaces cannot agree on the same defect unnoticed; +those references are not retained as duplicate permanent codegen algorithms. +The complete per-symbol retention and ownership decisions are defined by +`RegisterCodegenSymbolAudit.csv` and summarized with the build and artifact +inventory in `RegisterCodegenAudit.md`. + +## Acceptance criteria + +The final public surface and its qualification contract follow these decisions: + +- Template order is `Register`; the legacy + `Api` order is not propagated to new types. +- The core SimdLib target remains C++20. The Register interface is exposed only + when `SIMDLIB_REGISTER_INTERFACE_AVAILABLE` detects either + `__cpp_explicit_this_parameter >= 202110L` or the documented Microsoft C++ + fallback of `_MSC_VER >= 1944` and `_MSVC_LANG > 202002L`; no normalized + general language-version macro is introduced, and `_HAS_CXX23` is not used. +- Availability is computed by SimdLib, cannot be overridden, and has no initial + opt-out. `SimdLib::Register` is the C++23 opt-in target; the base target does + not impose that requirement. The opt-in target explicitly selects + `/std:c++latest` for Microsoft C++ and compile-probes the resulting language + mode. +- Register support has a separate validated compiler matrix from the C++20 core + matrix. A compiler's language-feature support alone does not admit it to the + zero-overhead support claim. +- Width selection is expressed through `NativeRegister`, not a + defaulted primary-template argument. +- Default construction explicitly uses the appropriate intrinsic-backed + zero-register operation and never leaves a register uninitialized. +- Scalar arithmetic requires an explicit broadcast. +- Every load, store, and lane-list constructor covers one complete register. +- `load` and `store` are the canonical unaligned element transfers; + `load_bytes` and `store_bytes` are the exact-width raw-bit transfers, and no + unsafe or partial transfer is exposed. +- Lane-wise comparisons return `RegisterMask`; whole-value equality returns + `bool`. +- `RegisterMask` is a one-member native aggregate and exposes compact lane bits, + Boolean reductions, bitwise composition, lane selection, and a by-value native + observer. Direct native initialization requires canonical all-zero/all-one + predicate lanes. Its normalized unsigned `bits_type` is selected from + `lane_count` rather than inherited from `Api::mask_t`. +- Comparison behavior exactly matches the selected underlying hardware + intrinsic, including floating-point edge cases and predicate-lane bit + patterns. +- Register-shaped comparisons are implemented directly by `Register` + through its selected `Api` type, without a redundant backend wrapper or + `Detail` names leaking to consumers. +- Numeric conversion and bit reinterpretation have separate names. +- Width-changing operations cannot silently discard active lanes. +- Type-changing operations return the exact constrained namespace-level result + alias documented in the operation ledger. +- Shift boundaries, immediate domains, and selector ranges are explicit public + contracts and are checked in runtime, constant-evaluation, and compile-failure + tests as applicable. +- Collection transforms and partial-register operations remain outside + `Register`. +- The operation ledger is the controlling migration boundary: every current + public `Api` operation has a preferred `Register` spelling or an explicit + compatibility-only classification. +- Zero overhead means that `Register` introduces no additional instructions, + moves, spills, reloads, stack traffic, temporaries, branches, or indirection + relative to equivalent raw-intrinsic code compiled in the same context; it + does not claim that raw SIMD values can never spill. +- Every non-static operation uses an explicit object parameter by value and the + appropriate `SIMD_FLAGS(...)` boundary mode, preserving member-call syntax + without an implicit `this` pointer. Compound assignment is intentionally absent; callers + use explicit reassignment through the by-value binary operators. +- Call-boundary behavior is validated separately for MSVC, clang-cl, Clang, + and GCC because the configured `SIMD_FLAGS(...)` boundary mode is a + calling-convention tool, not a physical register-residency guarantee. +- Non-inlined consumer-defined functions must declare the appropriate + `SIMD_FLAGS(...)` boundary mode to participate in the vector-calling- + convention guarantee. Default + convention signatures are compared with raw vectors separately and are not + included unless they independently pass the zero-overhead gate. +- Generated-code comparisons are mandatory for every public operation family, + overload shape, supported type, width, and ISA profile under identical + optimized settings; benchmarks are supplemental only, and each artifact + records its complete configuration provenance. +- All translation units exchanging `Register` or `RegisterMask` values use + compatible ISA, calling-convention, compiler-ABI, and SimdLib settings. +- `Api` remains supported throughout migration and is not immediately marked + deprecated. + +Implementation is complete only when the intended register-local operation +matrix is mapped, tests pass across the supported compiler and feature matrix, +documentation recommends `Register`, and generated-code plus call-boundary +evidence shows no abstraction penalty relative to direct `Api` or intrinsic +use. Any observed exception must be identified and discussed explicitly before +the affected configuration can be described as supported. diff --git a/docs/RegisterQualification.md b/docs/RegisterQualification.md new file mode 100644 index 0000000..9a76a73 --- /dev/null +++ b/docs/RegisterQualification.md @@ -0,0 +1,173 @@ +# Register Qualification Contract + +This document defines the supported `Register` and +`RegisterMask` qualification matrix, the evidence required for each +supported cell, and the exclusions that bound the zero-overhead claim. Generated +artifacts and individual execution results are intentionally not committed; the +commands below reproduce them under `build*/register-codegen` or +`out/pipeline`. + +## Supported matrix + +| Dimension | Supported cells | +| --- | --- | +| Architecture | x86-64 | +| Register widths | 128 and 256 bits | +| Availability floor | SSE4.2 exposes the 128-bit specialization; AVX2 additionally exposes the 256-bit specialization | +| Optimized zero-overhead profile | AVX2 for the complete 128-bit and 256-bit wrapper/raw corpus | +| Optimized diagnostic profile | SSE4.2 for the complete 128-bit wrapper/raw corpus | +| Element types | `int8_t`, `uint8_t`, `int16_t`, `uint16_t`, `int32_t`, `uint32_t`, `int64_t`, `uint64_t`, `float`, and `double` | +| Windows compilers | MSVC 19.44 and clang-cl 20 or newer | +| Linux compilers | GCC 14 and Clang 22 on the pinned Alpine/musl images | +| Optimized configuration | Release with strict wrapper/raw generated-code comparison | +| Optional diagnostic configurations | Explicitly selected Debug compiler; ASan+UBSan on Clang 22 only for an instrumentation investigation | +| FMA profiles | Explicitly disabled under SSE4.2; explicitly enabled and disabled under AVX2 | + +Every supported compiler must compile the C++23 interface, the complete runtime +and constexpr corpus for each ISA-available width, and the external consumer. +AVX2 participates in the strict optimized wrapper/raw gate. SSE4.2 compiles the +same 128-bit fixtures with Release optimization and records any differential; +it is a correctness-supported profile but is excluded from the zero-overhead +claim. An optimized zero-overhead cell is supported only when its applicable +wrapper/raw profiles are instruction-identical after allocation-independent +normalization, except for an exact exception listed below. + +## Correctness evidence + +- `tests/Register.tests.cpp`, `tests/RegisterBasicOperations.tests.cpp`, + `tests/RegisterSpecializedOperations.tests.cpp`, and + `tests/RegisterRearrangementConversion.tests.cpp` compare results with + independent scalar references. `Api` results are secondary migration checks, + not the sole oracle. +- `tests/constexpr/RegisterConstexpr.tests.cpp` instantiates both widths and all + element types for every Register and RegisterMask operation backed by a + constant-evaluable `Api` operation. Conversion, widening, and bit-cast cells + are evaluated across the complete source/target matrix subject to the MSVC + frontend exclusion below. +- `tests/RegisterPreconditionFailure.tests.cpp` runs alignment and invalid + runtime-shift failures in isolated processes. Valid boundary transfers, + conversions, shifts, rearrangements, and mask paths run in the ordinary test + corpus and in the Clang ASan+UBSan configuration. +- `tests/RegisterOperationMatrix.tests.cpp` is the compile-time availability + oracle. The generated-code type matrix emits a symbol only when the matching + `IRegister` operation is available, so unavailable floating modulus and shift + cells cannot be mistaken for supported identity operations. + +## Generated-code and ABI evidence + +`cmake/CompareRegisterCodegen.cmake` disassembles separately compiled wrapper +and raw objects, normalizes allocation-dependent details, and compares complete +instruction profiles. Optimized Release comparisons reject wrapper-only +instructions, moves, spills, reloads, stack traffic, return buffers, branches, +temporaries, and indirection. + +The permanent corpus assigns one contract to each fixture and one public raw +`Api` baseline to each parity comparison: + +The per-symbol ownership, category, baseline, validation owner, retention +decision, and rationale are recorded in +`RegisterCodegenSymbolAudit.csv`; `RegisterCodegenAudit.md` inventories the +source, target, record, CTest, CI-artifact, and documentation boundaries. + +- `RegisterCodegenFixture.h` retains composed expressions, mask composition and + reduction, broadcast reuse, nonzero lane extraction, immediate and complete + shifts, memory transfers, mutation, special members, reassignment, register + pressure, and opaque-call behavior. Its register-only, reassignment, and + memory/composition records use nonoverlapping symbol filters. +- `RegisterTypeMatrixCodegenFixture.h` is the canonical isolated-operation suite. + It emits one no-inline symbol for every available Register and RegisterMask + operation across all ten element types and every supported width. Construction, + load, store, byte transfer, and array observation are separate symbols; dynamic + indexing is excluded because it is not part of the Register surface. Its + comparison is partitioned into common non-modulus and integer-modulus records + so a compiler-specific scalar-remainder diagnostic cannot weaken unrelated + exact gates. +- `RegisterSpecializedCodegenFixture.h` covers the FMA-independent specialized + operation matrix once per width and ISA profile. +- `RegisterFmaCodegenFixture.h` contains only the single- and double-precision + multiply-add symbols and is compiled with FMA explicitly enabled and disabled + where the ISA profile permits it. +- `RegisterRearrangementCodegenFixture.h` covers selectors, rearrangements, + conversions, bit casts, width changes, and the public `Register::shuffle` + versus `Api::shuffle` baseline. + +Handwritten intrinsic or scalar mirrors are algorithm-evaluation tools, not +permanent codegen baselines, unless they protect a documented instruction +property that the public `Api` baseline cannot express. Behavioral tests and +benchmarks retain independent scalar oracles where correctness or performance +requires them. + +- `RegisterAbi.cpp` and `RegisterAbiRaw.cpp` mirror Register, RegisterMask, + native-vector, scalar-result, native-result, store, mutating-reference, and + downstream-consumer signatures as separately compiled no-inline functions. + +MSVC and clang-cl supported call-boundary claims use the appropriate +`SIMD_FLAGS(...)` boundary mode. On GCC and GNU-like Clang its +vector-calling-convention adapter is empty, so the paired raw/default platform +ABI is the supported boundary. Windows platform-default calling-convention artifacts are +recorded separately by `RecordRegisterDefaultAbi.cmake`; they are diagnostic and +do not participate in the Windows call-boundary guarantee. + +SSE4.2 Release builds compile the same wrapper/raw objects with identical flags +and record diagnostic-only differences. Debug and sanitizer wrapper/raw +comparisons are available only through the explicit `Record-Codegen.ps1` +operation for a selected investigation; ordinary runtime builds do not compile +their fixtures. Optimized Release AVX2 remains the zero-overhead gate except for +the exact diagnostic subsets listed below. Every artifact records `isa_profile` in addition +to the compiler, configuration, width, calling convention, and stack-protector +mode. Artifacts are separated under `register-codegen/sse42/128`, +`register-codegen/avx2/128`, and `register-codegen/avx2/256`. Each profile's +`RegisterExpressionCodegen` and `RegisterConsumerAbi` targets remain build +conveniences; the single `RegisterCodegen.` CTest owns validation of +every record in that profile exactly once. + +Method-attribute records and text evidence live under `method-flags-codegen` and +are published with the Register artifact roots. Their single validation owner is +the `MethodFlagsCodegen` CTest. + +## Exception and exclusion ledger + +| Cell | Disposition | Justification | +| --- | --- | --- | +| SSE4.2 generated-code corpus | Optimized diagnostic; excluded from the zero-overhead claim | Legacy two-operand SSE can expose aggregate-sensitive instruction selection and register coalescing. The complete 128-bit corpus is retained for compiler-by-compiler inspection without treating a recorded difference as an accepted optimized exception. | +| MSVC 19.44, 128-bit `Register::from_array` under SSE4.2 and AVX2 | Exact accepted Release exception | MSVC adds one `/GS` cookie prologue/epilogue to the wrapper path. The comparator separately recognizes the exact legacy `movdqu` SSE4.2 sequence and exact `vmovdqu` AVX2 sequence, then requires every remaining instruction to match the raw mirror. | +| MSVC memory-capable aggregate corpus | Recorded, outside the zero-overhead claim when `/GS` differs | Stores, transfers, array returns, mutating references, and other addressable paths intentionally retain `/GS`; applying the `RegisterOnly` modifier would suppress protection for functions that can write memory. | +| MSVC 19.44, AVX2/256 integer modulus | Recorded scheduling diagnostic; excluded from the strict parity claim | The `Register::operator%` and `Api::modulus` paths inline the same scalar lane-remainder algorithm, but MSVC schedules independent extract, divide, and insert operations differently after the aggregate operator boundary. The modulus symbols have their own record so this diagnostic cannot relax any other type-matrix operation. | +| MSVC constexpr bit-cast value matrix | Frontend evaluation excluded | MSVC 19.44 terminates with an internal compiler error when evaluating the first Register bit-cast cell. MSVC still compiles the complete availability matrix and validates runtime bit-cast values; GCC and both Clang drivers perform the complete constexpr value matrix. | +| clang-cl Windows platform-default aggregate ABI | Diagnostic only; failing signatures excluded | The platform-default convention may use hidden return storage for aggregate Register results. `SIMD_FLAGS(...)` wrapper/raw parity is the supported clang-cl boundary. | +| MSVC Windows platform-default aggregate ABI | Diagnostic only; hidden-return signatures excluded | The platform-default convention also returns aggregate Register results through caller-provided storage. The supported non-inline boundary uses the appropriate `SIMD_FLAGS(...)` mode; default-convention disassembly remains available without expanding the guarantee. | +| Debug wrapper/raw differences | Optional record, not accepted as Release overhead | Disabled optimization preserves abstraction structure and may add wrapper-only calls, temporaries, or stack traffic. An explicit diagnostic compiles both sides with identical Debug flags when that difference needs investigation. | +| ASan+UBSan wrapper/raw differences | Optional record, not accepted as Release overhead | An explicit Clang 22 diagnostic exposes instrumentation-induced wrapper/raw memory, control-flow, or ABI differences. Runtime sanitizer tests own correctness and absence of sanitizer diagnostics; instruction identity is not a default requirement. | +| 32-bit targets, non-x86 architectures, 512-bit registers, AVX-512, and compilers below the listed versions | Unsupported | No complete correctness, ABI, and zero-overhead matrix exists for these cells. | + +No other optimized Release performance exception is accepted. Adding one +requires an exact recognizer, a written justification here, and review of why +the operation cannot satisfy the supported zero-overhead contract. + +## Reproduction commands + +The formal scoped commands reproduce the native and pinned Linux Register +qualification. Release fingerprints enforce generated-code policy; ordinary +Debug and sanitizer fingerprints contain no Register generated-code workload: + +```powershell +tools/Build.ps1 -Scope Native -Compiler Msvc,ClangCl +tools/Run-Tests.ps1 -Scope Native -Compiler Msvc,ClangCl +tools/Build.ps1 -Scope Containers -Compiler Gcc14,Clang22 +tools/Run-Tests.ps1 -Scope Containers -Compiler Gcc14,Clang22 +tools/Record-Codegen.ps1 -Scope Native -Compiler Msvc -Cell Debug +tools/Record-Codegen.ps1 -Scope Containers -Compiler Clang22 -Cell Debug +tools/Record-Codegen.ps1 -Scope Containers -Compiler Clang22 -Cell AsanUbsan +tools/Build-Benchmarks.ps1 -Scope All -Compiler Msvc,ClangCl,Gcc14,Clang22 +tools/Run-Benchmarks.ps1 -Scope All -Compiler Msvc,ClangCl,Gcc14,Clang22 +``` + +The record command requires an explicit compiler and cell, builds only the +generated-code fixture and comparison targets, and writes dedicated provenance. +Its record-only outputs cannot satisfy a missing Release enforcement result. + +Benchmarks are supplemental and run only after strict generated-code gates. The +Register benchmark operands derive from a runtime clock seed and are returned +from each measured expression so constant folding and dead-code elimination +cannot replace the work. Benchmark timing never overrides an assembly failure +and has no pass/fail performance threshold. diff --git a/docs/StaticAssertionInventory.md b/docs/StaticAssertionInventory.md deleted file mode 100644 index d59dbc7..0000000 --- a/docs/StaticAssertionInventory.md +++ /dev/null @@ -1,24 +0,0 @@ -# Production Static-Assertion Inventory - -The production-header audit classifies every retained `static_assert` and rejects any new occurrence that is not listed with a justification in `cmake/PublicHeaderStaticAssertAllowlist.txt`. The CMake build target and CTest entry both execute `cmake/AuditPublicHeaderAssertions.cmake`. - -## Extraction result - -- `Bmi.h`: 121 test-example assertions moved verbatim to `tests/constexpr/BmiConstexpr.tests.cpp`. -- `UInt128.h`: six arithmetic, shift, and bit-helper examples moved verbatim to `tests/constexpr/UInt128Constexpr.tests.cpp`. -- Other production headers contained no namespace-scope or function-adjacent test examples. -- The dedicated sources keep the migrated assertions before separately labelled expanded contracts, so the original proof is preserved independently of later additions. - -## Retained assertions - -| Header | Count | Classification | Why evaluation must remain in production | -| --- | ---: | --- | --- | -| `UInt128.h` | 5 | Four ABI/layout invariants; one template-width constraint | Register conversion requires a 16-byte, 16-byte-aligned, standard-layout, trivially-copyable representation; invalid mask widths must fail at instantiation. | -| `Bmi.h` | 3 | Two template control-field constraints; one implementation safety invariant | Invalid immediate controls must be diagnosed and the 64-bit product split must retain its word-size assumption. | -| `Api.h` | 17 | Fourteen template constraints, one dependent unsupported-mapping diagnostic, two implementation safety invariants | Invalid widening, conversion, packed-result, shift, endian, and callable shapes must fail at the caller instantiation. | -| `SimdAlgo.h` | 2 | Template constraints | Invalid packed comparison result widths and storage shapes must fail at instantiation. | -| `Detail/Implementations.h` | 19 | Sixteen dependent unsupported-mapping diagnostics, two extraction-index constraints, one implementation safety invariant | Unsupported widening shapes need dependent diagnostics; extraction and scalar lane-size assumptions must be checked where instantiated. | -| `Detail/Extensions.h` | 2 | Template constraints | Negative immediate whole-register shifts must fail at instantiation. | -| **Total** | **48** | 23 template constraints, 17 unsupported-instantiation diagnostics, four ABI invariants, and four implementation safety invariants | No test-example assertion remains in production headers. | - -The allowlist has 30 entries because one justified rule covers repeated assertions with the same contract, such as the sixteen backend-dependent widening diagnostics. \ No newline at end of file diff --git a/docs/TestCoverage.md b/docs/TestCoverage.md index 829748d..20969af 100644 --- a/docs/TestCoverage.md +++ b/docs/TestCoverage.md @@ -1,8 +1,8 @@ -# Test coverage audit +# Test coverage contract -This document records the standalone SimdLib coverage audit completed on -2026-07-19. Coverage percentages are supporting evidence; the behavioral map -and the feature-profile matrix are the acceptance criteria. +This document defines SimdLib's enduring behavioral coverage and feature-profile +ownership. Run-specific percentages, counts, timings, and tool identities belong +in generated build receipts, reports, coverage artifacts, and CI results. ## Coverage layers @@ -14,7 +14,7 @@ and the feature-profile matrix are the acceptance criteria. | Configuration | Default detection, caller overrides, all instruction families disabled, FMA enabled/disabled, BMI1/BMI2 independently enabled, and portable/optimized/scalar UInt128 profiles | | Formatter and ODR | Scalar-formatter parity, vector and UInt128 formatting, umbrella/focused-header probes, and a two-translation-unit formatter executable | | Oracle/property testing | Deterministic scalar oracles for comparisons, transfers, BMI operations, UInt128 arithmetic/bit operations, algorithms, and resampling | -| Compiler/runtime diagnostics | Strict Release builds on MSVC 19.44 and clang-cl 22.1.8; Clang 22.1.8 ASan/UBSan Debug run | +| Compiler/runtime diagnostics | Every applicable supported Release compiler, representative MSVC Debug, and the independent Clang ASan/UBSan Debug cell | | External consumer | `tests/consumer` validates source-tree import, the interface-library target, public includes, and header-only linkage | Benchmarks are intentionally excluded from correctness counts. They exercise @@ -23,66 +23,90 @@ acceptance rules. ## Test inventory -The standard Clang coverage preset contributes 182 CTest entries: 172 -individual Catch2 test cases discovered by `catch_discover_tests()` and 10 -direct CTest audit, compile, example, and equivalence tests. The 13 -terminating precondition cases are discovered Catch2 cases, not direct CTest +Every runtime profile discovers individual Catch2 cases with +`catch_discover_tests()`. The Clang coverage profile owns only execution-bearing +runtime and checks/precondition targets; applicable Release compiler cells own +header, compiler-contract, constexpr, example, smoke, and ODR validation. +Terminating precondition cases are discovered Catch2 cases, not direct CTest driver scenarios. Catch2 executables remain grouped by these stable name prefixes: | Entry | Coverage role | | --- | --- | -| `SimdLib.HeaderOnlySmoke` | Multi-translation-unit umbrella-header use and header-only linkage | -| `SimdLib.Tests.BmiPortable.*` | Portable BMI behavior, constexpr checks, boundaries, signed bit patterns, and deterministic randomized oracles | -| `SimdLib.Tests.Format.*` | UInt128 and vector formatter behavior plus standard scalar parity | -| `SimdLib.FormatOdr` | Formatter specialization linkage across two translation units | -| `SimdLib.Tests.SSE42.*` | 128-bit `Api`, partial transfers, comparisons, conversion, movemask, and register metadata | -| `SimdLib.Tests.UInt128Optimized.*` | UInt128 with compiler carry primitives and available SIMD support | -| `SimdLib.Tests.UInt128Portable.*` | UInt128 with portable carry/borrow | -| `SimdLib.Tests.UInt128Scalar.*` | UInt128 with all SIMD, BMI, FMA, and compiler-carry features disabled | -| `SimdLib.Tests.UInt128ResultSetEquivalence` | Optimized-versus-portable deterministic result digest | -| `SimdLib.Tests.UInt128ScalarResultSetEquivalence` | Optimized-versus-scalar deterministic result digest | -| `SimdLib.Tests.AVX2.*` | 256-bit `Api`, partial transfers, comparisons, movemask, and register metadata | -| `SimdLib.Tests.FMA.Enabled.*` | FMA-enabled dispatch and expected result | -| `SimdLib.Tests.FMA.Disabled.*` | Non-FMA fallback dispatch and expected result | -| `SimdLib.Tests.Bmi.Bmi1Only.*` | BMI1 intrinsic profile | -| `SimdLib.Tests.Bmi.Bmi1Only.Equivalence` | BMI1-versus-portable deterministic result digest | -| `SimdLib.Tests.Bmi.Bmi2Only.*` | BMI2 intrinsic profile | -| `SimdLib.Tests.Bmi.Bmi2Only.Equivalence` | BMI2-versus-portable deterministic result digest | -| `SimdLib.Tests.Bmi.Bmi1AndBmi2.*` | Combined BMI1/BMI2 intrinsic profile | -| `SimdLib.Tests.Bmi.Bmi1AndBmi2.Equivalence` | Combined-profile-versus-portable deterministic result digest | -| `SimdLib.Tests.VectorAlgorithms.*` | `SimdVector`, `SimdAlgo`, and SIMD `SimdResample` behavior | -| `SimdLib.Tests.ResampleScalar.*` | Scalar-only `SimdResample` behavior and oracle parity | -| `SimdLib.Tests.Preconditions.*` | Individually discovered terminating caller-facing precondition contracts; marker-gated CTest success | -| `SimdLib.ApiExamples` | Public documented call sites compiled and run together | +| `HeaderOnlySmoke` | Multi-translation-unit umbrella-header use and header-only linkage | +| `RegisterOdr` | Multi-translation-unit Register and RegisterMask use through the C++23 interface target | +| `BmiPortable.*` | Portable BMI behavior, constexpr checks, boundaries, signed bit patterns, and deterministic randomized oracles | +| `Format.*` | UInt128 and vector formatter behavior plus standard scalar parity | +| `FormatOdr` | Formatter specialization linkage across two translation units | +| `Api.SSE42.*` | 128-bit `Api`, partial transfers, comparisons, conversion, movemask, and register metadata | +| `Api.AVX2.*` | 256-bit `Api`, partial transfers, comparisons, movemask, and register metadata | +| `Register.SSE42.*` | 128-bit Register and RegisterMask behavior under the SSE4.2 availability profile | +| `Register.AVX2.*` | 128-bit and 256-bit Register and RegisterMask behavior under AVX2 | +| `Register.AVX2Preconditions.*` | Marker-gated Register alignment and runtime-shift precondition failures | +| `UInt128Optimized.*` | UInt128 with compiler carry primitives and available SIMD support | +| `UInt128Portable.*` | UInt128 with portable carry/borrow | +| `UInt128Scalar.*` | UInt128 with all SIMD, BMI, FMA, and compiler-carry features disabled | +| `UInt128ResultSetEquivalence` | Optimized-versus-portable deterministic result digest | +| `UInt128ScalarResultSetEquivalence` | Optimized-versus-scalar deterministic result digest | +| `FMA.Enabled.*` | FMA-enabled dispatch and expected result | +| `FMA.Disabled.*` | Non-FMA fallback dispatch and expected result | +| `Bmi.Bmi1.*` | BMI1 intrinsic profile | +| `Bmi.Bmi1.Equivalence` | BMI1-versus-portable deterministic result digest | +| `Bmi.Bmi2.*` | BMI2 intrinsic profile | +| `Bmi.Bmi2.Equivalence` | BMI2-versus-portable deterministic result digest | +| `Bmi.Bmi1Bmi2.*` | Combined BMI1/BMI2 intrinsic profile | +| `Bmi.Bmi1Bmi2.Equivalence` | Combined-profile-versus-portable deterministic result digest | +| `VectorAlgorithms.*` | `SimdVector`, `SimdAlgo`, and SIMD `SimdResample` behavior | +| `VectorChecks.*` | Checks-enabled partial and full-vector result validation | +| `ResampleScalar.*` | Scalar-only `SimdResample` behavior and oracle parity | +| `Preconditions.*` | Individually discovered terminating caller-facing precondition contracts; marker-gated CTest success | +| `ApiExamples` | Public C++20 call sites compiled and run together | +| `RegisterExamples` | Public C++23 Register call sites compiled and run together | Compile-only targets cover: - `ApiDisabledProbe` and `ApiEnabledProbe` for API availability, supported lane types, register widths, and conversion constraints; - `ConfigDefaultProbe`, `ConfigDisabledInstructionsProbe`, - `ConfigDisabledPublicHeadersProbe`, `ConfigOverrideForceInlineProbe`, - `ConfigOverridePreconditionProbe`, `ConfigOverrideVectorcallProbe`, - `ConfigVendorAttributeProbe`, `ConfigClangUnsupportedTargetProbe`, and - `ConstexprProbe` for detection, override, disabled, attribute, target, and - constant-evaluation paths; -- first-and-only include probes for `Api.h`, `Bmi.h`, `Config.h`, `Format.h`, - `SimdAlgo.h`, the deprecated `SimdApi.h` compatibility include, `SimdLib.h`, - `SimdResample.h`, `SimdVector.h`, `TemplateTools.h`, and `UInt128.h`; and + `ConfigDisabledPublicHeadersProbe`, `MethodFlagsConfigOverrideProbe`, + `ConfigOverridePreconditionProbe`, `ConfigVendorAttributeProbe`, + `ConfigClangUnsupportedTargetProbe`, and `ConstexprProbe` for detection, + override, disabled, attribute, target, and constant-evaluation paths; +- first-and-only include probes for `Aliases.h`, `Api.h`, `Bmi.h`, `Config.h`, + `Format.h`, `SimdAlgo.h`, the deprecated `SimdApi.h` compatibility include, + `SimdLib.h`, `SimdResample.h`, `SimdVector.h`, `TemplateTools.h`, and `UInt128.h`; and - `PublicSurfaceHeaderProbe` for the supported umbrella/focused-header boundary and the guard against public `Detail` dependencies; and - dedicated BMI, UInt128, 128/256-bit API/vector, and disabled-feature constexpr - targets aggregated by `SimdLibConstexprProbes`. + targets aggregated by `ConstexprProbes`. -The retained-assertion classifications and mechanical allowlist are recorded in -[`StaticAssertionInventory.md`](StaticAssertionInventory.md). The complete -constexpr/compiler matrix, runtime-path evidence, and consumer compile-time -measurements are recorded in -[`ConstexprCompilerEvidence.md`](ConstexprCompilerEvidence.md). +The constexpr sources are ordinary object-library probes aggregated by +`ConstexprProbes`, which is owned by `ExhaustiveArtifacts`. The +`ConstexprProbes.Artifacts` CTest entry validates their recorded object hashes +without recompiling them. Profile ownership is: + +| Contract source | Compile profiles | +| --- | --- | +| `BmiConstexpr.tests.cpp` | Portable, BMI1 only, BMI2 only, and BMI1 with BMI2 | +| `UInt128Constexpr.tests.cpp` | Compiler carry, portable carry, and scalar with SIMD, BMI, and FMA disabled | +| `Api128Constexpr.tests.cpp` | SSE4.2 public API and four-lane `SimdVector` | +| `Api256Constexpr.tests.cpp` | AVX2 public API and eight-lane `SimdVector` | +| `ApiDisabledConstexpr.tests.cpp` | All instruction families disabled | + +Runtime parity targets rebuild deterministic inputs through volatile scalars +before exercising comparisons, extrema, lane shifts, addition, and subtraction. +MSVC x64 owns the `_addcarry_u64` and `_subborrow_u64` UInt128 path; Clang and +GCC own the `__builtin_add_overflow` and `__builtin_sub_overflow` path. Portable +and scalar profiles disable compiler carry intrinsics. + +Production `static_assert` declarations remain local constraints and diagnostics +in their owning headers. Public-header probes and dedicated constexpr targets +compile those declarations under the applicable compiler profiles; no +source-text occurrence count is treated as correctness or compile-time evidence. `tests/consumer` separately imports the source tree through `add_subdirectory`, verifies that `SimdLib::SimdLib` is an interface target, -and runs an external header-only consumer. `SimdLib.benchmarks.cpp` is the sole +and runs an external header-only consumer. `Core.benchmarks.cpp` is the core benchmark executable and samples 128/256-bit API addition, BMI extraction, UInt128 addition, and resampling; each operation also has a correctness test. @@ -90,7 +114,7 @@ UInt128 addition, and resampling; each operation also has a correctness test. | Surface | Directly covered contracts | Profiles | Remaining gap or justification | | --- | --- | --- | --- | -| `Api` | Arithmetic, signed and unsigned comparisons, equality masks, movemasks, loads/stores, unaligned and partial transfers, same-shape transforms, packed transforms with full batches and tails, conversion between signed 32-bit lanes and float, shifts, shuffles, blends, reductions, casts, extraction, and register metadata | 128-bit SSE and 256-bit AVX2; FMA on/off; availability-disabled probes | Some inherited backend helper names are implementation exposure rather than a promised public family. Exhaustively testing them would freeze an accidental contract; the inheritance boundary should be clarified before such tests are added. More conversion rounding/overflow cases are medium-risk follow-up work. | +| `Api` | Arithmetic, signed and unsigned comparisons, equality masks, movemasks, loads/stores, unaligned and partial transfers, same-shape transforms, packed transforms with full batches and tails, conversion between signed 32-bit lanes and float, shifts, logical shuffles for all arithmetic element types, blends, reductions, casts, extraction, and register metadata | 128-bit SSE and 256-bit AVX2; FMA on/off; availability-disabled probes | Some inherited backend helper names are implementation exposure rather than a promised public family. Exhaustively testing them would freeze an accidental contract; the inheritance boundary should be clarified before such tests are added. More conversion rounding/overflow cases are medium-risk follow-up work. | | `SimdVector` | Construction, lane access, arithmetic, comparisons, masks, partial divide/modulus/clamp identity handling, direct active-lane area reduction, lane-local magnitudes, 128/256-bit float/double dot products, floating hashing, inactive-lane min/max behavior, and checks-enabled result validation | Representative signed, unsigned, float, and double lane types; full and partial 128/256-bit extents; Release and checks-enabled profiles | Convenience overloads that delegate directly to `Api` are not all tested individually. Their underlying behavior is covered; add overload-specific tests when they acquire distinct contracts. | | `SimdAlgo` | `AnyEqual` and `AllEqual` full-register/tail outcomes, bitwise transforms, conversions, comparison packing, scalar parity, non-register-multiple tails, and destination canaries | Read widths 8/16/32/64; empty, single, multi-element, exact-register, multi-register, and tail extents | General comparison currently supports `WriteWidth == 1`; unsupported widths are a compile-time precondition, not an untested runtime branch. | | `SimdResample` | Scalar-reference parity for reductions and expansion, boundary dimensions, randomized inputs, and SIMD/scalar equivalence | SIMD enabled and scalar-only profiles | No material gap found. This remains the strongest standalone surface. | @@ -110,13 +134,13 @@ call site is the checks-enabled `SimdVector` inactive-lane result invariant; it is not caller-triggerable through a supported operation, so its direct proof observes successful partial-vector checks and the full-vector bypass. -`SimdLibPreconditionTests` overrides `SIMDLIB_PRECONDITION`, writes the +`PreconditionTests` overrides `SIMDLIB_PRECONDITION`, writes the private `SIMDLIB_PRECONDITION_FAILURE_EXPECTED_18A7E3` marker to stderr, flushes it, and exits with diagnostic status 73 on failure. CTest discovers each Catch2 case as a separate process and requires that marker for success; a missing marker, access violation, unrelated crash, or timeout fails the case. The executable's target-aware coverage prefix is -`SimdLib.Tests.Preconditions`, so its terminating profiles map only to +`Preconditions`, so its terminating profiles map only to that executable in the LCOV report. The override remains active in Release, where the default `assert` policy is compiled out by `NDEBUG`. @@ -127,15 +151,12 @@ no runtime `SIMDLIB_PRECONDITION` governing an index, divisor, or overlap; compile-time constraints and explicitly unsafe entry points retain their existing classifications. -Focused MSVC Release, Clang coverage, and Clang ASan/UBSan runs each pass all -13 isolated failure scenarios. The valid-boundary selection passes 19 -assertions across three cases and covers exact aligned/raw capacities, empty -and one-element partial loads, matching empty/one-element algorithm spans, and -empty/minimum resampling shapes. The complete strict suites pass 179/179 with -MSVC Release and 182/182 with Clang coverage. Clang 22.1.8 ASan/UBSan Debug -passes 151/151 with no diagnostics. The target-aware coverage report maps 190 -profiles to 19 executables, including all 13 failure-probe profiles and the -public API example executable. +The precondition inventory assigns every isolated failure scenario to the +MSVC Release, Clang coverage, and Clang ASan/UBSan cells. Its valid-boundary +cases cover exact aligned/raw capacities, empty and one-element partial loads, +matching empty/one-element algorithm spans, and empty/minimum resampling +shapes. The target-aware coverage configuration includes the failure probes +and public API example as independently owned executable profiles. ## SimdVector full, partial, and wide-vector matrix @@ -144,12 +165,12 @@ public API example executable. | Divide and modulus | Three-lane `int32_t` vectors pass a raw divisor register whose inactive lane is zero. Both value-returning and compound operators produce exact active quotients/remainders and restore the inactive result lane to zero, proving the divisor is filled with multiplicative identity before evaluation. Matching full four-lane cases prove the non-partial route. | | Clamp | A partial `int32_t` vector uses per-lane lower/upper registers with adversarial inactive bounds (`100` and `-100`); active results match their individual bounds and the inactive result is zero. A full four-lane scalar-bound case covers the direct route. | | `area` | Signed `int8_t[5]`, cross-128-bit-lane `int16_t[9]` and `int64_t[3]`, unsigned `uint16_t[5]`, cross-128-bit-lane `uint8_t[17]` and `uint32_t[5]`, odd signed `int32_t[3]`, full `int32_t[4]`, and `int64_t[2]` cases exercise narrow/wide types, odd counts, full/partial reductions, both register halves, and modular signed overflow. | -| Integer magnitude | Partial 256-bit `int16_t[9]` and `uint8_t[17]` inputs produce exact lane-local magnitudes in both 128-bit halves. The first high-lane active value is isolated so omission or cross-lane mixing is observable. | +| Integer magnitude | Every signed and unsigned lane width at 128 and 256 bits covers unchecked representable inputs, checked representable inputs, exact maximum boundaries, multi-lane overflow, and signed-minimum overflow. Partial 256-bit `int16_t[9]` and `uint8_t[17]` vectors verify sparse group-leading magnitudes and adjacent checked overflow masks in both 128-bit halves; an isolated first high-half value makes omission or cross-group mixing observable. | | Min/max position | Partial `uint16_t[3]` proves inactive zero lanes cannot win; full `uint16_t[8]` proves the no-fill route and exact positions. | | Float dot product | A partial 128-bit three-float case remains covered. Counts four through eight cover full 128-bit, partial 256-bit, and full 256-bit vectors; counts five through eight require the high 128-bit lane to contribute to the scalar result. | | Double dot product | Counts one through four cover partial/full 128-bit and partial/full 256-bit vectors. The three- and four-element cases require the high 128-bit lane to contribute. | | Floating hash | Nonzero float and double vectors assert nonzero hashes, copy/equal-value consistency, and selected distinct logical-lane results. Infinity and two representative NaN encodings per type are evaluated with copy consistency; no assertion requires unequal NaNs to hash differently. Existing float and double `+0`/`-0` equality and equal-hash regressions remain direct. | -| Debug result validation | `SimdLibTestsVectorChecks` forces `SIMDLIB_ENABLE_CHECKS=1` and installs an observing precondition hook. Divide, modulus, and clamp on a partial vector invoke the inactive-lane result check three times with true conditions; the same operations on a full vector invoke it zero times. | +| Debug result validation | `VectorChecksTests` forces `SIMDLIB_ENABLE_CHECKS=1` and installs an observing precondition hook for partial and full-vector result checks. | The cross-lane `area` case exposed a register-shape defect: recursive pair reduction could infer a narrower `SimdVector` even though its pair-product @@ -161,16 +182,10 @@ still uses the unsigned object representation so signed overflow remains modular. Narrow cross-lane vectors no longer require unsupported 512- or 1024-bit widened intermediates. -Focused validation on 2026-07-19 passes 266 assertions across 14 public -`SimdVector` cases and 10 assertions in the checks-enabled case with both MSVC -Release and Clang coverage builds. Separate Clang profiles report 100.00% branch -coverage for both the public-vector and checks-enabled instantiations; -counters show three partial-result checks, -zero checks for the full-vector specializations, direct area reduction across -8-, 16-, 32-, and 64-bit lanes, four high-lane float dot additions, and two -high-lane double dot additions. -The complete strict suites pass 162/162 with MSVC Release and 165/165 with -Clang coverage. +Separate public-vector and checks-enabled profiles keep partial-result +validation distinct from full-vector specializations. The coverage contract +requires direct area reduction across 8-, 16-, 32-, and 64-bit lanes and +requires high-lane contributions in the floating-point dot-product cases. ## uint128_t boundary and compatibility matrix @@ -191,18 +206,13 @@ the preferred `Bmi::bextr` replacement remain unchanged. If the deprecated API is intentionally removed later, its compatibility tests should be removed with the declaration rather than transferred into a new preferred surface. -The optimized, portable-carry, and scalar-only executables each run the same -six focused boundary cases with 131 assertions. Separate Clang profiles preserve -object/profile provenance: the scalar profile records both outcomes for -comparison, extraction/truncation, five-bit mask offsets, boolean normalization, -zero/oversized shifts, and `bit_ceil`; the optimized profile records runtime SIMD -shift dispatch. Randomized two-word and compiler-native oracles plus optimized- -versus-portable and optimized-versus-scalar result-set comparisons remain intact. - -Validation on 2026-07-19 passes 35/35 focused `UINT128` tests with MSVC -Release and 38/38 with Clang Debug coverage. Each of the three Clang runtime -profiles passes 131 assertions across the six focused boundary cases. The -complete strict suites pass 154/154 and 157/157 respectively. +The optimized, portable-carry, and scalar-only executables run the same focused +boundary cases. Separate Clang profiles preserve object/profile provenance: +the scalar profile owns comparison, extraction/truncation, five-bit mask +offsets, boolean normalization, zero/oversized shifts, and `bit_ceil`; the +optimized profile owns runtime SIMD shift dispatch. Randomized two-word and +compiler-native oracles plus optimized-versus-portable and +optimized-versus-scalar result-set comparisons remain part of the inventory. ## Formatter grammar matrix @@ -233,15 +243,14 @@ checked against `uint64_t` over zero, small values, a mixed high-bit pattern, an zero and nonzero values across default alignment, explicit alignment, zero padding, and insufficient widths. `Format.h` remains the first include in its standalone header probe, and the formatter specializations remain linked and run -from two translation units by `SimdLib.FormatOdr`. +from two translation units by `FormatOdr`. -Validation on 2026-07-19 runs 267 assertions across the seven `[format]` -cases. The focused formatter and ODR matrix passes 8/8 with MSVC Release and -Clang Debug coverage, the `Format.h` first-include probe compiles with both -compilers, and the complete suites pass 148/148 and 151/151 respectively. A -dedicated Clang profile records the checked width-overflow throw once, both +The formatter runtime inventory executes in every applicable Release cell and +the representative Debug, sanitizer, and coverage profiles. Applicable Release +compiler cells alone own `FormatOdr` and the `Format.h` first-include probe. The +dedicated Clang coverage profile exercises checked width overflow, both trailing-input outcomes, alternate-octal zero and nonzero outcomes, explicit -and default alignment, and both outcomes of insufficient-width zero padding. +and default alignment, and both insufficient-width zero-padding outcomes. ## SimdAlgo outcome and boundary matrix @@ -262,12 +271,10 @@ safety case is unreachable through any supported public `AnyEqual` instantiation; directly exposing the private helper solely for a test would create an implementation test seam. -Validation on 2026-07-19 runs 614 assertions across the seven `[algo]` cases. -The focused matrix passes 7/7 with MSVC Release and Clang Debug coverage; the -complete suites pass 147/147 and 150/150 respectively. A dedicated Clang -profile records the zero-count `LowBits` return four times, both outcomes of -the full-register search conditions, exact-traversal returns, and both tail -results. The `count >= 32` return remains at zero as justified above. +A dedicated Clang profile owns the zero-count `LowBits` return, both outcomes +of the full-register search conditions, exact-traversal returns, and both tail +results. The `count >= 32` branch remains structurally unreachable through the +public API for the reason above. ## High-risk findings resolved by the audit @@ -310,42 +317,39 @@ The randomized/property suites are reproducible. BMI uses seeds `0xD1B54A32D192ED03`, and `0xA0761D6478BD642F`. UInt128 uses `0xD1B54A32D192ED03`, `0x94D049BB133111EB`, and `0xA0761D6478BD642F`. Resampling derives its `std::mt19937` seed from -the tested dimensions so a failing case can be reproduced directly. The final -manual Catch2 assertion inventory used decimal seed `1592594996` for both -release compiler matrices. New -table-driven API comparison and partial-transfer checks report the lane type, +the tested dimensions so a failing case can be reproduced directly. +Table-driven API comparison and partial-transfer checks report the lane type, register width, active count, and failing values through Catch2 captures. ## Source-based coverage Clang's LLVM instrumentation is available through -`SIMDLIB_ENABLE_COVERAGE`. CMake 4.4 or newer is required because CTest 4.4 is -the first release with native `LLVM-COV` dashboard coverage support. Coverage +`SIMDLIB_ENABLE_COVERAGE`. CMake 3.31 or newer drives the instrumented CTest +inventory, after which the pipeline invokes `llvm-profdata`, `llvm-cov`, and +`llvm-readobj` directly to generate the source-coverage report. Coverage configuration intentionally fails for unsupported compiler drivers rather than silently producing misleading data. -The checked-in presets make CTest the authoritative runner. From the SimdLib -repository root: +The unified native coverage fingerprint makes CTest the authoritative runner. +From the SimdLib repository root: ```powershell -cmake --preset clang-coverage -cmake --build --preset coverage -cmake --build build-coverage --target SimdLibCoverageReset -ctest --preset coverage --output-on-failure -cmake --build build-coverage --target SimdLibCoverageReport +tools/Build.ps1 -Scope Native -Compiler ClangCoverage +tools/Run-Tests.ps1 -Scope Native -Compiler ClangCoverage ``` -The CMake Tools extension is the workspace's VS Code test and coverage -provider. Select the `clang-coverage` configure preset and `coverage` build and -test presets, then use **Run with Coverage** in VS Code's Testing view. CMake -Tools runs the configured reset target, invokes CTest, runs the report target, -and imports `build-coverage/coverage.info` into VS Code's native Test Coverage -view. Restart VS Code after installing CMake or adding LLVM's `bin` directory -to `PATH` so the extension sees the tools. +The coverage operation resets profiles, runs the instrumented CTest inventory, +and generates `coverage.info` in the receipt-owned directory +`out/pipeline/windows-clang-coverage/debug-coverage-/build`. +The workspace does not configure a static CMake Tools import path because a +literal “latest” alias could display coverage from an incompatible or stale +fingerprint. Open or import the `coverage.info` referenced by the current +receipt when inspecting coverage in an editor. Coverage report generation does not merge differently configured executables into one `llvm-profdata` database. CMake generates -`build-coverage/coverage-targets-Debug.txt`, which records each instrumented +`coverage-targets-Debug.txt` in that same fingerprint-owned build directory, +which records each instrumented executable, its object path, and its CTest profile prefix. The report target also reads the embedded platform binary identity (COFF/PDB on this baseline) from every executable and profile. This identity maps CTest-created @@ -364,257 +368,48 @@ single-object profiles. The report fails on an unknown binary identity, a filename/identity disagreement, a missing executable profile, any LLVM export diagnostic, or an export with no SimdLib source records. -### Corrected trustworthy baseline - -Before the coverage-pipeline correction, VS Code displayed 2,293/3,348 lines (68.5%), 361/433 -branches (83.4%), and 442/558 functions (79.2%). That report also emitted -`621 functions have mismatched data` after combining 16 differently -configured executables into one incompatible profile database. Those values -are preserved only as the pre-correction baseline. - -The trustworthy baseline below was reproduced on 2026-07-18 with CMake/CTest -4.4.0 and Clang/LLVM 22.1.8. A clean reset followed by all 113 CTest entries -produced 111 per-test `.profdata` files and two CTest-retained `.profraw` -files. The report mapped 108 single-executable profiles to 16 instrumented -executables and excluded five multi-executable equivalence profiles. LLVM -emitted no mismatched-function warning or other export diagnostic. - -| Header | Lines | Branches | Functions | -| --- | ---: | ---: | ---: | -| `Api.h` | 324/425 (76.24%) | 63/160 (39.38%) | 583/618 (94.34%) | -| `Bmi.h` | 266/517 (51.45%) | 116/144 (80.56%) | 109/473 (23.04%) | -| `Config.h` | 1/1 (100.00%) | 0/0 | 0/0 | -| `Detail/Extensions.h` | 134/495 (27.07%) | 80/88 (90.91%) | 61/166 (36.75%) | -| `Detail/Implementations.h` | 618/812 (76.11%) | 39/60 (65.00%) | 387/432 (89.58%) | -| `Format.h` | 212/224 (94.64%) | 200/332 (60.24%) | 18/18 (100.00%) | -| `SimdAlgo.h` | 174/180 (96.67%) | 10/20 (50.00%) | 48/48 (100.00%) | -| `SimdResample.h` | 128/128 (100.00%) | 60/60 (100.00%) | 6/6 (100.00%) | -| `SimdVector.h` | 267/275 (97.09%) | 10/14 (71.43%) | 144/177 (81.36%) | -| `UInt128.h` | 342/409 (83.62%) | 187/264 (70.83%) | 82/95 (86.32%) | -| **Aggregate** | **2,466/3,466 (71.15%)** | **765/1,142 (66.99%)** | **1,438/2,033 (70.73%)** | - -The larger corrected function and branch denominators are intentional. The -old incompatible database discarded or collided mutually exclusive template -and branch records. The corrected LCOV file preserves their union, so these -totals are not directly comparable with the legacy aggregate percentages. - -Direct single-executable `llvm-cov report` checks provided an independent -comparison for the required headers: - -| Header | Executable/profile | Regions | Functions | Lines | Branches | -| --- | --- | ---: | ---: | ---: | ---: | -| `Bmi.h` | `SimdLibTestsBmiPortable` | 67/111 (60.36%) | 23/67 (34.33%) | 173/362 (47.79%) | 28/28 (100.00%) | -| `Api.h` | `SimdLibTests128` | 89/127 (70.08%) | 40/41 (97.56%) | 256/349 (73.35%) | 19/39 (48.72%) | -| `UInt128.h` | `SimdLibTestsUInt128Optimized` | 162/197 (82.23%) | 62/74 (83.78%) | 300/378 (79.37%) | 61/84 (72.62%) | -| `Detail/Implementations.h` | `SimdLibTests128` | 100/104 (96.15%) | 69/70 (98.57%) | 234/248 (94.35%) | 7/7 (100.00%) | - -### Final trustworthy close-out totals - -The final clean-reset run passed 187/187 CTest entries and mapped 190 profiles -to 19 single-executable exports. No multi-executable or tool profile was -included in the final preset run. The LCOV merger now identifies a branch by -its source path, line, block, and branch number, and sums that identity across -executables. This prevents one covered header-template branch from being -reported again as an uncovered copy in every other executable. The final -accumulated report is: - -| Header | Lines | Branches | Functions | -| --- | ---: | ---: | ---: | -| `Api.h` | 407/538 (75.65%) | 51/114 (44.74%) | 944/944 (100.00%) | -| `Bmi.h` | 474/499 (94.99%) | 39/50 (78.00%) | 182/232 (78.45%) | -| `Config.h` | 1/1 (100.00%) | 0/0 | 0/0 | -| `Detail/Extensions.h` | 349/507 (68.84%) | 46/50 (92.00%) | 162/196 (82.65%) | -| `Detail/Implementations.h` | 1,524/1,627 (93.67%) | 42/58 (72.41%) | 787/806 (97.64%) | -| `Format.h` | 224/224 (100.00%) | 152/166 (91.57%) | 20/20 (100.00%) | -| `SimdAlgo.h` | 192/195 (98.46%) | 20/30 (66.67%) | 121/126 (96.03%) | -| `SimdResample.h` | 128/137 (93.43%) | 46/46 (100.00%) | 6/6 (100.00%) | -| `SimdVector.h` | 282/283 (99.65%) | 13/16 (81.25%) | 208/210 (99.05%) | -| `UInt128.h` | 371/409 (90.71%) | 96/108 (88.89%) | 96/102 (94.12%) | -| **Aggregate** | **3,952/4,420 (89.41%)** | **505/638 (79.15%)** | **2,532/2,645 (95.73%)** | - -For `Api.h`, every runtime-profiled alternative is covered: 51/51 (100.00%). -The remaining 63 raw alternatives consist of the constant-evaluation sides of -15 `std::is_constant_evaluated()` gates and 48 branches within their -constant-evaluation-only bodies. The dedicated 128-bit and 256-bit constexpr -targets prove those contracts at compile time, but LLVM runtime profiles cannot -increment their counters. The raw 51/114 total and the classified 51/51 runtime -total are therefore reported together; the latter is a project classification, -not a native LLVM percentage. - -The final `coverage.info` has SHA-256 -`9B07AFE889701BE3670504CFA28FE35CB0AA944C6697C4B952990EB77DC24A2C`. -A clean reset before CTest ensures the report cannot inherit stale profiles. - -### Reviewed red-gutter exclusions - -The table below exhaustively classifies every distinct `DA` line with a zero -count in the final LCOV file. `non-code` includes blank/comment/preprocessor -lines and counterless fully inlined wrapper or `if constexpr` selection sites -whose public callers are directly proved by `ApiOperationMatrix.md`. These are -line-gutter classifications; unhit LCOV branch alternatives remain visible in -the totals and are covered by the compiler/configuration matrix or the same -reviewed compile-time and availability constraints. - -| Header | Zero-count line ranges | Category and reviewed reason | -| --- | --- | --- | -| `Api.h` | 222-227, 579-582, 598-601, 751-764, 778-787, 802-811, 826-835, 850-859, 884-893, 1079-1088, 1102-1112, 1125-1134, 1154-1164, 1183-1193 | constexpr-only: these are the constant-evaluation bodies; dedicated API constexpr targets prove the same contracts. | -| `Bmi.h` | 153-154, 160-161, 203, 232, 266, 289, 342, 387, 776, 816-817, 820, 823, 831, 841, 851, 878-879, 882, 885, 893, 903, 913 | non-code: blank/comment/preprocessor lines and counterless template-selection sites; the selected multiplication bodies and public BMI operations have exhaustive/runtime profiles. | -| `SimdResample.h` | 61, 78, 95, 106, 114, 125, 145, 163, 171 | non-code: blank and preprocessor-alternative lines. | -| `SimdAlgo.h` | 26 | unreachable: `LowBits` is called only for a count below the selected register's lane count, which cannot reach 32. | -| `SimdAlgo.h` | 82, 135 | non-code: LLVM assigns no separate line counter to the terminal return after the loop; exact-register no-match and all-match assertions directly prove both returns. | -| `SimdVector.h` | 112 | non-code: the fully inlined `to_array` assignment has no retained line counter; the signed/unsigned full, partial, odd, and cross-lane `area()` matrix directly executes the reduction. | -| `UInt128.h` | 162, 173, 184, 195, 427, 454, 497, 525 | non-code: preprocessor terminators. | -| `UInt128.h` | 354-356 | constexpr-only: the compatibility `getBlock` contract is asserted in the constexpr snapshot. | -| `UInt128.h` | 409-417, 436-444 | compiler-specific: MSVC carry intrinsics and Clang/GCC overflow builtins are separately selected and proved by the strict compiler profiles; the portable profile cannot execute them. | -| `UInt128.h` | 461, 464-465, 467, 551, 554-555, 558-559 | non-code: counterless `if constexpr` selection and brace lines; boolean/signed shift normalization and all three bitwise selections have direct assertions. | -| `Detail/Extensions.h` | 27-74, 83-130 | compiler-specific: MSVC intrinsic-register union access is preprocessor-excluded from the Clang LCOV build and is covered by the strict MSVC matrix. | -| `Detail/Extensions.h` | 324, 327, 366-368, 372, 377-380, 393, 399, 404-407, 411-413, 417-419, 442-444, 448, 477-479, 512-515, 519-522, 590, 611, 669, 672, 678-681, 717-719, 728, 738, 748, 805-808, 812-815, 915-917 | non-code: comments, blank lines, and counterless fully inlined backend wrappers/selection sites. Their supported public operation/type cells are directly tested at 128 and 256 bits. | -| `Detail/Implementations.h` | 543, 1998, 2000, 2002, 2186, 2188, 2190, 2202, 2204, 2206, 2218, 2220, 2222, 2233, 2235, 2237, 2249, 2251, 2253, 4310-4312, 4314-4315 | non-code: counterless inlined/template selection sites; the selected public extrema, construction, and bitwise cells are directly tested for every supported lane family. | -| `Detail/Implementations.h` | 1993-1995, 2012-2014, 2024-2026, 2036-2040, 4305-4307, 4324-4326, 4336-4338, 4348-4352 | constexpr-only: 128/256-bit construction bodies are proved by the dedicated constexpr targets. | -| `Detail/Implementations.h` | 1389-1400, 2694-2717, 2887-2901 | intentionally unsupported: inherited signed-64 adjacent multiplication and integer square-root backend helpers are not supported public operation/type cells. They remain subject to the post-plan unavailable-area API review rather than being promoted through tests. | - -### Historical audit totals (legacy incompatible merge) - -The following before/after table belongs to the original audit. It used the -single incompatible profile database that produced `621 functions have -mismatched data`; retain it as historical directional evidence only. - -| Header | Regions before | Regions after | Functions before | Functions after | Lines before | Lines after | Branches before | Branches after | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| `Api.h` | 68.75% | 71.83% | 97.92% | 98.18% | 69.21% | 71.79% | 48.72% | 50.00% | -| `Bmi.h` | 67.33% | 68.21% | 32.84% | 34.33% | 45.80% | 47.21% | 100.00% | 100.00% | -| `Format.h` | 98.06% | 98.06% | 100.00% | 100.00% | 94.64% | 94.64% | 86.75% | 87.35% | -| `SimdAlgo.h` | 91.53% | 92.96% | 100.00% | 100.00% | 97.06% | 97.42% | 64.29% | 77.27% | -| `SimdResample.h` | 100.00% | 100.00% | 100.00% | 100.00% | 100.00% | 100.00% | 100.00% | 100.00% | -| `SimdVector.h` | 98.44% | 92.55% | 100.00% | 98.31% | 100.00% | 97.14% | 75.00% | 90.00% | -| `UInt128.h` | 82.11% | 85.07% | 85.14% | 89.19% | 78.51% | 83.07% | 80.00% | 77.17% | -| `Detail/Extensions.h` | 52.94% | 60.50% | 24.32% | 36.49% | 21.14% | 27.42% | 100.00% | 100.00% | -| `Detail/Implementations.h` | 90.04% | 91.87% | 89.19% | 91.94% | 67.98% | 74.06% | 100.00% | 100.00% | -| Aggregate | 82.25% | 84.43% | 74.20% | 79.17% | 65.08% | 69.46% | 84.35% | 84.40% | - -The lower percentage for `SimdVector` is caused by instantiating previously -unseen members, which increased the denominator; the new signed-tail, -min/max-position, area, floating equality, and hashing branches are -directly exercised. UInt128's aggregate branch percentage is similarly -affected by merging mutually exclusive optimized and scalar profiles. - -The raw profiles, merged `coverage.profdata`, and exported `coverage.info` are -generated artifacts under `build-coverage` and are intentionally not -source-controlled. The historical -`baseline.profdata` and `final.profdata` used for the table above were likewise -generated artifacts rather than source-controlled inputs. - -## Final validation record - -All final runs used CMake/CTest 4.4.0. The MSVC tree used MSVC -19.44.35222.0 with the Visual Studio 17 2022 generator. The clang-cl Release, -Clang coverage Debug, and Clang ASan/UBSan Debug trees used LLVM 22.1.8 and -Ninja. Every tree enabled strict warnings and examples; benchmarks were -excluded from correctness runs. The sanitizer tree intentionally omitted the -optional compiler-feature profiles. +### Execution evidence -```powershell -cmake --build build --config Release --parallel -ctest --test-dir build -C Release --output-on-failure -cmake --build build-phase9-clangcl-ninja --parallel -ctest --test-dir build-phase9-clangcl-ninja --output-on-failure -cmake --build build-coverage --parallel -cmake --build build-coverage --target SimdLibCoverageReset -ctest --preset coverage --output-on-failure -cmake --build build-coverage --target SimdLibCoverageReport -$env:PATH='C:\Program Files\LLVM\lib\clang\22\lib\windows;' + $env:PATH -cmake --build build-phase8-sanitize --parallel -ctest --test-dir build-phase8-sanitize --output-on-failure -``` +Coverage percentages, test and profile counts, elapsed times, generated-file +hashes, compiler and tool versions, and line-number-specific exclusion reviews are +execution evidence. Keep them in the generated reports and artifacts below the +owning fingerprint rather than duplicating them as enduring claims in this +coverage contract. -| Matrix | Result | Catch2 cases/assertions | Measured CTest wall time | CTest log | -| --- | ---: | ---: | ---: | --- | -| strict MSVC Release | 179/179 | 156 / 4,324,488 | 4.175 s | `build/Testing/Temporary/LastTest.log` | -| strict clang-cl Release | 182/182 | 159 / 4,435,080 | 2.583 s | `build-phase9-clangcl-ninja/Testing/Temporary/LastTest.log` | -| Clang Debug coverage | 182/182 | same 159 discovered Catch2 cases | 1.321 s | `build-coverage/Testing/Temporary/LastTest.log` | -| Clang ASan/UBSan Debug | 146/146, no diagnostics | optional profiles intentionally omitted | 6.099 s | `build-phase8-sanitize/Testing/Temporary/LastTest.log` | - -The Catch2 totals are the sum of every `SimdLibTests*.exe` compact summary with -`--rng-seed 1592594996`. `SimdLibPreconditionTests.exe` is intentionally -excluded because it terminates after its selected contract case; its 13 -independently discovered CTest entries remain part of the CTest totals. The -aggregate intentionally counts repeated portable, intrinsic, carry, scalar, -checks-enabled, SSE, and AVX2 profiles because those profiles are separate -behavioral evidence. The remaining CTest entries cover -header isolation, configuration/availability probes, formatter ODR, five -result-set equivalence runs, 13 isolated precondition failures, the public -example, the public-header assertion audit, and the constexpr target group. -All required portable, scalar-only, FMA on/off, BMI1-only, BMI2-only, -BMI1+BMI2, SSE4.2, and AVX2 profiles are present in the complete release and -coverage matrices. - -Both freshly configured external consumers pass 1/1: MSVC in 0.084 s at -`build-phase9-consumer-msvc/Testing/Temporary/LastTest.log`, and clang-cl in -0.063 s at -`build-phase9-consumer-clangcl/Testing/Temporary/LastTest.log`. The clang-cl -consumer reports the expected ignored `[[msvc::flatten]]` vendor-attribute -diagnostics; SimdLib's strict clang-cl targets apply the documented private -suppression and are warning-clean. - -Focused `clang-format --dry-run --Werror` passes for the two newly added -precondition sources after applying the checked-in style. `clang-tidy` 22.1.8 -passes those sources; its only diagnostics are -`bugprone-throwing-static-initialization` reports originating from Catch2's -`TEST_CASE` registration macro. The configure/build assertion audit validates -48 production-header occurrences against 30 reviewed allowlist entries. A -source audit over `tests` and `examples` finds no `SimdLib::Detail`, -direct `Detail` include, or backend-routing reference. All dedicated constexpr -profiles build in both complete release matrices and in the Clang coverage -matrix. - -## Consumer-header compile-time comparison - -The final measurement repeats the method in `ConstexprCompilerEvidence.md`: -one header and the same empty `extern "C"` anchor, Clang 22.1.8, -`-std=c++20 -O2 -msse4.2 -mavx2`, a discarded warm-up, and the median of 15 -clean object compiles. Generated fixtures and objects remain under the ignored -`build-phase9-compile-time` directory. - -| Header | Extraction baseline | Final median | Change | -| --- | ---: | ---: | ---: | -| `Bmi.h` | 271.48 ms | 255.33 ms | -5.95% | -| `UInt128.h` | 509.06 ms | 441.86 ms | -13.20% | -| `SimdLib.h` | 527.17 ms | 515.44 ms | -2.23% | - -No measured consumer header regressed against the extraction baseline. - -## VS Code coverage integration - -VS Code CMake Tools 1.23.52 is installed and recommended by -`.vscode/extensions.json`. The workspace enables CTest Test Explorer -integration, resets coverage before a run, generates the target-aware report -afterward, and imports exactly -`${workspaceFolder}/build-coverage/coverage.info`. The installed extension registers these exact settings; its LCOV handler reads -each configured file, constructs native scode.FileCoverage records for -lines, branches, and functions, and calls TestRun.addCoverage. Parsing the -same imported file produces the per-header and aggregate totals recorded above. -The command-line environment cannot inspect pixels in the native Test Coverage -view, so this check proves the provider/import contract and data agreement -without claiming a manual GUI observation. - -## Reviewed remaining gaps - -The earlier statement that no unresolved high-risk correctness gap remains is -consistent with the completed evidence: every supported operation/type cell in -`ApiOperationMatrix.md` has a direct public test, and every zero-count source -line is classified above. The remaining items are reviewed API-design or -lower-risk expansion work rather than known correctness defects: +LLVM runtime profiles cannot increment constant-evaluation-only branches. +Compile-time probes therefore own those contracts, while compiler-specific +runtime branches remain assigned to their corresponding compiler cells. The +generated LCOV report remains authoritative for the exact line, branch, and +function totals of a particular run. + +Consumer-header compile-time measurements are also execution evidence rather +than correctness gates. Their method and results belong in the validation record +for the run that produced them. + +Generated `.profraw`, `.profdata`, LCOV, binary, object, log, and temporary +analysis files remain ignored and untracked. + +## VS Code coverage inspection + +The workspace recommends VS Code CMake Tools through `.vscode/extensions.json` +and keeps CTest Test Explorer integration enabled. Coverage generation is owned +by the formal fingerprinted command rather than a static workspace path. After +that command completes, an LCOV-capable editor extension can open the current +receipt's `coverage.info`. Execution totals come from that generated LCOV file; +editor rendering is not validation evidence. + +## Coverage expansion policy + +Every supported operation/type cell in `ApiOperationMatrix.md` requires a +direct public test. Generated zero-count source lines must be classified in the +execution evidence for the run that produced them. Candidate expansion areas +include: - inherited backend names that are not supported public operation/type cells; - conversion rounding/overflow and direct-transform overlap behavior beyond the current documented cases; -- convenience overloads whose behavior currently delegates to directly tested +- convenience overloads whose behavior delegates to directly tested core operations; and -- the explicitly planned review of operation/type cells marked `unavailable` - after the current coverage plan, before deciding whether any should gain an +- review of operation/type cells marked `unavailable` before deciding whether + any should gain an implementation. - -Generated `.profraw`, `.profdata`, LCOV, binary, object, log, and temporary -analysis files remain ignored and untracked. The final source diff is limited -to formatter normalization of the two new precondition tests plus this -close-out documentation and planning evidence. diff --git a/docs/TestCoverageExpansion.todo b/docs/TestCoverageExpansion.todo index 60a9bb9..10938e9 100644 --- a/docs/TestCoverageExpansion.todo +++ b/docs/TestCoverageExpansion.todo @@ -64,7 +64,7 @@ SimdLib Test Coverage Expansion: ☒ Add a 128/256-bit `min_position` and `max_position` matrix for every supported integer lane type. ☒ Cover first/last extrema, duplicate extrema, signed minima/maxima, unsigned high-bit values, and first-position tie semantics. ☒ Add public coverage for `uint64_t::multiply_add_adjacent` and verify its exact lane ordering and overflow contract. - ☒ Add missing public tests for floating `set1`, floating bitwise operations, and 128/256-bit `get_element`/`set_element` behavior. + ☒ Add missing public tests for floating `set1`, floating bitwise operations, and 128/256-bit runtime-selected `extract`/`insert` behavior. ☒ Add exact 64-bit-result and native-word-boundary-crossing cases for `transform_pack`, including a final partial output word and canary-protected destination storage. ☒ Avoid direct `Detail/Extensions.h` or `Detail/Implementations.h` tests when a public call can provide the same proof. ☒ End Phase 2 only when every supported public specialization is represented in the operation/type matrix and the previously uncovered backend families are reached through that matrix or explicitly justified. @@ -83,13 +83,13 @@ SimdLib Test Coverage Expansion: ☒ Compile the dedicated UInt128 constexpr sources under optimized carry, portable carry, and scalar-only configurations. ☒ Compile the dedicated `Api`/vector constexpr sources under the supported SSE/AVX and disabled-feature profiles. ☒ Ensure compile-time test failures are visible through CTest/CMake target output with the source file and assertion expression that failed. - ☒ Add a mechanical audit that rejects new test-example `static_assert` blocks in public headers unless the assertion is allowlisted with an invariant/constraint justification. + ☒ Classify retained production-header `static_assert` declarations as constraints, diagnostics, or invariants after migrating test examples; the temporary textual allowlist was retired after this extraction. ☒ Measure clean compile time for minimal translation units including `Bmi.h`, `UInt128.h`, and `SimdLib.h` before and after extraction using the same compiler, flags, and repeated-run method. ☒ Record preprocessing size and compiler front-end timing where supported, and verify the extraction does not increase consumer compile time or introduce additional emitted code. ☒ Verify the move does not change public declarations, constraints, diagnostics for invalid instantiations, ABI/layout, or runtime behavior. ☒ Create constexpr contract helpers that can be reused by compile-only probes and runtime parity tests without relying on runtime coverage counters for constant evaluation. - ☒ Expand `Api` `static_assert`/`consteval` coverage for `setzero`, `setr`, `construct`, `set1`, `to_array`, `get_element`, and `set_element` at 128 and 256 bits. - ☒ Expand constexpr comparison coverage for `cmp_eq`, `cmp_eq_mask`, `cmp_gt`, `cmp_ge`, `cmp_lt`, `cmp_le`, and the internal comparison operation choices reached by them. + ☒ Expand `Api` `static_assert`/`consteval` coverage for `setzero`, `setr`, `construct`, `set1`, `to_array`, runtime-selected `extract`, and runtime-selected `insert` at 128 and 256 bits. + ☒ Expand constexpr comparison coverage for native `compare_*`, byte-granular `cmp_*_mask`, lane-granular `cmp_*_slim`, and the internal comparison operation choices reached by them. ☒ Expand constexpr `movemask` and `movemask_slim` beyond the current representative types to signed, unsigned, float, and double lane families at both widths. ☒ Add constexpr `min_position`, `max_position`, lane-shift, and whole-register-shift boundary checks. ☒ Add runtime parity checks constructed from volatile inputs so optimized runtime paths cannot be satisfied solely by compile-time folding. @@ -169,7 +169,7 @@ SimdLib Test Coverage Expansion: ☒ Reconcile the old statement that no unresolved high-risk coverage gap remains with the evidence produced by this todo. ☒ Record every excluded red gutter as constexpr-only, compiler-specific, unreachable, non-code, or intentionally unsupported; do not leave unexplained exclusions. ☒ Verify a source audit finds no unjustified direct `SimdLib::Detail` usage or backend implementation routing in tests or shared test support. - ☒ Verify a source audit finds no unallowlisted test-example `static_assert` blocks in public headers and that all dedicated constexpr targets participate in the documented compiler/configuration matrix. + ☒ Verify that public-header probes and dedicated constexpr targets compile the retained constraints and migrated test examples in the documented compiler/configuration matrix. ☒ Compare the final consumer-header compile-time measurements with the Phase 3 baseline and record the result in `docs/TestCoverage.md`. ☒ Verify the VS Code Test Coverage view imports the final `coverage.info` and agrees with the documented per-header totals. ☒ Verify `git diff --check` passes and no generated profiles, LCOV reports, binaries, logs, or temporary analysis files are tracked. @@ -191,7 +191,7 @@ SimdLib Test Coverage Expansion: ☒ Phase 0 corrected coverage pipeline, clean warning output, object/profile provenance, and corrected baseline totals recorded. ☒ Phase 1 BMI contract decisions, helper matrix, deterministic inputs, and portable/intrinsic equivalence results recorded. ☒ Phase 2 public `Api` operation/type matrix and backend-family reachability results recorded: no direct `Detail` test routes remain; focused MSVC Release and Clang coverage runs pass 33/33 tests, and the complete MSVC Release suite passes 137/137 tests. - ☒ Phase 3 migrated-header assertion inventory, retained-invariant justifications, dedicated constexpr-target matrix, consumer compile-time measurements, and separate MSVC/Clang/compiler-path evidence recorded in `docs/StaticAssertionInventory.md` and `docs/ConstexprCompilerEvidence.md`; strict MSVC Release passes 144/144, Clang passes 147/147, and the external consumer passes 1/1. + ☒ Phase 3 migrated test-example assertions into dedicated constexpr targets, retained production constraints and diagnostics in their owning headers, and recorded durable constexpr target/profile ownership in `docs/TestCoverage.md`; the completed execution also measured consumer compile time and validated the separate MSVC/Clang compiler paths, with strict MSVC Release passing 144/144, Clang passing 147/147, and the external consumer passing 1/1. ☒ Phase 4 `SimdAlgo` full-register/tail outcome matrix recorded in `docs/TestCoverage.md`: focused MSVC and Clang runs pass 7/7 tests with 614 assertions, the strict MSVC suite passes 147/147, and the Clang suite passes 150/150. ☒ Phase 5 accepted/rejected formatter grammar matrix recorded in `docs/TestCoverage.md`: focused MSVC and Clang runs pass 8/8 tests, the formatter suite passes 267 assertions, the strict MSVC suite passes 148/148, and the Clang suite passes 151/151. ☒ Phase 6 `uint128_t` boundary and compatibility matrix recorded in `docs/TestCoverage.md`: focused MSVC passes 35/35, focused Clang passes 38/38, each Clang profile passes 131 assertions across six boundary cases, the strict MSVC suite passes 154/154, and the Clang suite passes 157/157. diff --git a/docs/Validation.md b/docs/Validation.md deleted file mode 100644 index 70c02b3..0000000 --- a/docs/Validation.md +++ /dev/null @@ -1,105 +0,0 @@ -# Validation evidence - -Validation was completed on 2026-07-17 with strict warnings enabled for every -SimdLib-owned target. Each full configuration built the configuration and -constexpr probes, first-and-only header probes, multi-translation-unit ODR -smoke test, API example, portable and optimized UInt128 variants, scalar and -SIMD resampling paths, FMA enabled/disabled paths, and all BMI1/BMI2 profiles. - -## Local compiler matrix - -| Compiler | Target | Configuration | Result | -| --- | --- | --- | --- | -| MSVC 19.44 | x64 | Debug, Release | 19/19 tests passed in each configuration | -| MSVC 19.44 | x86 | Debug, Release | 19/19 tests passed in each configuration | -| clang-cl 22.1.8 | x64 | Debug, Release | 19/19 tests passed in each configuration | -| clang-cl 22.1.8 | x86 | Debug, Release | 19/19 tests passed in each configuration | -| Clang 22.1.8 | x64 | Release | 19/19 tests passed | -| Clang 22.1.8 | x86 | Debug, Release | 19/19 tests passed in each configuration | -| GCC 13.2 | x64 | Debug, Release | 19/19 tests passed in each configuration | - -The local MinGW GCC installation is x64-only and cannot link `-m32` because it -has no 32-bit UCRT/import libraries or multilib. The Linux CI x86 jobs install -`g++-multilib` explicitly, so x86 GCC and Clang remain part of the committed CI -contract rather than being silently omitted. - -Clang ASan and UBSan validation used Debug symbols, `-O1`, frame pointers, and -strict warnings. All 13 runtime tests passed with no sanitizer diagnostics. -On Windows, the release CRT was selected for this run because Clang ASan and -the MSVC Debug CRT allocator instrumentation are incompatible. - -The only intentionally suppressed diagnostics are unsupported/ignored vendor -attributes, Clang's diagnostic for Catch2's `__COUNTER__` extension use, and -compiler SIMD-register template attributes. The exact warning switches are -documented in [CompilerConfiguration.md](../cmake/CompilerConfiguration.md). - -## Header-only consumer gate - -`tests/consumer` imports the source tree with `add_subdirectory`, asserts that -the `SimdLib` CMake target is an `INTERFACE_LIBRARY`, and builds only its own -executable. The MSVC, Clang, and GCC Release consumer configurations each pass -their 1/1 CTest smoke test and produce no SimdLib library binary. - -## Namespace stabilization gate - -SimdLib 0.2.0 was revalidated after adopting root `Api`, keeping root -`SimdVector` and `uint128_t`, and consolidating wide-integer operations under -`Bmi`. The final MSVC 19.44 strict Release matrix under -`build-m12-msvc` passes 19/19 CTest entries. The external clang-cl 22.1.8 -strict Release matrix under `build-m12-clang-ninja` also passes 19/19 entries; -its direct Catch executables cover 74 cases and 4,228,913 assertions. Both -matrices compile 12 first-and-only public-header probes, the availability and -configuration probes, the multi-translation-unit smoke executable, all -optional BMI profiles and equivalence checks, and the API example. - -The strengthened external consumer instantiates `Api`, `SimdVector`, `Bmi`, -and `uint128_t` through `SimdLib::SimdLib`. Fresh MSVC and clang-cl consumer -builds each pass their 1/1 CTest entry under `build-m12-consumer-msvc` and -`build-m12-consumer-clang`. The library target remains an -`INTERFACE_LIBRARY`. A configure-time source guard also rejects any public -example, consumer, smoke source, or header probe that names `SimdLib::Detail` -or includes a `Detail` header. - -The representative namespace-stabilization benchmarks passed 1/1 case on -both compilers: - -| Operation | MSVC 19.44 | clang-cl 22.1.8 | -| --- | ---: | ---: | -| `Api` 128-bit add | 0.444812 ns | 0.293360 ns | -| `Api` 256-bit add | 0.444038 ns | 0.428736 ns | -| BMI2 `pext_u64` | 0.222357 ns | 0.203131 ns | -| `uint128_t` add | 0.405140 ns | 0.306990 ns | -| Reduce-by-8 resample | 9.54669 ns | 7.27583 ns | - -The MSVC resample sample was noisy (6.43357 ns standard deviation), and the -compiler/harness difference makes cross-column comparison directional rather -than a regression measurement. Authoritative logs are -`build-m12-msvc/{configure-final,build-final,ctest-final,benchmark-final}.log`, -`build-m12-clang-ninja/{configure,build,ctest,benchmark}.log`, and the -corresponding consumer build directories. - -## Pre-extraction benchmark comparison - -The standalone GCC Release benchmark was sampled 100 times in three runs. The -last stable run is compared with the pre-extraction baseline below. - -| Operation | Pre-extraction | Standalone | Change | -| --- | ---: | ---: | ---: | -| Api 128-bit add | 0.435963 ns | 0.296309 ns | -32.03% | -| Api 256-bit add | 0.470220 ns | 0.303181 ns | -35.52% | -| BMI2 `pext_u64` | 0.204546 ns | 0.280442 ns | +37.10% | -| `uint128_t` add | 0.267830 ns | 0.276382 ns | +3.19% | -| Reduce-by-8 resample | 7.694720 ns | 7.894700 ns | +2.60% | - -The resample case was noisy in its first run, then stabilized at -7.89470-7.90786 ns, so it does not show a material regression. Disassembly -confirmed the expected `vpaddd` instructions in the SIMD cases and compare plus -movemask instructions in the resampler. - -The apparent BMI2 percentage is only 0.075896 ns and is not an intrinsic -regression: disassembly contains no `pext` because the constant-input benchmark -was folded to an immediate result. The pre-extraction case used the same representative -constant-input shape, so this comparison measures sub-nanosecond loop/compiler -overhead. The constant-input UInt128 operation is likewise precomputed. Neither -result warrants an implementation change; future microarchitecture measurement -should use a runtime-generated input corpus. diff --git a/docs/project.todo b/docs/project.todo index 54fe844..89776fc 100644 --- a/docs/project.todo +++ b/docs/project.todo @@ -1,14 +1,27 @@ -☐ Design a `SimdLib::Register` class to represent SIMD registers and provide methods for loading, storing, and manipulating data in a SIMD context. - The Register type should supercede SimdLib::Api as the recommended interface for SIMD operations, providing a more intuitive and efficient way to work with SIMD registers. - This trype will resemble the existing `SimdLib::Vector` class, but it will be much more low-level/restrictive, and will not provide an "element_count" template input, meaning it will not auto fill "inactive lanes" because ALL lanes are considered "active". - -☐ Design a `SimdLib::Tensor` class to represent multi-dimensional arrays (tensors) and provide methods for performing tensor operations in a SIMD context. - The Tensor type should support various data types and dimensions, allowing for efficient manipulation of large datasets in parallel. - It should also facilitate tensors with a templated compile-time fixed size, as well as dynamic size tensors that can be resized at runtime via std::spans. - It should also provide methods for broadcasting, reshaping, and slicing tensors, as well as performing element-wise operations and reductions. - -☐ Ensure test coverage of all `SimdImplementation::negate()` methods. -☐ Review test coverage of all `SimdImplementation` namespace methods. -☐ Improve performance of `Bmi::portable_pdep()`. -☐ Improve performance of `Bmi::portable_pext()`. -☐ Benchmark and Optimize `SimdVector::area()`. \ No newline at end of file +Code Architecture: + ☒ Remove `MethodFlagsInventory.csv` from the repo and audit tooling. + ☒ Remove `MethodFlagsRegisterOnly.csv` from the repo and audit tooling. + ☐ Remove `shuffle_lo` and `shuffle_hi` methods from Register class (to be replaced with generic templated shuffle method). + ☐ Analyze `Implementation::shuffle<...>()` type methods to ensure they handle shuffling optimally, e.g. using `shuffle_lo` and `shuffle_hi` when appropriate, and ensure that the `shuffle<...>()` methods are implemented in a way that is both efficient and maintainable. + ☐ Implement a `SimdLib::ImmMask` class to represent compile-time immediate-mode masks for SIMD intrinsics, providing methods for creating and manipulating masks based on compile-time conditions. This class should be compatible with the `SimdLib::Register` and `SimdLib::Tensor` classes, allowing for efficient lane control in SIMD operations. + + ☐ Design a `SimdLib::Tensor` class to represent multi-dimensional arrays (tensors) and provide methods for performing tensor operations in a SIMD context. + The Tensor type should support various data types and dimensions, allowing for efficient manipulation of large datasets in parallel. + It should also facilitate tensors with a templated compile-time fixed size, as well as dynamic size tensors that can be resized at runtime via std::spans. + It should also provide methods for broadcasting, reshaping, and slicing tensors, as well as performing element-wise operations and reductions. + +Testing: + ☐ Ensure test coverage of all `SimdImplementation::negate()` methods. + ☐ Review test coverage of all `SimdImplementation` namespace methods. + ☐ Review test coverage for Api layer runtime methods. + ☐ Review test coverage for Api layer compile-time methods. + +Performance: + ☐ Analyze if there is a more optimal implementation for `SimdImplementation::magnitude()`. + ☐ Improve performance of `Bmi::portable_pdep()`. + ☐ Improve performance of `Bmi::portable_pext()`. + ☐ Benchmark and Optimize `SimdVector::area()`. + +Compiler Support: + ☐ Add Intel oneAPI DPC++/C++ Compiler (ICX/ICPX) as an explicitly supported toolchain, including compiler detection, strict-warning builds, runtime tests, external-consumer coverage, and Register ABI/generated-code validation. + ☐ Add NVIDIA HPC SDK NVC++ as an explicitly supported toolchain, including dedicated compiler detection, x86-family intrinsic coverage on x64, C++23 Register availability, compiler-attribute mappings, runtime tests, external-consumer coverage, and generated-code validation. diff --git a/examples/RegisterExamples.cpp b/examples/RegisterExamples.cpp new file mode 100644 index 0000000..d21ed12 --- /dev/null +++ b/examples/RegisterExamples.cpp @@ -0,0 +1,52 @@ +#include + +#include +#include +#include + +namespace +{ +using StableRegister = SimdLib::Register; + +/** + * @brief Demonstrates a stable-width non-inline consumer boundary. + * @param value Input register. + * @return Input lanes increased by one. + */ +StableRegister SIMD_FLAGS(InOut) add_one(StableRegister value) noexcept +{ + return value + StableRegister::broadcast(1.0F); +} +} // namespace + +/** + * @brief Exercises complete-register and RegisterMask workflows from the umbrella header. + * @return Zero when every example result satisfies its contract. + */ +int main() +{ + using Register = SimdLib::NativeRegister; + using Mask = Register::mask_type; + + const Register values = Register::broadcast(2.0F); + const Register threshold = Register::broadcast(1.0F); + const Mask greater = values.compare_greater(threshold); + const Mask equal = values.compare_equal(values); + const Mask selected_lanes = greater & equal; + const typename Mask::native_type observed_predicate = selected_lanes.native; + const Mask restored_predicate{observed_predicate}; + const Register selected = restored_predicate.select(values, Register::zero()); + if (!restored_predicate.any() || !restored_predicate.all() || restored_predicate.none() || restored_predicate.bits() == 0 || selected != values) + return 1; + + const Register nan = Register::broadcast(std::numeric_limits::quiet_NaN()); + if (nan.compare_equal(nan).any()) + return 2; + + const Register positive_zero = Register::broadcast(0.0F); + const Register negative_zero = Register::broadcast(-0.0F); + if (!positive_zero.compare_equal(negative_zero).all()) + return 3; + + return add_one(StableRegister::broadcast(4.0F)) == StableRegister::broadcast(5.0F) ? 0 : 4; +} diff --git a/include/SimdLib/Aliases.h b/include/SimdLib/Aliases.h new file mode 100644 index 0000000..fbcf2d0 --- /dev/null +++ b/include/SimdLib/Aliases.h @@ -0,0 +1,61 @@ +#pragma once + +#include + +#include + +namespace SimdLib +{ + +#if SIMDLIB_HAS_SSE42 +#pragma region Vector Types + +// TODO: Remove these "Vector..." aliases in favor of the more descriptive "int8x16" style aliases below. +using VectorInt8 = Register; +using VectorUInt8 = Register; + +using VectorInt16 = Register; +using VectorUInt16 = Register; + +using VectorInt32 = Register; +using VectorUInt32 = Register; + +#if SIMDLIB_HAS_AVX2 +using VectorInt64 = Register; +using VectorUInt64 = Register; +#endif + +#pragma endregion + +#pragma region Type Aliases (Unsigned) + +using uint8x16 = Register; +using uint16x8 = Register; +using uint32x4 = Register; +using uint64x2 = Register; +#if SIMDLIB_HAS_AVX2 +using uint8x32 = Register; +using uint16x16 = Register; +using uint32x8 = Register; +using uint64x4 = Register; +#endif + +#pragma endregion + +#pragma region Type Aliases (Signed) + +using int8x16 = Register; +using int16x8 = Register; +using int32x4 = Register; +using int64x2 = Register; +#if SIMDLIB_HAS_AVX2 +using int8x32 = Register; +using int16x16 = Register; +using int32x8 = Register; +using int64x4 = Register; +#endif + +#pragma endregion + +#endif +} // namespace SimdLib diff --git a/include/SimdLib/Api.h b/include/SimdLib/Api.h index 46433bc..208f149 100644 --- a/include/SimdLib/Api.h +++ b/include/SimdLib/Api.h @@ -1,6 +1,7 @@ #pragma once -#include #include +#include +#include #include #include #include @@ -11,15 +12,14 @@ #include #include #include +#if SIMDLIB_COMPILER_MSVC && SIMDLIB_TARGET_X86 #include +#endif #include #include #include #include -// This file contains SIMD implementations for 128-bit and 256-bit integer and floating-point types. -// REFERENCE: http://www.alfredklomp.com/programming/sse-intrinsics/ - namespace SimdLib { @@ -32,20 +32,8 @@ enum class comparison_operation equivalent, unordered, }; -} // namespace Detail - -template -inline constexpr bool is_api_available_v = - (std::same_as || std::same_as || - std::same_as || std::same_as || - std::same_as || std::same_as || - std::same_as || std::same_as || - std::same_as || std::same_as) && - Config::target_x86 && - ((register_width == 128 && Config::has_sse42) || (register_width == 256 && Config::has_sse42 && Config::has_avx2)); -template -concept ApiAvailable = is_api_available_v; +} // namespace Detail /** * @brief Primary API for SIMD operations, parameterized by register width and element type. @@ -104,32 +92,35 @@ struct Api : public Detail::SimdMappings (source_count * result_bit_width + std::numeric_limits>::digits - 1) / std::numeric_limits>::digits; - template - constexpr static inline bool is_widen_target_v = requires { - typename target_simd::element_type; - typename target_simd::vector_t; - { target_simd::register_width } -> std::convertible_to; - }; - #pragma region Data Transfer /** @brief Loads element data into a SIMD register. * @param data Source elements matching the full register width. * @return Register populated with the provided elements. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL load(std::span data) noexcept + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load(std::span data) noexcept { return impl::load_unaligned(data.data()); } + /** + * @brief Loads one complete register bit pattern from an exact byte span. + * @param data Source containing exactly one register of bytes. + * @return Register containing the source object representation. + */ + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load(std::span data) noexcept + { + return impl::load_bytes(data.data()); + } + /** @brief Loads a full register from storage aligned to the register byte width. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL load_aligned(std::span data) noexcept + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_aligned(std::span data) noexcept { SIMDLIB_PRECONDITION(reinterpret_cast(data.data()) % byte_count == 0, "Aligned SIMD load requires register-width alignment"); return impl::load(data.data()); } /** @brief Explicit spelling for an unaligned full-register load. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL load_unaligned(std::span data) noexcept + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_unaligned(std::span data) noexcept { return impl::load_unaligned(data.data()); } @@ -140,7 +131,7 @@ struct Api : public Detail::SimdMappings * @return Register containing the requested active values followed by zero-filled inactive lanes. */ template - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL load_partial(std::span data) noexcept + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_partial(std::span data) noexcept requires(active_count <= element_count) { if (!std::is_constant_evaluated()) @@ -163,7 +154,7 @@ struct Api : public Detail::SimdMappings * @param data Source span whose leading elements are read into the register. * @return Register populated from the provided span. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL load_unsafe(std::span data) noexcept + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_unsafe(std::span data) noexcept { return impl::load_unaligned(data.data()); } @@ -173,20 +164,30 @@ struct Api : public Detail::SimdMappings * @param data Destination span that receives all register elements. * @return None. */ - SIMDLIB_FORCE_INLINE static void VECTORCALL store(vector_t vector, std::span data) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store(vector_t vector, std::span data) noexcept + { + impl::store_unaligned(vector, data.data()); + } + + /** + * @brief Stores one complete register bit pattern to an exact byte span. + * @param vector Register value to store. + * @param data Destination containing exactly one register of bytes. + */ + static void SIMD_FLAGS(In, ForceInline, Flatten) store(vector_t vector, std::span data) noexcept { impl::store_unaligned(vector, data.data()); } /** @brief Stores a full register to storage aligned to the register byte width. */ - SIMDLIB_FORCE_INLINE static void VECTORCALL store_aligned(vector_t vector, std::span data) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store_aligned(vector_t vector, std::span data) noexcept { SIMDLIB_PRECONDITION(reinterpret_cast(data.data()) % byte_count == 0, "Aligned SIMD store requires register-width alignment"); impl::store(vector, data.data()); } /** @brief Explicit spelling for an unaligned full-register store. */ - SIMDLIB_FORCE_INLINE static void VECTORCALL store_unaligned(vector_t vector, std::span data) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store_unaligned(vector_t vector, std::span data) noexcept { impl::store_unaligned(vector, data.data()); } @@ -196,7 +197,7 @@ struct Api : public Detail::SimdMappings * @param data Destination byte span with capacity for the full register payload. * @return None. */ - SIMDLIB_FORCE_INLINE static void VECTORCALL store(vector_t vector, std::span data) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store(vector_t vector, std::span data) noexcept { SIMDLIB_PRECONDITION(data.size() >= byte_count, "Data byte span must be at least the byte size of the register"); impl::store_unaligned(vector, data.data()); @@ -206,7 +207,7 @@ struct Api : public Detail::SimdMappings * @param data Source array containing one full register worth of elements. * @return Register populated with the provided array contents. */ - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL construct(const std::array &data) noexcept + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) construct(const std::array &data) noexcept { return impl::construct(data); } @@ -215,59 +216,15 @@ struct Api : public Detail::SimdMappings * @param vector Register value to unpack. * @return Array containing the register elements in lane order. */ - SIMDLIB_FORCE_INLINE constexpr static std::array VECTORCALL to_array(const vector_t vector) noexcept + constexpr static std::array SIMD_FLAGS(In, ForceInline, Flatten) to_array(const vector_t vector) noexcept { - std::array result{}; if (std::is_constant_evaluated()) - { - for (std::size_t index = 0; index < element_count; ++index) - { - result[index] = impl::get_element(vector, static_cast(index)); - } - } - else - { - impl::store_unaligned(vector, result.data()); - } + return to_array_constexpr(vector); + std::array result{}; + impl::store_unaligned(vector, result.data()); return result; } - /** @brief Finishes integer magnitude by summing the SIMD-produced pairwise squares per 128-bit lane and broadcasting the root. - * @tparam partial_element_t Integer lane type produced by the first pairwise square-and-sum step. - * @param pairSums Register containing `x*x + y*y` style partial sums for each 128-bit lane group. - * @return Register containing the lane-local magnitudes broadcast to every source lane. - */ - template - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL - FinishIntegerMagnitudeFromPairSums(typename Api::vector_t pairSums) noexcept - { - using partial_simd = Api; - using accumulation_t = std::conditional_t, int64_t, uint64_t>; - constexpr std::size_t LaneGroupCount = register_width / 128; - constexpr std::size_t SourceLaneWidth = element_count / LaneGroupCount; - constexpr std::size_t PartialLaneWidth = partial_simd::element_count / LaneGroupCount; - - const auto partialValues = partial_simd::to_array(pairSums); - std::array output{}; - for (std::size_t groupIndex = 0; groupIndex < LaneGroupCount; ++groupIndex) - { - accumulation_t total{}; - const std::size_t partialStart = groupIndex * PartialLaneWidth; - for (std::size_t partialOffset = 0; partialOffset < PartialLaneWidth; ++partialOffset) - { - total += static_cast(partialValues[partialStart + partialOffset]); - } - - const element_t laneMagnitude = static_cast(std::round(std::sqrt(static_cast(total)))); - const std::size_t laneStart = groupIndex * SourceLaneWidth; - for (std::size_t laneOffset = 0; laneOffset < SourceLaneWidth; ++laneOffset) - { - output[laneStart + laneOffset] = laneMagnitude; - } - } - - return construct(output); - } #pragma endregion #pragma region Arithmetic Operations @@ -275,8 +232,8 @@ struct Api : public Detail::SimdMappings /** @brief Returns a zero-initialized SIMD register. * @return Register with every lane initialized to zero. */ - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL setzero() noexcept - requires requires { impl::setzero(); } + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) setzero() noexcept + requires IImpl::SetZero { return impl::setzero(); } @@ -285,8 +242,8 @@ struct Api : public Detail::SimdMappings * @param value Scalar value to broadcast. * @return Register with every lane initialized to `value`. */ - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL set1(const element_t value) noexcept - requires requires(element_t scalar) { impl::set1(scalar); } + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) set1(const element_t value) noexcept + requires IImpl::SetOne { return impl::set1(value); } @@ -297,8 +254,8 @@ struct Api : public Detail::SimdMappings * @return Register containing the provided lane values. */ template - SIMDLIB_FORCE_INLINE constexpr static auto VECTORCALL set(Args &&...args) noexcept - requires requires(Args &&...values) { impl::set(std::forward(values)...); } + constexpr static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) set(Args &&...args) noexcept + requires IImpl::Set { return impl::set(std::forward(args)...); } @@ -309,11 +266,11 @@ struct Api : public Detail::SimdMappings * @return Register containing the provided lanes with any remaining lanes initialized to zero. */ template - SIMDLIB_FORCE_INLINE constexpr static auto VECTORCALL set_partial(Args &&...args) noexcept + constexpr static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) set_partial(Args &&...args) noexcept requires(sizeof...(Args) <= element_count) { return [](std::index_sequence, Args &&...values) constexpr noexcept - requires requires(Args &&...forwardedValues) { impl::set(std::forward(forwardedValues)..., ((void)ZeroIndices, element_t{})...); } + requires IImpl::Set { return impl::set(std::forward(values)..., ((void)ZeroIndices, element_t{})...); }(std::make_index_sequence{}, std::forward(args)...); } @@ -324,8 +281,8 @@ struct Api : public Detail::SimdMappings * @return Register containing the provided lane values. */ template - SIMDLIB_FORCE_INLINE constexpr static auto VECTORCALL setr(Args &&...args) noexcept - requires requires(Args &&...values) { impl::setr(std::forward(values)...); } + constexpr static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) setr(Args &&...args) noexcept + requires IImpl::SetReverse { return impl::setr(std::forward(args)...); } @@ -336,11 +293,11 @@ struct Api : public Detail::SimdMappings * @return Register containing the provided lanes with any remaining lanes initialized to zero. */ template - SIMDLIB_FORCE_INLINE constexpr static auto VECTORCALL setr_partial(Args &&...args) noexcept + constexpr static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) setr_partial(Args &&...args) noexcept requires(sizeof...(Args) <= element_count) { return [](std::index_sequence, Args &&...values) constexpr noexcept - requires requires(Args &&...forwardedValues) { impl::setr(std::forward(forwardedValues)..., ((void)ZeroIndices, element_t{})...); } + requires IImpl::SetReverse { return impl::setr(std::forward(values)..., ((void)ZeroIndices, element_t{})...); }(std::make_index_sequence{}, std::forward(args)...); } @@ -351,8 +308,8 @@ struct Api : public Detail::SimdMappings * @param addend Register added to the product. * @return Register containing the multiply-add result. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add(const vector_t lhs, const vector_t rhs, const vector_t addend) noexcept - requires requires(vector_t left, vector_t right, vector_t sum) { impl::multiply_add(left, right, sum); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add(const vector_t lhs, const vector_t rhs, const vector_t addend) noexcept + requires IImpl::MultiplyAdd { return impl::multiply_add(lhs, rhs, addend); } @@ -362,26 +319,17 @@ struct Api : public Detail::SimdMappings * @param lhs Source register to widen. * @return Destination register widened according to the source element signedness. */ - template SIMDLIB_FORCE_INLINE static typename target_simd::vector_t VECTORCALL widen(const vector_t lhs) noexcept + template + requires IApi::WidenTarget && using_int && std::is_integral_v && + (std::is_signed_v == std::is_signed_v) && + (sizeof(element_t) < sizeof(typename target_simd::element_type)) && (register_width == 128) && + (target_simd::register_width == 128 || target_simd::register_width == 256) && + ApiAvailable && IImpl::Widen + constexpr static typename target_simd::vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) widen(const vector_t lhs) noexcept { - static_assert(is_widen_target_v, - "Api::widen requires a destination SIMD type with element_type, vector_t, and register_width."); - static_assert(using_int, "Api::widen only supports integral source SIMD specializations."); - static_assert(std::is_integral_v, "Api::widen only supports integral destination SIMD specializations."); - static_assert(sizeof(element_t) < sizeof(typename target_simd::element_type), - "Api::widen requires the destination element type to be wider than the source element type."); - static_assert(target_simd::register_width == 128 || target_simd::register_width == 256, - "Api::widen currently supports only 128-bit or 256-bit destination SIMD widths."); - - if constexpr (requires(vector_t value) { impl::template widen(value); }) - { - return impl::template widen(lhs); - } - else - { - static_assert(requires(vector_t value) { impl::template widen(value); }, - "Api::widen does not yet have a backend mapping for this source/destination SIMD pair."); - } + if (std::is_constant_evaluated()) + return widen_constexpr(lhs); + return impl::template widen(lhs); } /** @brief Computes the remainder of each lhs element divided by the corresponding rhs element. @@ -389,8 +337,8 @@ struct Api : public Detail::SimdMappings * @param rhs Divisor register. * @return Register containing per-lane remainder results. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL modulus(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::modulus(left, right); } + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::Modulus { return impl::modulus(lhs, rhs); } @@ -399,8 +347,8 @@ struct Api : public Detail::SimdMappings * @param lhs Input register. * @return Register containing the negated element values. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL negate(const vector_t lhs) noexcept - requires requires(vector_t value) { impl::negate(value); } + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) negate(const vector_t lhs) noexcept + requires IImpl::Negate { return impl::negate(lhs); } @@ -409,8 +357,8 @@ struct Api : public Detail::SimdMappings * @param lhs Input register. * @return Register containing per-lane absolute values. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL absolute(const vector_t lhs) noexcept - requires requires(vector_t value) { impl::absolute(value); } + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(const vector_t lhs) noexcept + requires IImpl::Absolute { return impl::absolute(lhs); } @@ -419,69 +367,38 @@ struct Api : public Detail::SimdMappings * @param lhs Input register. * @return Register containing per-lane square roots. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(const vector_t lhs) noexcept - requires requires(vector_t value) { impl::sqrt(value); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(const vector_t lhs) noexcept + requires IImpl::Sqrt { return impl::sqrt(lhs); } - /** @brief Computes the vector magnitude per 128-bit lane. - * @param lhs Input register. - * @return Register containing the lane-local magnitudes broadcast within each 128-bit lane. + /** @brief Computes the vector magnitude independently for each 128-bit group. + * @param lhs Input register. Integer inputs require a magnitude representable by `element_t`. + * @return Floating magnitudes broadcast within each group, or unchecked integer magnitudes in each group-leading lane. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL magnitude(const vector_t lhs) noexcept - requires((std::is_floating_point_v && requires(vector_t left, vector_t right) { - impl::sqrt(left); - impl::template dot_product<0x11>(left, right); - }) || (using_int && requires(vector_t value) { - impl::sqrt(value); - impl::multiply_add_adjacent(value, value); - })) + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(const vector_t lhs) noexcept + requires IImpl::Magnitude { - if constexpr (std::is_floating_point_v) - { - if constexpr (std::same_as) - { - return sqrt(dot_product<0xFF>(lhs, lhs)); - } - else - { - return sqrt(dot_product<0x33>(lhs, lhs)); - } - } - else - { - if constexpr (sizeof(element_t) == 1) - { - using partial_element_t = std::conditional_t; - return FinishIntegerMagnitudeFromPairSums(multiply_add_adjacent(lhs, lhs)); - } - else if constexpr (sizeof(element_t) == 2) - { - using partial_element_t = std::conditional_t; - return FinishIntegerMagnitudeFromPairSums(multiply_add_adjacent(lhs, lhs)); - } - else if constexpr (sizeof(element_t) == 4) - { - using partial_element_t = std::conditional_t; - return FinishIntegerMagnitudeFromPairSums(multiply_add_adjacent(lhs, lhs)); - } - else - { - return FinishIntegerMagnitudeFromPairSums(multiply_add_adjacent(lhs, lhs)); - } - } + return impl::magnitude(lhs); + } + + /** @brief Computes saturated integer magnitudes with canonical overflow masks. + * @param lhs Input integer register. + * @return Each 128-bit group stores its magnitude in lane zero and a zero/all-ones overflow mask in lane one. + */ + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(const vector_t lhs) noexcept + requires(using_int && IImpl::MagnitudeChecked) + { + return impl::magnitude_checked(lhs); } /** @brief Normalizes floating-point lanes using the vector length computed per 128-bit lane. * @param lhs Input floating-point register. * @return Register containing the normalized per-lane values. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL normalize(const vector_t lhs) noexcept - requires(std::is_floating_point_v && requires(vector_t left, vector_t right) { - magnitude(left); - impl::divide(left, right); - }) + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) normalize(const vector_t lhs) noexcept + requires(std::is_floating_point_v && IImpl::Normalize) { return divide(lhs, magnitude(lhs)); } @@ -491,8 +408,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing per-lane averages. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL avg(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::avg(left, right); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) avg(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::Average { return impl::avg(lhs, rhs); } @@ -502,8 +419,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing pairwise horizontal sums. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL add_horizontal(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::add_horizontal(left, right); } + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::HorizontalAdd { return impl::add_horizontal(lhs, rhs); } @@ -513,8 +430,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing pairwise horizontal differences. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL subtract_horizontal(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::subtract_horizontal(left, right); } + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::HorizontalSubtract { return impl::subtract_horizontal(lhs, rhs); } @@ -524,8 +441,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register whose lane type follows the promoted integer mapping rather than `vector_t`. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(const vector_t lhs, const vector_t rhs) noexcept - requires(using_int && requires(vector_t left, vector_t right) { impl::multiply_add_adjacent(left, right); }) + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(const vector_t lhs, const vector_t rhs) noexcept + requires(using_int && IImpl::MultiplyAddAdjacent) { return impl::multiply_add_adjacent(lhs, rhs); } @@ -535,8 +452,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register whose bytes are interpreted as signed. * @return Register containing signed 16-bit accumulation results derived from the raw register bytes. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(const vector_t lhs, const vector_t rhs) noexcept - requires(using_int && requires(vector_t left, vector_t right) { impl::multiply_add_unsigned_signed_bytes(left, right); }) + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(const vector_t lhs, const vector_t rhs) noexcept + requires(using_int && IImpl::ByteMultiplyAdd) { return impl::multiply_add_unsigned_signed_bytes(lhs, rhs); } @@ -546,8 +463,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register interpreted byte-wise. * @return Register containing 64-bit absolute-difference accumulations derived from the raw register bytes. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(const vector_t lhs, const vector_t rhs) noexcept - requires(using_int && requires(vector_t left, vector_t right) { impl::sum_absolute_byte_differences(left, right); }) + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(const vector_t lhs, const vector_t rhs) noexcept + requires(using_int && IImpl::Sad) { return impl::sum_absolute_byte_differences(lhs, rhs); } @@ -559,8 +476,8 @@ struct Api : public Detail::SimdMappings * @return Register containing byte-window absolute-difference accumulations derived from the raw register bytes. */ template - SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(const vector_t lhs, const vector_t rhs) noexcept - requires(using_int && requires(vector_t left, vector_t right) { impl::template multi_sum_absolute_byte_differences(left, right); }) + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(const vector_t lhs, const vector_t rhs) noexcept + requires(using_int && IImpl::MultiSad) { return impl::template multi_sum_absolute_byte_differences(lhs, rhs); } @@ -569,17 +486,11 @@ struct Api : public Detail::SimdMappings * @param lhs Input register. * @return Zero-based index of the first minimum element across the full SIMD register. */ - SIMDLIB_FORCE_INLINE constexpr static std::size_t VECTORCALL min_position(const vector_t lhs) noexcept - requires(using_int && requires(vector_t value) { - impl::min_position(value); - impl::template extract<1>(value); - }) + constexpr static std::size_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(const vector_t lhs) noexcept + requires(using_int && IImpl::Position) { if (std::is_constant_evaluated()) - { - const auto values = to_array(lhs); - return static_cast(std::min_element(values.begin(), values.end()) - values.begin()); - } + return min_position_constexpr(lhs); return static_cast(impl::template extract<1>(impl::min_position(lhs))); } @@ -588,17 +499,11 @@ struct Api : public Detail::SimdMappings * @param lhs Input register. * @return Zero-based index of the first maximum element across the full SIMD register. */ - SIMDLIB_FORCE_INLINE constexpr static std::size_t VECTORCALL max_position(const vector_t lhs) noexcept - requires(using_int && requires(vector_t value) { - impl::min_position(value); - impl::template extract<1>(value); - }) + constexpr static std::size_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) max_position(const vector_t lhs) noexcept + requires(using_int && IImpl::Position) { if (std::is_constant_evaluated()) - { - const auto values = to_array(lhs); - return static_cast(std::max_element(values.begin(), values.end()) - values.begin()); - } + return max_position_constexpr(lhs); if constexpr (using_unsigned) { @@ -616,8 +521,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing saturated sums. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_saturated(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::add_saturated(left, right); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::AddSaturated { return impl::add_saturated(lhs, rhs); } @@ -627,8 +532,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing saturated differences. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_saturated(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::subtract_saturated(left, right); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::SubtractSaturated { return impl::subtract_saturated(lhs, rhs); } @@ -638,8 +543,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing saturated horizontal sums. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL hadd_saturated(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::hadd_saturated(left, right); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hadd_saturated(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::HorizontalAddSaturated { return impl::hadd_saturated(lhs, rhs); } @@ -649,8 +554,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing saturated horizontal differences. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL hsubtract_saturated(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::hsubtract_saturated(left, right); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hsubtract_saturated(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::HorizontalSubtractSaturated { return impl::hsubtract_saturated(lhs, rhs); } @@ -660,8 +565,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing alternating subtract/add results. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_subtract(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::add_subtract(left, right); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_subtract(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::AddSubtract { return impl::add_subtract(lhs, rhs); } @@ -673,8 +578,8 @@ struct Api : public Detail::SimdMappings * @return Register containing the masked dot-product result. */ template - SIMDLIB_FORCE_INLINE static auto VECTORCALL dot_product(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::template dot_product(left, right); } + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) dot_product(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::DotProduct { return impl::template dot_product(lhs, rhs); } @@ -688,10 +593,13 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing the bitwise AND result. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL bitwise_and(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::bitwise_and(left, right); } + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_and(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::BitwiseAnd { - return impl::bitwise_and(lhs, rhs); + if (std::is_constant_evaluated()) + return bitwise_and_constexpr(lhs, rhs); + else + return impl::bitwise_and(lhs, rhs); } /** @brief Computes a bitwise OR of two registers. @@ -699,10 +607,13 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing the bitwise OR result. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL bitwise_or(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::bitwise_or(left, right); } + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_or(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::BitwiseOr { - return impl::bitwise_or(lhs, rhs); + if (std::is_constant_evaluated()) + return bitwise_or_constexpr(lhs, rhs); + else + return impl::bitwise_or(lhs, rhs); } /** @brief Computes a bitwise XOR of two registers. @@ -710,10 +621,13 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing the bitwise XOR result. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL bitwise_xor(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::bitwise_xor(left, right); } + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_xor(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::BitwiseXor { - return impl::bitwise_xor(lhs, rhs); + if (std::is_constant_evaluated()) + return bitwise_xor_constexpr(lhs, rhs); + else + return impl::bitwise_xor(lhs, rhs); } /** @brief Computes a bitwise AND-NOT of two registers. @@ -721,47 +635,62 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing the bitwise AND-NOT result. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL bitwise_andnot(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::bitwise_andnot(left, right); } + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_andnot(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::BitwiseAndNot { - return impl::bitwise_andnot(lhs, rhs); + if (std::is_constant_evaluated()) + return bitwise_andnot_constexpr(lhs, rhs); + else + return impl::bitwise_andnot(lhs, rhs); } /** @brief Computes a bitwise NOT of a register. * @param lhs Input register. * @return Register containing the bitwise NOT result. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL bitwise_not(const vector_t lhs) noexcept - requires requires(vector_t value) { impl::bitwise_not(value); } + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_not(const vector_t lhs) noexcept + requires IImpl::BitwiseNot + { + if (std::is_constant_evaluated()) + return bitwise_not_constexpr(lhs); + else + return impl::bitwise_not(lhs); + } + +#pragma endregion + +#pragma region Selection Operations + + /** @brief Selects lanes from two registers using a canonical native predicate. + * @param condition Canonical predicate register containing all-zero or all-one lanes. + * @param when_true Register selected where the corresponding predicate lane is true. + * @param when_false Register selected where the corresponding predicate lane is false. + * @return Register containing the selected lanes without reducing the predicate. + */ + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) + select(const vector_t condition, const vector_t when_true, const vector_t when_false) noexcept + requires IImpl::Select { - return impl::bitwise_not(lhs); + if (std::is_constant_evaluated()) + return select_constexpr(condition, when_true, when_false); + else + return impl::select(condition, when_true, when_false); } #pragma endregion #pragma region Comparison Operations +#pragma region Mask Reductions + /** @brief Returns a mask composed from the most significant bit of each byte in the register. * @param lhs Input register. * @return Byte-granular movemask for the register contents. */ - SIMDLIB_FORCE_INLINE constexpr static mask_t VECTORCALL movemask(const vector_t lhs) noexcept + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) movemask(const vector_t lhs) noexcept { if (std::is_constant_evaluated()) - { - const auto lanes = to_array(lhs); - mask_t result = 0; - for (std::size_t laneIndex = 0; laneIndex < element_count; ++laneIndex) - { - const auto laneBytes = std::bit_cast>(lanes[laneIndex]); - for (std::size_t byteIndex = 0; byteIndex < sizeof(element_t); ++byteIndex) - { - const std::size_t maskIndex = laneIndex * sizeof(element_t) + byteIndex; - result |= static_cast((laneBytes[byteIndex] >> 7) & 1u) << maskIndex; - } - } - return result; - } + return movemask_constexpr(lhs); else { return impl::movemask(lhs); @@ -772,139 +701,240 @@ struct Api : public Detail::SimdMappings * @param lhs Input register. * @return Element-granular movemask for the register contents. */ - SIMDLIB_FORCE_INLINE constexpr static mask_t VECTORCALL movemask_slim(const vector_t lhs) noexcept + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) movemask_slim(const vector_t lhs) noexcept { if (std::is_constant_evaluated()) - { - const auto lanes = to_array(lhs); - mask_t result = 0; - for (std::size_t laneIndex = 0; laneIndex < element_count; ++laneIndex) - { - const auto laneBytes = std::bit_cast>(lanes[laneIndex]); - result |= static_cast((laneBytes.back() >> 7) & 1u) << laneIndex; - } - return result; - } + return movemask_slim_constexpr(lhs); else { return impl::movemask_slim(lhs); } } - /** @brief Computes an equality comparison mask for two registers. +#pragma endregion + +#pragma region Native Predicate Comparisons + + /** @brief Compares corresponding lanes for ordered equality without reducing the result. * @param lhs Left-hand input register. * @param rhs Right-hand input register. - * @return Mask with bits set where corresponding elements are equal. + * @return Native predicate register containing an all-one true lane or an all-zero false lane. */ - SIMDLIB_FORCE_INLINE constexpr static mask_t VECTORCALL cmp_eq(const vector_t lhs, const vector_t rhs) noexcept + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) compare_equal(const vector_t lhs, const vector_t rhs) noexcept { if (std::is_constant_evaluated()) - { - const auto lhsValues = to_array(lhs); - const auto rhsValues = to_array(rhs); - mask_t result = 0; - constexpr mask_t laneMask = static_cast((mask_t{1} << sizeof(element_t)) - 1); - for (std::size_t index = 0; index < element_count; ++index) - if (lhsValues[index] == rhsValues[index]) - result |= laneMask << (index * sizeof(element_t)); - return result; - } + return compare_equal_constexpr(lhs, rhs); else - { - return impl::movemask(impl::cmpeq(lhs, rhs)); - } + return impl::cmpeq(lhs, rhs); } - /** @brief Computes a byte-granular equality comparison mask for two registers of this SIMD shape. + /** @brief Compares corresponding lanes for greater-than ordering without reducing the result. * @param lhs Left-hand input register. * @param rhs Right-hand input register. - * @return Mask with bits set where the underlying compare produced all-one bytes. + * @return Native predicate register containing an all-one true lane or an all-zero false lane. */ - SIMDLIB_FORCE_INLINE constexpr static mask_t VECTORCALL cmp_eq_mask(const vector_t lhs, const vector_t rhs) noexcept + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) compare_greater(const vector_t lhs, const vector_t rhs) noexcept { if (std::is_constant_evaluated()) - { - const auto lhsValues = to_array(lhs); - const auto rhsValues = to_array(rhs); - mask_t result = 0; - constexpr mask_t laneMask = static_cast((mask_t{1} << sizeof(element_t)) - 1); - for (std::size_t index = 0; index < element_count; ++index) - if (lhsValues[index] == rhsValues[index]) - result |= laneMask << (index * sizeof(element_t)); - return result; - } + return compare_greater_constexpr(lhs, rhs); else - { - return impl::movemask(impl::cmpeq(lhs, rhs)); - } + return impl::cmpgt(lhs, rhs); } - /** @brief Computes a greater-than comparison mask for two registers. + /** @brief Compares corresponding lanes for greater-than-or-equal ordering without reducing the result. * @param lhs Left-hand input register. * @param rhs Right-hand input register. - * @return Mask with bits set where lhs elements are greater than rhs elements. + * @return Native predicate register containing an all-one true lane or an all-zero false lane. */ - SIMDLIB_FORCE_INLINE constexpr static mask_t VECTORCALL cmp_gt(const vector_t lhs, const vector_t rhs) noexcept + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) compare_greater_equal(const vector_t lhs, const vector_t rhs) noexcept { if (std::is_constant_evaluated()) - { - const auto lhsValues = to_array(lhs); - const auto rhsValues = to_array(rhs); - mask_t result = 0; - constexpr mask_t laneMask = static_cast((mask_t{1} << sizeof(element_t)) - 1); - for (std::size_t index = 0; index < element_count; ++index) - if (lhsValues[index] > rhsValues[index]) - result |= laneMask << (index * sizeof(element_t)); - return result; - } + return compare_greater_equal_constexpr(lhs, rhs); else - { - return impl::movemask(impl::cmpgt(lhs, rhs)); - } + return bitwise_or(compare_equal(lhs, rhs), compare_greater(lhs, rhs)); } - /** @brief Computes a greater-than-or-equal comparison mask for two registers. + /** @brief Compares corresponding lanes for less-than ordering without reducing the result. * @param lhs Left-hand input register. * @param rhs Right-hand input register. - * @return Mask with bits set where lhs elements are greater than or equal to rhs elements. + * @return Native predicate register containing an all-one true lane or an all-zero false lane. */ - SIMDLIB_FORCE_INLINE constexpr static mask_t VECTORCALL cmp_ge(const vector_t lhs, const vector_t rhs) noexcept + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) compare_less(const vector_t lhs, const vector_t rhs) noexcept { - return cmp_eq(lhs, rhs) | cmp_gt(lhs, rhs); + if (std::is_constant_evaluated()) + return compare_less_constexpr(lhs, rhs); + else + return impl::cmpgt(rhs, lhs); } - /** @brief Computes a less-than comparison mask for two registers. + /** @brief Compares corresponding lanes for less-than-or-equal ordering without reducing the result. * @param lhs Left-hand input register. * @param rhs Right-hand input register. - * @return Mask with bits set where lhs elements are less than rhs elements. + * @return Native predicate register containing an all-one true lane or an all-zero false lane. */ - SIMDLIB_FORCE_INLINE constexpr static mask_t VECTORCALL cmp_lt(const vector_t lhs, const vector_t rhs) noexcept + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) compare_less_equal(const vector_t lhs, const vector_t rhs) noexcept { if (std::is_constant_evaluated()) - { - const auto lhsValues = to_array(lhs); - const auto rhsValues = to_array(rhs); - mask_t result = 0; - constexpr mask_t laneMask = static_cast((mask_t{1} << sizeof(element_t)) - 1); - for (std::size_t index = 0; index < element_count; ++index) - if (lhsValues[index] < rhsValues[index]) - result |= laneMask << (index * sizeof(element_t)); - return result; - } + return compare_less_equal_constexpr(lhs, rhs); else - { - return impl::movemask(impl::cmpgt(rhs, lhs)); - } + return bitwise_or(compare_equal(lhs, rhs), compare_less(lhs, rhs)); + } + +#pragma endregion + +#pragma region Byte Comparison Masks + + /** @brief Reduces an equality comparison to a byte-granular scalar mask. + * @param lhs Left-hand input register. + * @param rhs Right-hand input register. + * @return Mask with one set bit for every all-one byte produced by the comparison. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_eq_mask(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask(compare_equal(lhs, rhs)); + } + + /** @brief Reduces a greater-than comparison to a byte-granular scalar mask. + * @param lhs Left-hand input register. + * @param rhs Right-hand input register. + * @return Mask with one set bit for every all-one byte produced by the comparison. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_gt_mask(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask(compare_greater(lhs, rhs)); + } + + /** @brief Reduces a greater-than-or-equal comparison to a byte-granular scalar mask. + * @param lhs Left-hand input register. + * @param rhs Right-hand input register. + * @return Mask with one set bit for every all-one byte produced by the comparison. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_ge_mask(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask(compare_greater_equal(lhs, rhs)); + } + + /** @brief Reduces a less-than comparison to a byte-granular scalar mask. + * @param lhs Left-hand input register. + * @param rhs Right-hand input register. + * @return Mask with one set bit for every all-one byte produced by the comparison. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_lt_mask(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask(compare_less(lhs, rhs)); + } + + /** @brief Reduces a less-than-or-equal comparison to a byte-granular scalar mask. + * @param lhs Left-hand input register. + * @param rhs Right-hand input register. + * @return Mask with one set bit for every all-one byte produced by the comparison. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_le_mask(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask(compare_less_equal(lhs, rhs)); + } + +#pragma endregion + +#pragma region Slim Comparison Masks + + /** @brief Reduces an equality comparison to one scalar bit per logical lane. + * @param lhs Left-hand input register. + * @param rhs Right-hand input register. + * @return Mask with one set bit for every true predicate lane. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_eq_slim(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask_slim(compare_equal(lhs, rhs)); + } + + /** @brief Reduces a greater-than comparison to one scalar bit per logical lane. + * @param lhs Left-hand input register. + * @param rhs Right-hand input register. + * @return Mask with one set bit for every true predicate lane. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_gt_slim(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask_slim(compare_greater(lhs, rhs)); + } + + /** @brief Reduces a greater-than-or-equal comparison to one scalar bit per logical lane. + * @param lhs Left-hand input register. + * @param rhs Right-hand input register. + * @return Mask with one set bit for every true predicate lane. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_ge_slim(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask_slim(compare_greater_equal(lhs, rhs)); } - /** @brief Computes a less-than-or-equal comparison mask for two registers. + /** @brief Reduces a less-than comparison to one scalar bit per logical lane. * @param lhs Left-hand input register. * @param rhs Right-hand input register. - * @return Mask with bits set where lhs elements are less than or equal to rhs elements. + * @return Mask with one set bit for every true predicate lane. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_lt_slim(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask_slim(compare_less(lhs, rhs)); + } + + /** @brief Reduces a less-than-or-equal comparison to one scalar bit per logical lane. + * @param lhs Left-hand input register. + * @param rhs Right-hand input register. + * @return Mask with one set bit for every true predicate lane. + */ + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_le_slim(const vector_t lhs, const vector_t rhs) noexcept + { + return movemask_slim(compare_less_equal(lhs, rhs)); + } + +#pragma endregion + +#pragma region Deprecated Comparison Masks + + /** @brief Legacy byte-granular equality mask spelling. + * @deprecated Use cmp_eq_mask() instead. + */ + [[deprecated("Use cmp_eq_mask() instead.")]] + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_eq(const vector_t lhs, const vector_t rhs) noexcept + { + return cmp_eq_mask(lhs, rhs); + } + + /** @brief Legacy byte-granular greater-than mask spelling. + * @deprecated Use cmp_gt_mask() instead. + */ + [[deprecated("Use cmp_gt_mask() instead.")]] + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_gt(const vector_t lhs, const vector_t rhs) noexcept + { + return cmp_gt_mask(lhs, rhs); + } + + /** @brief Legacy byte-granular greater-than-or-equal mask spelling. + * @deprecated Use cmp_ge_mask() instead. + */ + [[deprecated("Use cmp_ge_mask() instead.")]] + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_ge(const vector_t lhs, const vector_t rhs) noexcept + { + return cmp_ge_mask(lhs, rhs); + } + + /** @brief Legacy byte-granular less-than mask spelling. + * @deprecated Use cmp_lt_mask() instead. + */ + [[deprecated("Use cmp_lt_mask() instead.")]] + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_lt(const vector_t lhs, const vector_t rhs) noexcept + { + return cmp_lt_mask(lhs, rhs); + } + + /** @brief Legacy byte-granular less-than-or-equal mask spelling. + * @deprecated Use cmp_le_mask() instead. */ - SIMDLIB_FORCE_INLINE constexpr static mask_t VECTORCALL cmp_le(const vector_t lhs, const vector_t rhs) noexcept + [[deprecated("Use cmp_le_mask() instead.")]] + constexpr static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) cmp_le(const vector_t lhs, const vector_t rhs) noexcept { - return cmp_eq(lhs, rhs) | cmp_lt(lhs, rhs); + return cmp_le_mask(lhs, rhs); } #pragma endregion @@ -918,8 +948,8 @@ struct Api : public Detail::SimdMappings * @param rhs Auxiliary source register when required by the implementation. * @return Expanded register value. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::expand(left, right); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) expand(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::Expand { return impl::expand(lhs, rhs); } @@ -929,8 +959,8 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand source register. * @return Compressed register value. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL compress(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::compress(left, right); } + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) compress(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::Compress { return impl::compress(lhs, rhs); } @@ -941,9 +971,10 @@ struct Api : public Detail::SimdMappings * @return Extracted value as defined by the specialization. */ template - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(const vector_t lhs) noexcept - requires requires(vector_t value) { impl::template extract(value); } + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) extract(const vector_t lhs) noexcept + requires IImpl::IndexedExtract { + static_assert(index >= 0 && static_cast(index) < element_count, "Api::extract index out of range."); return impl::template extract(lhs); } @@ -951,34 +982,60 @@ struct Api : public Detail::SimdMappings * @param lhs Source register. * @param rhs Extract selector. * @return Extracted value as defined by the specialization. + * @note `_slow` marks runtime emulation of an immediate lane selector. */ template - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(const vector_t lhs, selector_t rhs) noexcept - requires requires(vector_t left, selector_t selector) { impl::extract(left, selector); } + constexpr static auto SIMD_FLAGS(InOut, ForceInline, Flatten) extract_slow(const vector_t lhs, selector_t rhs) noexcept + requires IImpl::ExtractSlow { - return impl::extract(lhs, rhs); + if (std::is_constant_evaluated()) + return extract_constexpr(lhs, static_cast(rhs)); + return impl::extract_slow(lhs, rhs); } /** @brief Returns the low 128-bit half of a 256-bit register when the specialization supports it. * @param lhs Source register. * @return Register containing the low 128-bit half in the corresponding 128-bit SIMD family. */ - SIMDLIB_FORCE_INLINE static typename SimdLib::Detail::SimdMappings<128, element_t>::vector_t VECTORCALL lower_half(const vector_t lhs) noexcept - requires(register_width == 256 && requires(vector_t value) { impl::lower_half(value); }) + constexpr static typename SimdLib::Detail::SimdMappings<128, element_t>::vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) + lower_half(const vector_t lhs) noexcept + requires(register_width == 256 && IImpl::LowerHalf) { + if (std::is_constant_evaluated()) + return lower_half_constexpr(lhs); return impl::lower_half(lhs); } - /** @brief Inserts a lane or subvalue into a register. - * @tparam Args Argument pack matching the implementation-specific insert signature. - * @param args Arguments forwarded to the specialization insert operation. - * @return Register containing the inserted value. + /** @brief Inserts a compile-time-selected scalar lane into a register. + * @tparam index Compile-time logical lane index. + * @param lhs Register whose unselected lanes are preserved. + * @param rhs Scalar replacement value. + * @return Register with lane `index` replaced. */ - template - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(Args &&...args) noexcept - requires requires(Args &&...values) { impl::insert(std::forward(values)...); } + template + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) insert(const vector_t lhs, const element_t rhs) noexcept + requires(index < element_count) { - return impl::insert(std::forward(args)...); + if (std::is_constant_evaluated()) + return impl::template insert_constexpr(index)>(lhs, rhs); + else + return impl::template insert(index)>(lhs, rhs); + } + + /** + * @brief Replaces one runtime-selected scalar lane in a register. + * @param lhs Register whose unselected lanes are preserved. + * @param rhs Scalar replacement value. + * @param index Runtime-selected logical lane index. + * @return Register with the selected lane replaced. + * @note `_slow` marks runtime emulation of an immediate lane selector. + */ + constexpr static vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) insert_slow(const vector_t lhs, const element_t rhs, const int index) noexcept + requires IImpl::InsertSlow + { + if (std::is_constant_evaluated()) + return insert_constexpr(lhs, rhs, index); + return impl::insert_slow(lhs, rhs, index); } /** @brief Unpacks the low lanes of two registers. @@ -986,9 +1043,11 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing the unpacked low-lane interleave. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::unpack_lo(left, right); } + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) unpack_lo(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::UnpackLow { + if (std::is_constant_evaluated()) + return unpack_constexpr(lhs, rhs); return impl::unpack_lo(lhs, rhs); } @@ -997,21 +1056,26 @@ struct Api : public Detail::SimdMappings * @param rhs Right-hand input register. * @return Register containing the unpacked high-lane interleave. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(const vector_t lhs, const vector_t rhs) noexcept - requires requires(vector_t left, vector_t right) { impl::unpack_hi(left, right); } + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) unpack_hi(const vector_t lhs, const vector_t rhs) noexcept + requires IImpl::UnpackHigh { + if (std::is_constant_evaluated()) + return unpack_constexpr(lhs, rhs); return impl::unpack_hi(lhs, rhs); } - /** @brief Shuffles register contents according to the implementation-specific control form. - * @tparam Args Argument pack matching the specialization shuffle signature. - * @param args Arguments forwarded to the specialization shuffle operation. - * @return Register containing the shuffled result. + /** @brief Rearranges logical lanes with one compile-time selector per result lane. + * @tparam indices Exact selector sequence in logical result-lane order. + * @param lhs Source register. + * @return Register containing the selected lanes. + * @note Every selector may name any logical lane in the complete source register. */ template - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(const int_vector_t lhs) noexcept - requires requires(int_vector_t value) { impl::template shuffle(value); } + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(const vector_t lhs) noexcept + requires(Api::template logical_shuffle_indices_valid() && IImpl::IndexedShuffle) { + if (std::is_constant_evaluated()) + return shuffle_constexpr(lhs); return impl::template shuffle(lhs); } @@ -1021,48 +1085,126 @@ struct Api : public Detail::SimdMappings * @return Register containing the shuffled result. */ template - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(Args &&...args) noexcept - requires requires(Args &&...values) { impl::shuffle(std::forward(values)...); } + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) shuffle(Args &&...args) noexcept + requires IImpl::Shuffle { return impl::shuffle(std::forward(args)...); } + /** + * @brief Emulates an immediate-controlled shuffle from a runtime scalar control. + * @tparam Args Argument pack matching the implementation slow-path shuffle signature. + * @param args Arguments forwarded to the specialization slow-path shuffle operation. + * @return Register containing the shuffled result. + * @note `_slow` identifies a deliberate runtime substitute for an immediate-controlled operation and may require dispatch, branching, or a longer + * synthesized instruction sequence. + */ + template + static auto SIMD_FLAGS(Out, ForceInline, Flatten) shuffle_slow(Args &&...args) noexcept + requires IImpl::ShuffleSlow + { + return impl::shuffle_slow(std::forward(args)...); + } + /** @brief Shuffles the low four 16-bit lanes in each 128-bit group using an immediate control. + * @tparam imm8 Immediate control in the inclusive range `0..255`; every two-bit field selects one lane. + * @param lhs Source register. + * @return Register with each low four-lane group shuffled and all high four-lane groups preserved. + */ + template + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_lo(const vector_t lhs) noexcept + requires(using_int && element_width == 16 && imm8 >= 0 && imm8 <= 255 && IImpl::IndexedShuffleLow) + { + if (std::is_constant_evaluated()) + return shuffle_half_constexpr(lhs); + return impl::template shuffle_lo(lhs); + } + /** @brief Shuffles the low half of a register where the specialization supports it. - * @tparam Args Argument pack matching the specialization shuffle-low signature. + * @tparam Args Argument pack matching the specialization + * shuffle-low signature. * @param args Arguments forwarded to the specialization shuffle-low operation. * @return Register containing the shuffled low-half result. + * @note `_slow` marks runtime emulation of an immediate control byte and may require a longer synthesized sequence. */ template - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_lo(Args &&...args) noexcept - requires requires(Args &&...values) { impl::shuffle_lo(std::forward(values)...); } + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) shuffle_lo_slow(Args &&...args) noexcept + requires IImpl::ShuffleLowSlow + { + return impl::shuffle_lo_slow(std::forward(args)...); + } + + /** @brief Shuffles the high four 16-bit lanes in each 128-bit group using an immediate control. + * @tparam imm8 Immediate control in the inclusive range `0..255`; every two-bit field selects one lane. + * @param lhs Source register. + * @return Register with each high four-lane group shuffled and all low four-lane groups preserved. + */ + template + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_hi(const vector_t lhs) noexcept + requires(using_int && element_width == 16 && imm8 >= 0 && imm8 <= 255 && IImpl::IndexedShuffleHigh) { - return impl::shuffle_lo(std::forward(args)...); + if (std::is_constant_evaluated()) + return shuffle_half_constexpr(lhs); + return impl::template shuffle_hi(lhs); } /** @brief Shuffles the high half of a register where the specialization supports it. - * @tparam Args Argument pack matching the specialization shuffle-high signature. + * @tparam Args Argument pack matching the specialization + * shuffle-high signature. * @param args Arguments forwarded to the specialization shuffle-high operation. * @return Register containing the shuffled high-half result. + * @note `_slow` marks runtime emulation of an immediate control byte and may require a longer synthesized sequence. */ template - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_hi(Args &&...args) noexcept - requires requires(Args &&...values) { impl::shuffle_hi(std::forward(values)...); } + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) shuffle_hi_slow(Args &&...args) noexcept + requires IImpl::ShuffleHighSlow + { + return impl::shuffle_hi_slow(std::forward(args)...); + } + + /** @brief Selects corresponding lanes from two registers using an immediate bit mask. + * @tparam imm8 Immediate control in the inclusive range `0..255`; + * set bits select `rhs`. + * @param lhs Register selected by cleared applicable control bits. + * @param rhs Register selected by set applicable + * control bits. + * @return Register containing the intrinsic-defined immediate blend. + * @note Bits unused by the selected intrinsic have no effect. + * A 256-bit 16-bit blend repeats the eight mask bits in each 128-bit group. + */ + template + constexpr static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const vector_t lhs, const vector_t rhs) noexcept + requires(imm8 >= 0 && imm8 <= 255 && IImpl::IndexedBlend) { - return impl::shuffle_hi(std::forward(args)...); + return impl::template blend(lhs, rhs); } /** @brief Blends two registers according to the implementation-specific control form. - * @tparam Args Argument pack matching the specialization blend signature. + * @tparam Args Argument pack matching the specialization blend + * signature. * @param args Arguments forwarded to the specialization blend operation. * @return Register containing the blended result. */ template - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(Args &&...args) noexcept - requires requires(Args &&...values) { impl::blend(std::forward(values)...); } + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) blend(Args &&...args) noexcept + requires IImpl::Blend { return impl::blend(std::forward(args)...); } + /** + * @brief Emulates an immediate-controlled blend from a runtime scalar control. + * @tparam Args Argument pack matching the implementation slow-path blend signature. + * @param args Arguments forwarded to the specialization slow-path blend operation. + * @return Register containing the blended result. + * @note `_slow` identifies a deliberate runtime substitute for an immediate-controlled operation and may require dispatch, branching, or a longer + * synthesized instruction sequence. + */ + template + static auto SIMD_FLAGS(Out, ForceInline, Flatten) blend_slow(Args &&...args) noexcept + requires IImpl::BlendSlow + { + return impl::blend_slow(std::forward(args)...); + } #pragma endregion #pragma region Shifting Operations @@ -1072,20 +1214,12 @@ struct Api : public Detail::SimdMappings * @param shift Shift count applied to each lane. * @return Register containing per-lane left-shifted values. */ - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL shift_left(const int_vector_t lhs, int shift) noexcept + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_left(const int_vector_t lhs, int shift) noexcept requires(using_int) { + SIMDLIB_PRECONDITION(shift >= 0, "Per-lane left shifts require a nonnegative count"); if (std::is_constant_evaluated()) - { - if (shift >= static_cast(element_width)) - return impl::setzero(); - std::array results{}; - for (std::size_t index = 0; index < element_count; ++index) - { - results[index] = static_cast(impl::get_element(lhs, static_cast(index)) << shift); - } - return impl::construct(results); - } + return shift_left_constexpr(lhs, shift); return impl::shift_left(lhs, shift); } @@ -1095,20 +1229,12 @@ struct Api : public Detail::SimdMappings * @param shift Shift count applied to each lane. * @return Register containing per-lane right-shifted values. */ - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL shift_right(const int_vector_t lhs, int shift) noexcept + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_right(const int_vector_t lhs, int shift) noexcept requires(using_int) { + SIMDLIB_PRECONDITION(shift >= 0, "Per-lane logical right shifts require a nonnegative count"); if (std::is_constant_evaluated()) - { - if (shift >= static_cast(element_width)) - return impl::setzero(); - std::array results{}; - for (std::size_t index = 0; index < element_count; ++index) - { - results[index] = static_cast(static_cast>(impl::get_element(lhs, static_cast(index))) >> shift); - } - return impl::construct(results); - } + return shift_right_constexpr(lhs, shift); return impl::shift_right(lhs, shift); } @@ -1118,20 +1244,12 @@ struct Api : public Detail::SimdMappings * @param shift Shift count applied to each lane. * @return Register containing per-lane arithmetic right-shifted values. */ - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL shift_right_arithmetic(const int_vector_t lhs, int shift) noexcept + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_right_arithmetic(const int_vector_t lhs, int shift) noexcept requires(using_int) { + SIMDLIB_PRECONDITION(shift >= 0, "Per-lane arithmetic right shifts require a nonnegative count"); if (std::is_constant_evaluated()) - { - if (shift >= static_cast(element_width)) - shift = static_cast(element_width) - 1; - std::array results{}; - for (std::size_t index = 0; index < element_count; ++index) - { - results[index] = static_cast(impl::get_element(lhs, static_cast(index)) >> shift); - } - return impl::construct(results); - } + return shift_right_arithmetic_constexpr(lhs, shift); return impl::shift_right_arithmetic(lhs, shift); } @@ -1140,112 +1258,159 @@ struct Api : public Detail::SimdMappings * @brief Shifts every byte in a 128-bit register toward higher byte indices. * * A zero or negative count returns the input unchanged. A count greater than - * or equal to the register byte width returns zero. [eg: byte_shift_left( + * or equal to the register byte width returns zero. [eg: shift_bytes_left_slow( * {0x01, 0x02, ...}, 1) => {0x00, 0x01, 0x02, ...}] * * @param lhs The source register. * @param shift The runtime byte count. * @return The byte-shifted register. + * @note `_slow` marks runtime emulation of an immediate byte count. */ - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL byte_shift_left(const int_vector_t lhs, const int shift) noexcept + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_left_slow(const int_vector_t lhs, const int shift) noexcept requires(using_int && register_width == 128) { if (std::is_constant_evaluated()) - { - if (shift <= 0) - return lhs; - if (shift >= static_cast(byte_count)) - return impl::setzero(); - const auto sourceBytes = std::bit_cast>(to_array(lhs)); - std::array resultBytes{}; - for (std::size_t index = static_cast(shift); index < byte_count; ++index) - resultBytes[index] = sourceBytes[index - static_cast(shift)]; - return construct(std::bit_cast>(resultBytes)); - } - return impl::byte_shift_left(lhs, shift); + return shift_bytes_left_constexpr(lhs, shift); + return impl::shift_bytes_left_slow(lhs, shift); + } + + /** + * @brief Shifts every byte in a complete integral register toward higher byte indices. + * + * The count is encoded as an immediate. Zero returns the input unchanged; counts + * at least as large as the register byte width return zero. + * + * @tparam count Nonnegative compile-time byte count. + * @param lhs The source register. + * @return The byte-shifted register with zero-filled low bytes. + */ + template + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_left(const int_vector_t lhs) noexcept + requires(using_int && IImpl::ShiftBytesLeft) + { + static_assert(count >= 0, "Complete-register byte shifts require a nonnegative count."); + if (std::is_constant_evaluated()) + return shift_bytes_left_constexpr(lhs, count); + return impl::template shift_bytes_left(lhs); } /** * @brief Shifts every byte in a 128-bit register toward lower byte indices. * * A zero or negative count returns the input unchanged. A count greater than - * or equal to the register byte width returns zero. [eg: byte_shift_right( + * or equal to the register byte width returns zero. [eg: shift_bytes_right_slow( * {0x01, 0x02, ...}, 1) => {0x02, ..., 0x00}] * * @param lhs The source register. * @param shift The runtime byte count. * @return The byte-shifted register. + * @note `_slow` marks runtime emulation of an immediate byte count. */ - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL byte_shift_right(const int_vector_t lhs, const int shift) noexcept + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_right_slow(const int_vector_t lhs, const int shift) noexcept requires(using_int && register_width == 128) { if (std::is_constant_evaluated()) - { - if (shift <= 0) - return lhs; - if (shift >= static_cast(byte_count)) - return impl::setzero(); - const auto sourceBytes = std::bit_cast>(to_array(lhs)); - std::array resultBytes{}; - for (std::size_t index = 0; index + static_cast(shift) < byte_count; ++index) - resultBytes[index] = sourceBytes[index + static_cast(shift)]; - return construct(std::bit_cast>(resultBytes)); - } - return impl::byte_shift_right(lhs, shift); + return shift_bytes_right_constexpr(lhs, shift); + return impl::shift_bytes_right_slow(lhs, shift); + } + + /** + * @brief Shifts every byte in a complete integral register toward lower byte indices. + * + * The count is encoded as an immediate. Zero returns the input unchanged; counts + * at least as large as the register byte width return zero. + * + * @tparam count Nonnegative compile-time byte count. + * @param lhs The source register. + * @return The byte-shifted register with zero-filled high bytes. + */ + template + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_right(const int_vector_t lhs) noexcept + requires(using_int && IImpl::ShiftBytesRight) + { + static_assert(count >= 0, "Complete-register byte shifts require a nonnegative count."); + if (std::is_constant_evaluated()) + return shift_bytes_right_constexpr(lhs, count); + return impl::template shift_bytes_right(lhs); } /** @brief Shifts the complete 128-bit register left, carrying bits across lane boundaries. * Unlike `shift_left`, this treats the register as one * unsigned 128-bit bit string. * A zero or negative runtime count returns the input; counts of 128 or more return zero. + * @note `_slow` marks the synthesized runtime-count substitute for immediate complete-register shifts. */ - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL bit_shift_left(const int_vector_t lhs, const int shift) noexcept + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_left_slow(const int_vector_t lhs, const int shift) noexcept requires(using_int && register_width == 128) { - return impl::bit_shift_left(lhs, shift); + if (std::is_constant_evaluated()) + return shift_bits_left_constexpr(lhs, shift); + return impl::shift_bits_left_slow(lhs, shift); } /** @brief Compile-time complete-register left shift. Counts of 128 or more return zero. */ template - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL bit_shift_left(const int_vector_t lhs) noexcept + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_left(const int_vector_t lhs) noexcept requires(using_int && register_width == 128) { static_assert(shift >= 0, "Whole-register shifts require a non-negative count."); - return impl::template bit_shift_left(lhs); + if (std::is_constant_evaluated()) + return shift_bits_left_constexpr(lhs, shift); + return impl::template shift_bits_left(lhs); } /** @brief Shifts the complete 128-bit register right, carrying bits across lane boundaries. * Unlike `shift_right`, this treats the register as one * unsigned 128-bit bit string. * A zero or negative runtime count returns the input; counts of 128 or more return zero. + * @note `_slow` marks the synthesized runtime-count substitute for immediate complete-register shifts. */ - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL bit_shift_right(const int_vector_t lhs, const int shift) noexcept + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_right_slow(const int_vector_t lhs, const int shift) noexcept requires(using_int && register_width == 128) { - return impl::bit_shift_right(lhs, shift); + if (std::is_constant_evaluated()) + return shift_bits_right_constexpr(lhs, shift); + return impl::shift_bits_right_slow(lhs, shift); } /** @brief Compile-time complete-register right shift. Counts of 128 or more return zero. */ template - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL bit_shift_right(const int_vector_t lhs) noexcept + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_right(const int_vector_t lhs) noexcept requires(using_int && register_width == 128) { static_assert(shift >= 0, "Whole-register shifts require a non-negative count."); - return impl::template bit_shift_right(lhs); + if (std::is_constant_evaluated()) + return shift_bits_right_constexpr(lhs, shift); + return impl::template shift_bits_right(lhs); } #pragma endregion #pragma region Conversion Operations + /** @brief Reinterprets every bit of a complete register as another supported lane type. + * @tparam target_t Destination lane interpretation at the same register width. + * @param vector Source register whose complete bit pattern is preserved. + * @return Destination native register containing exactly the source bits. + */ + template + constexpr static mapped_vector_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) bit_cast(const vector_t vector) noexcept + requires ApiAvailable + { + if (std::is_constant_evaluated()) + return bit_cast_constexpr(vector); + return std::bit_cast>(vector); + } + /** @brief Converts 32-bit integer lanes into floating-point lanes. * @param vector Input integer register. * @return Floating-point register containing the converted lane values. */ - SIMDLIB_FORCE_INLINE static float_vector_t VECTORCALL convert_to_float(int_vector_t vector) noexcept - requires(element_width == 32) + constexpr static float_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) convert_to_float(int_vector_t vector) noexcept + requires(element_width == 32 && using_int) { - static_assert(element_width == 32, "Only 32 bit integers can be converted to floats"); + if (std::is_constant_evaluated()) + return convert_to_float_constexpr(vector); if constexpr (register_width == 128) { if constexpr (using_unsigned) @@ -1266,10 +1431,11 @@ struct Api : public Detail::SimdMappings * @param vector Input floating-point register. * @return Integer register containing the converted lane values. */ - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL convert_to_int(float_vector_t vector) noexcept - requires(element_width == 32) + constexpr static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) convert_to_int(float_vector_t vector) noexcept + requires(element_width == 32 && std::same_as) { - static_assert(element_width == 32, "Only 32 bit floats can be converted to integers"); + if (std::is_constant_evaluated()) + return convert_to_int_constexpr(vector); if constexpr (register_width == 128) { return _mm_cvtps_epi32(vector); @@ -1284,7 +1450,7 @@ struct Api : public Detail::SimdMappings * @param vector Input register. * @return Register converted to the complementary 32-bit scalar representation. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL convert(vector_t vector) noexcept + constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) convert(vector_t vector) noexcept requires(element_width == 32) { if constexpr (std::is_floating_point_v) @@ -1293,6 +1459,23 @@ struct Api : public Detail::SimdMappings return convert_to_float(vector); } + /** @brief Numerically converts every source lane into one complete destination register. + * @tparam target_t Explicit numeric destination lane type. + * @param vector Source register. + * @return Complete destination native register containing the converted lane values. + * @note The initial conversion surface supports signed or unsigned 32-bit integers to `float`, and `float` to signed 32-bit integers. + */ + template + constexpr static mapped_vector_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) convert(const vector_t vector) noexcept + requires((std::same_as && (std::same_as || std::same_as)) || + (std::same_as && std::same_as)) + { + if constexpr (std::same_as) + return convert_to_float(vector); + else + return convert_to_int(vector); + } + #pragma endregion #pragma region Transform @@ -1300,17 +1483,17 @@ struct Api : public Detail::SimdMappings /** @brief Applies a SIMD transform whose fixed-width lane results are packed contiguously into integer storage. * @tparam result_bit_width Number of logical result bits produced per source element. * @tparam count Number of source elements. - * @tparam Func Callable that accepts `vector_t` and returns an unsigned integer containing packed lane results, with lane zero in the least-significant bits. + * @tparam Func Callable that accepts `vector_t` and returns an unsigned integer containing packed lane results, with lane zero in the least-significant + * bits. * @param read Source elements to transform. * @param write Destination storage for the packed result bit stream. * @param func SIMD transformation that returns one packed result for each loaded register. * @return None. */ template Func> - SIMDLIB_FORCE_INLINE constexpr static void transform_pack( - std::span read, - std::span, packed_element_count> write, - Func &&func) noexcept + constexpr static void SIMD_FLAGS(Neither, ForceInline, Flatten) + transform_pack(std::span read, + std::span, packed_element_count> write, Func &&func) noexcept requires(result_bit_width > 0 && result_bit_width <= 64) { using result_t = std::remove_cvref_t>; @@ -1318,16 +1501,13 @@ struct Api : public Detail::SimdMappings // uintptr_t is the standard unsigned type that most closely represents the target's native integer register width. // Accumulating into it lets us write whole machine words instead of updating individual destination bytes. using native_word_t = std::uintptr_t; - static_assert(std::unsigned_integral && !std::same_as, - "Packed SIMD transforms must return an unsigned integer"); - static_assert(element_count * result_bit_width <= 64, - "A packed SIMD register result cannot exceed 64 bits"); + static_assert(std::unsigned_integral && !std::same_as, "Packed SIMD transforms must return an unsigned integer"); + static_assert(element_count * result_bit_width <= 64, "A packed SIMD register result cannot exceed 64 bits"); static_assert(element_count * result_bit_width <= std::numeric_limits::digits, - "The packed transform result type must contain every result bit for one SIMD register"); + "The packed transform result type must contain every result bit for one SIMD register"); // Callback results place the first SIMD lane in the least-significant bits. Copying the accumulator directly to // sequential storage preserves that lane order only when the least-significant byte is stored first. - static_assert(std::endian::native == std::endian::little, - "Packed SIMD transforms require little-endian integer storage"); + static_assert(std::endian::native == std::endian::little, "Packed SIMD transforms require little-endian integer storage"); constexpr std::size_t native_word_width = std::numeric_limits::digits; constexpr std::size_t total_result_bit_count = count * result_bit_width; @@ -1353,9 +1533,8 @@ struct Api : public Detail::SimdMappings { const std::size_t available_bit_count = native_word_width - pending_bit_count; const std::size_t consumed_bit_count = std::min(result_bit_count, available_bit_count); - const std::uint64_t consumed_mask = consumed_bit_count == 64 - ? std::numeric_limits::max() - : (std::uint64_t{1} << consumed_bit_count) - 1; + const std::uint64_t consumed_mask = + consumed_bit_count == 64 ? std::numeric_limits::max() : (std::uint64_t{1} << consumed_bit_count) - 1; pending |= static_cast((remaining & consumed_mask) << pending_bit_count); remaining = consumed_bit_count == 64 ? 0 : remaining >> consumed_bit_count; @@ -1363,12 +1542,15 @@ struct Api : public Detail::SimdMappings result_bit_count -= consumed_bit_count; // memcpy permits a native-width store without imposing alignment or aliasing requirements on write_t. - if (pending_bit_count == native_word_width) + if constexpr (flushed_native_word_count != 0) { - std::memcpy(write_bytes.data() + write_byte_offset, &pending, sizeof(pending)); - write_byte_offset += sizeof(pending); - pending = 0; - pending_bit_count = 0; + if (pending_bit_count == native_word_width) + { + std::memcpy(write_bytes.data() + write_byte_offset, &pending, sizeof(pending)); + write_byte_offset += sizeof(pending); + pending = 0; + pending_bit_count = 0; + } } } }; @@ -1407,7 +1589,7 @@ struct Api : public Detail::SimdMappings * @param func Unary SIMD transform to apply. * @return None. */ - template Func> static inline void transform(std::span data, Func &&func) noexcept + template Func> static void SIMD_FLAGS(Neither, Flatten) transform(std::span data, Func &&func) noexcept { const auto Length = data.size(); for (std::size_t i = 0; i < Length / element_count; ++i) @@ -1437,7 +1619,8 @@ struct Api : public Detail::SimdMappings * @param func Unary SIMD transform to apply. * @return None. */ - template Func> static inline void transform(std::span lhs, std::span write, Func &&func) noexcept + template Func> + static void SIMD_FLAGS(Neither, Flatten) transform(std::span lhs, std::span write, Func &&func) noexcept { static_assert(std::is_invocable_r_v, "Function must return a value of vector_t"); const auto Length = lhs.size(); @@ -1469,7 +1652,8 @@ struct Api : public Detail::SimdMappings * @return None. */ template Func> - static inline void transform(std::span lhs, std::span rhs, std::span write, Func &&func) noexcept + static void SIMD_FLAGS(Neither, Flatten) + transform(std::span lhs, std::span rhs, std::span write, Func &&func) noexcept { static_assert(std::is_invocable_r_v, "Function must return an vector_t"); const auto Length = lhs.size(); @@ -1500,11 +1684,550 @@ struct Api : public Detail::SimdMappings #pragma region Internal protected: + /** @brief Validates a logical lane-shuffle selector sequence at overload resolution. + * @tparam indices Logical source-lane indices for every output lane. + * @return `true` when the selector count is exact and every selector names a lane in the complete source register. + */ + template [[nodiscard]] consteval static bool logical_shuffle_indices_valid() noexcept + { + return sizeof...(indices) == element_count && ((indices < element_count) && ...); + } + + /** @brief Extracts the low 128-bit lanes during constant evaluation. */ + [[nodiscard]] constexpr static typename SimdLib::Detail::SimdMappings<128, element_t>::vector_t lower_half_constexpr(const vector_t value) noexcept + { + using target_api = Api<128, element_t>; + const auto source = to_array(value); + std::array result{}; + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = source[lane]; + return target_api::construct(result); + } + + /** @brief Interleaves low or high lane halves within each 128-bit group during constant evaluation. + * @tparam high Selects the high source half when + * `true`, otherwise the low source half. + */ + template [[nodiscard]] constexpr static vector_t unpack_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + constexpr std::size_t lanes_per_group = 128 / element_width; + constexpr std::size_t lanes_per_half = lanes_per_group / 2; + for (std::size_t group = 0; group < element_count; group += lanes_per_group) + { + constexpr std::size_t source_half_offset = high ? lanes_per_half : 0; + for (std::size_t lane = 0; lane < lanes_per_half; ++lane) + { + result[group + lane * 2] = left[group + source_half_offset + lane]; + result[group + lane * 2 + 1] = right[group + source_half_offset + lane]; + } + } + return construct(result); + } + + /** @brief Applies a validated logical lane shuffle during constant evaluation. */ + template [[nodiscard]] constexpr static vector_t shuffle_constexpr(const vector_t value) noexcept + { + const auto source = to_array(value); + constexpr std::array selectors{indices...}; + std::array result{}; + for (std::size_t lane = 0; lane < element_count; ++lane) + result[lane] = source[selectors[lane]]; + return construct(result); + } + + /** @brief Applies an immediate 16-bit half shuffle during constant evaluation. + * @tparam imm8 Immediate selector fields. + * @tparam high Selects + * the high four-lane half in each 128-bit group. + */ + template [[nodiscard]] constexpr static vector_t shuffle_half_constexpr(const vector_t value) noexcept + { + const auto source = to_array(value); + auto result = source; + constexpr std::size_t lanes_per_group = 8; + constexpr std::size_t half_offset = high ? 4 : 0; + for (std::size_t group = 0; group < element_count; group += lanes_per_group) + { + for (std::size_t lane = 0; lane < 4; ++lane) + { + const std::size_t selected = static_cast(imm8) >> (lane * 2) & 0x3u; + result[group + half_offset + lane] = source[group + half_offset + selected]; + } + } + return construct(result); + } + + /** @brief Reinterprets a complete register bit pattern during constant evaluation. */ + template [[nodiscard]] constexpr static mapped_vector_t bit_cast_constexpr(const vector_t value) noexcept + { + using target_api = Api; + const auto target_lanes = std::bit_cast>(to_array(value)); + return target_api::construct(target_lanes); + } + + /** @brief Widens only the source prefix required to fill one target register during constant evaluation. */ + template [[nodiscard]] constexpr static typename target_simd::vector_t widen_constexpr(const vector_t value) noexcept + { + using target_element_t = typename target_simd::element_type; + const auto source = to_array(value); + std::array result{}; + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = static_cast(source[lane]); + return target_simd::construct(result); + } + + /** @brief Converts signed or unsigned 32-bit integer lanes to float during constant evaluation. */ + [[nodiscard]] constexpr static float_vector_t convert_to_float_constexpr(const int_vector_t value) noexcept + { + using target_api = Api; + const auto source = to_array(value); + std::array result{}; + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = static_cast(source[lane]); + return target_api::construct(result); + } + + /** @brief Converts one float with default-MXCSR round-to-nearest-even semantics. */ + [[nodiscard]] constexpr static std::int32_t convert_float_lane_to_int(const float value) noexcept + { + constexpr float minimum = -2147483648.0F; + constexpr float upper_exclusive = 2147483648.0F; + if (!(value >= minimum && value < upper_exclusive)) + return std::numeric_limits::min(); + + std::int32_t rounded = static_cast(value); + const float fraction = value - static_cast(rounded); + if (fraction > 0.5F || (fraction == 0.5F && rounded % 2 != 0)) + ++rounded; + else if (fraction < -0.5F || (fraction == -0.5F && rounded % 2 != 0)) + --rounded; + return rounded; + } + + /** @brief Converts float lanes to signed 32-bit integers during constant evaluation. */ + [[nodiscard]] constexpr static int_vector_t convert_to_int_constexpr(const float_vector_t value) noexcept + { + using source_api = Api; + using target_api = Api; + const auto source = source_api::to_array(value); + std::array result{}; + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = convert_float_lane_to_int(source[lane]); + return target_api::construct(result); + } + + /** @brief Applies bitwise AND during constant evaluation. + * @param lhs Left-hand input register represented in constant evaluation. + * @param rhs Right-hand input register represented in constant evaluation. + * @return Register containing the bitwise intersection. + */ + constexpr static vector_t bitwise_and_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + using unsigned_element_t = select_unsigned_integer_t; + for (std::size_t lane = 0; lane < element_count; ++lane) + { + const auto left_bits = std::bit_cast(left[lane]); + const auto right_bits = std::bit_cast(right[lane]); + result[lane] = std::bit_cast(static_cast(left_bits & right_bits)); + } + return construct(result); + } + + /** @brief Applies bitwise OR during constant evaluation. + * @param lhs Left-hand input register represented in constant evaluation. + * @param rhs Right-hand input register represented in constant evaluation. + * @return Register containing the bitwise union. + */ + constexpr static vector_t bitwise_or_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + using unsigned_element_t = select_unsigned_integer_t; + for (std::size_t lane = 0; lane < element_count; ++lane) + { + const auto left_bits = std::bit_cast(left[lane]); + const auto right_bits = std::bit_cast(right[lane]); + result[lane] = std::bit_cast(static_cast(left_bits | right_bits)); + } + return construct(result); + } + + /** @brief Applies bitwise XOR during constant evaluation. + * @param lhs Left-hand input register represented in constant evaluation. + * @param rhs Right-hand input register represented in constant evaluation. + * @return Register containing the bitwise exclusive union. + */ + constexpr static vector_t bitwise_xor_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + using unsigned_element_t = select_unsigned_integer_t; + for (std::size_t lane = 0; lane < element_count; ++lane) + { + const auto left_bits = std::bit_cast(left[lane]); + const auto right_bits = std::bit_cast(right[lane]); + result[lane] = std::bit_cast(static_cast(left_bits ^ right_bits)); + } + return construct(result); + } + + /** @brief Applies bitwise AND-NOT during constant evaluation. + * @param lhs Left-hand input register inverted before intersection. + * @param rhs Right-hand input register represented in constant evaluation. + * @return Register containing the intersection of inverted lhs and rhs. + */ + constexpr static vector_t bitwise_andnot_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + using unsigned_element_t = select_unsigned_integer_t; + for (std::size_t lane = 0; lane < element_count; ++lane) + { + const auto left_bits = std::bit_cast(left[lane]); + const auto right_bits = std::bit_cast(right[lane]); + result[lane] = std::bit_cast(static_cast(~left_bits & right_bits)); + } + return construct(result); + } + + /** @brief Applies bitwise inversion during constant evaluation. + * @param value Input register represented in constant evaluation. + * @return Register containing the bitwise inverse. + */ + constexpr static vector_t bitwise_not_constexpr(const vector_t value) noexcept + { + const auto lanes = to_array(value); + std::array result{}; + using unsigned_element_t = select_unsigned_integer_t; + for (std::size_t lane = 0; lane < element_count; ++lane) + { + const auto bits = std::bit_cast(lanes[lane]); + result[lane] = std::bit_cast(static_cast(~bits)); + } + return construct(result); + } + + /** @brief Selects lanes using a canonical predicate during constant evaluation. + * @param condition Canonical predicate register containing all-zero or all-one lanes. + * @param when_true Register selected where the corresponding predicate lane is true. + * @param when_false Register selected where the corresponding predicate lane is false. + * @return Register containing the selected lanes. + */ + constexpr static vector_t select_constexpr(const vector_t condition, const vector_t when_true, const vector_t when_false) noexcept + { + return bitwise_or(bitwise_and(condition, when_true), bitwise_andnot(condition, when_false)); + } + + /** @brief Converts a register to lane storage during constant evaluation. + * @param vector Register represented in constant evaluation. + * @return Array containing the register elements in lane order. + */ + constexpr static std::array to_array_constexpr(const vector_t vector) noexcept + { + std::array result{}; + for (std::size_t index = 0; index < element_count; ++index) + result[index] = extract_constexpr(vector, static_cast(index)); + return result; + } + + /** + * @brief Extracts one lane through the portable constant-evaluation representation. + * @param lhs Source register represented during constant evaluation. + * @param index Selected lane index. + * @return Selected scalar lane. + */ + constexpr static element_t extract_constexpr(const vector_t lhs, const int index) noexcept + { + return Detail::register_get_constexpr(lhs, static_cast(index)); + } + + /** + * @brief Replaces one lane through the portable constant-evaluation representation. + * @param lhs Source register represented during constant evaluation. + * @param rhs Replacement scalar lane. + * @param index Selected lane index. + * @return Register with the selected lane replaced. + */ + constexpr static vector_t insert_constexpr(const vector_t lhs, const element_t rhs, const int index) noexcept + { + return Detail::register_insert_constexpr(lhs, rhs, static_cast(index)); + } + + /** @brief Computes the byte-granular movemask during constant evaluation. + * @param lhs Input register represented in constant evaluation. + * @return Byte-granular movemask for the register contents. + */ + constexpr static mask_t movemask_constexpr(const vector_t lhs) noexcept + { + const auto lanes = to_array(lhs); + mask_t result = 0; + for (std::size_t laneIndex = 0; laneIndex < element_count; ++laneIndex) + { + const auto laneBytes = std::bit_cast>(lanes[laneIndex]); + for (std::size_t byteIndex = 0; byteIndex < sizeof(element_t); ++byteIndex) + { + const std::size_t maskIndex = laneIndex * sizeof(element_t) + byteIndex; + result |= static_cast((laneBytes[byteIndex] >> 7) & 1u) << maskIndex; + } + } + return result; + } + + /** @brief Finds the first minimum lane during constant evaluation. + * @param lhs Input register represented in constant evaluation. + * @return Zero-based index of the first minimum element. + */ + constexpr static std::size_t min_position_constexpr(const vector_t lhs) noexcept + { + const auto values = to_array(lhs); + return static_cast(std::min_element(values.begin(), values.end()) - values.begin()); + } + + /** @brief Finds the first maximum lane during constant evaluation. + * @param lhs Input register represented in constant evaluation. + * @return Zero-based index of the first maximum element. + */ + constexpr static std::size_t max_position_constexpr(const vector_t lhs) noexcept + { + const auto values = to_array(lhs); + return static_cast(std::max_element(values.begin(), values.end()) - values.begin()); + } + + /** @brief Computes the element-granular movemask during constant evaluation. + * @param lhs Input register represented in constant evaluation. + * @return Element-granular movemask for the register contents. + */ + constexpr static mask_t movemask_slim_constexpr(const vector_t lhs) noexcept + { + const auto lanes = to_array(lhs); + mask_t result = 0; + for (std::size_t laneIndex = 0; laneIndex < element_count; ++laneIndex) + { + const auto laneBytes = std::bit_cast>(lanes[laneIndex]); + result |= static_cast((laneBytes.back() >> 7) & 1u) << laneIndex; + } + return result; + } + + /** @brief Returns the canonical all-one predicate value for one lane. */ + [[nodiscard]] constexpr static element_t comparison_true_lane() noexcept + { + using unsigned_element_t = select_unsigned_integer_t; + return std::bit_cast(std::numeric_limits::max()); + } + + /** @brief Compares lanes for equality during constant evaluation. */ + [[nodiscard]] constexpr static vector_t compare_equal_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + for (std::size_t index = 0; index < element_count; ++index) + result[index] = left[index] == right[index] ? comparison_true_lane() : element_t{}; + return construct(result); + } + + /** @brief Compares lanes for greater-than ordering during constant evaluation. */ + [[nodiscard]] constexpr static vector_t compare_greater_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + for (std::size_t index = 0; index < element_count; ++index) + result[index] = left[index] > right[index] ? comparison_true_lane() : element_t{}; + return construct(result); + } + + /** @brief Compares lanes for greater-than-or-equal ordering during constant evaluation. */ + [[nodiscard]] constexpr static vector_t compare_greater_equal_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + for (std::size_t index = 0; index < element_count; ++index) + result[index] = left[index] >= right[index] ? comparison_true_lane() : element_t{}; + return construct(result); + } + + /** @brief Compares lanes for less-than ordering during constant evaluation. */ + [[nodiscard]] constexpr static vector_t compare_less_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + for (std::size_t index = 0; index < element_count; ++index) + result[index] = left[index] < right[index] ? comparison_true_lane() : element_t{}; + return construct(result); + } + + /** @brief Compares lanes for less-than-or-equal ordering during constant evaluation. */ + [[nodiscard]] constexpr static vector_t compare_less_equal_constexpr(const vector_t lhs, const vector_t rhs) noexcept + { + const auto left = to_array(lhs); + const auto right = to_array(rhs); + std::array result{}; + for (std::size_t index = 0; index < element_count; ++index) + result[index] = left[index] <= right[index] ? comparison_true_lane() : element_t{}; + return construct(result); + } + + /** @brief Left-shifts integer lanes during constant evaluation. + * @param lhs Input integer register represented in constant evaluation. + * @param shift Shift count applied to each lane. + * @return Register containing shifted lanes. + */ + constexpr static int_vector_t shift_left_constexpr(const int_vector_t lhs, const int shift) noexcept + { + if (shift >= static_cast(element_width)) + return impl::setzero(); + std::array results{}; + for (std::size_t index = 0; index < element_count; ++index) + results[index] = static_cast(extract_constexpr(lhs, static_cast(index)) << shift); + return impl::construct(results); + } + + /** @brief Logically right-shifts integer lanes during constant evaluation. + * @param lhs Input integer register represented in constant evaluation. + * @param shift Shift count applied to each lane. + * @return Register containing shifted lanes. + */ + constexpr static int_vector_t shift_right_constexpr(const int_vector_t lhs, const int shift) noexcept + { + if (shift >= static_cast(element_width)) + return impl::setzero(); + std::array results{}; + for (std::size_t index = 0; index < element_count; ++index) + { + results[index] = static_cast(static_cast>(extract_constexpr(lhs, static_cast(index))) >> shift); + } + return impl::construct(results); + } + + /** @brief Arithmetically right-shifts integer lanes during constant evaluation. + * @param lhs Input integer register represented in constant evaluation. + * @param shift Shift count applied to each lane. + * @return Register containing shifted lanes. + */ + constexpr static int_vector_t shift_right_arithmetic_constexpr(const int_vector_t lhs, int shift) noexcept + { + if (shift >= static_cast(element_width)) + shift = static_cast(element_width) - 1; + std::array results{}; + for (std::size_t index = 0; index < element_count; ++index) + results[index] = static_cast(extract_constexpr(lhs, static_cast(index)) >> shift); + return impl::construct(results); + } + + /** @brief Shifts a complete register toward higher byte indices during constant evaluation. + * @param lhs Input integer register represented in constant evaluation. + * @param shift Runtime-compatible byte count. + * @return Byte-shifted register. + */ + constexpr static int_vector_t shift_bytes_left_constexpr(const int_vector_t lhs, const int shift) noexcept + { + if (shift <= 0) + return lhs; + if (shift >= static_cast(byte_count)) + return impl::setzero(); + const auto sourceBytes = std::bit_cast>(to_array(lhs)); + std::array resultBytes{}; + for (std::size_t index = static_cast(shift); index < byte_count; ++index) + resultBytes[index] = sourceBytes[index - static_cast(shift)]; + return construct(std::bit_cast>(resultBytes)); + } + + /** @brief Shifts a complete register toward lower byte indices during constant evaluation. + * @param lhs Input integer register represented in constant evaluation. + * @param shift Runtime-compatible byte count. + * @return Byte-shifted register. + */ + constexpr static int_vector_t shift_bytes_right_constexpr(const int_vector_t lhs, const int shift) noexcept + { + if (shift <= 0) + return lhs; + if (shift >= static_cast(byte_count)) + return impl::setzero(); + const auto sourceBytes = std::bit_cast>(to_array(lhs)); + std::array resultBytes{}; + for (std::size_t index = 0; index + static_cast(shift) < byte_count; ++index) + resultBytes[index] = sourceBytes[index + static_cast(shift)]; + return construct(std::bit_cast>(resultBytes)); + } + + /** + * @brief Shifts a complete 128-bit register left during constant evaluation. + * @param lhs Input integer register represented in constant evaluation. + * @param shift Runtime-compatible bit count. + * @return Shifted register with zero-filled low bits. + */ + constexpr static int_vector_t shift_bits_left_constexpr(const int_vector_t lhs, const int shift) noexcept + { + if (shift <= 0) + return lhs; + if (shift >= 128) + return impl::setzero(); + + const auto source = std::bit_cast>(to_array(lhs)); + std::array result{}; + if (shift < 64) + { + result = {source[0] << shift, (source[1] << shift) | (source[0] >> (64 - shift))}; + } + else if (shift == 64) + { + result = {0, source[0]}; + } + else + { + result = {0, source[0] << (shift - 64)}; + } + return construct(std::bit_cast>(result)); + } + + /** + * @brief Shifts a complete 128-bit register right during constant evaluation. + * @param lhs Input integer register represented in constant evaluation. + * @param shift Runtime-compatible bit count. + * @return Shifted register with zero-filled high bits. + */ + constexpr static int_vector_t shift_bits_right_constexpr(const int_vector_t lhs, const int shift) noexcept + { + if (shift <= 0) + return lhs; + if (shift >= 128) + return impl::setzero(); + + const auto source = std::bit_cast>(to_array(lhs)); + std::array result{}; + if (shift < 64) + { + result = {(source[0] >> shift) | (source[1] << (64 - shift)), source[1] >> shift}; + } + else if (shift == 64) + { + result = {source[1], 0}; + } + else + { + result = {source[1] >> (shift - 64), 0}; + } + return construct(std::bit_cast>(result)); + } + /** @brief Re-encodes integer lanes so a minimum-position backend yields the first maximum index. * @param lhs Input integer register. * @return Transformed register whose first minimum corresponds to the original first maximum. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL TransformForMaxPosition(const vector_t lhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) TransformForMaxPosition(const vector_t lhs) noexcept { if constexpr (using_unsigned) { @@ -1558,6 +2281,6 @@ struct Api : public Detail::SimdMappings */ template requires ApiAvailable<128, element_t> -using NativeApi = Api ? 256 : 128, element_t>; +using NativeApi = Api<(is_api_available_v<256, element_t> ? 256 : 128), element_t>; } // namespace SimdLib diff --git a/include/SimdLib/Bmi.h b/include/SimdLib/Bmi.h index 0210011..d445981 100644 --- a/include/SimdLib/Bmi.h +++ b/include/SimdLib/Bmi.h @@ -19,15 +19,14 @@ namespace SimdLib::Bmi { template -concept integer_like = std::numeric_limits::is_specialized && std::numeric_limits::is_integer && - !std::same_as, bool>; +concept integer_like = std::numeric_limits::is_specialized && std::numeric_limits::is_integer && !std::same_as, bool>; #pragma region Pre-Optimized Generic Integer Operations // These methods are versions of common std methods that would usually optimize down into roughtly the same code as is written here, but we optimize these ahead // of time to guarantee the optimizations will be applied even in debug builds. /// @brief Turns a boolean value into an integer-width bitmask of all ones or zeros (0 for false, all 1s for true). -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t boolmask(const bool state) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) boolmask(const bool state) noexcept { if constexpr (std::is_integral_v) { @@ -42,27 +41,27 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati /// @brief Branchless selection between two values based on a switch bit. /// @param selectionBit The bit that will determine which value to select. (0 = lhs, 1 = rhs) template -[[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t select(const int_t lhs, const int_t rhs, const bool selectionBit) noexcept +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) select(const int_t lhs, const int_t rhs, const bool selectionBit) noexcept { const int_t mask = boolmask(selectionBit); return (~mask & lhs) | (rhs & mask); // Select between lhs and rhs } /// @brief Branchless find maximum of two values. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t max(const int_t lhs, const int_t rhs) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) max(const int_t lhs, const int_t rhs) noexcept { return select(lhs, rhs, lhs < rhs); } /// @brief Branchless find minimum of two values. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t min(const int_t lhs, const int_t rhs) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) min(const int_t lhs, const int_t rhs) noexcept { return select(lhs, rhs, lhs > rhs); } /// @brief Branchless find absolute value of the input. /// @note For the minimum signed value, returns the unchanged two's-complement magnitude bit pattern because its positive magnitude is not representable. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t abs(const int_t lhs) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) abs(const int_t lhs) noexcept { if constexpr (std::is_signed_v) { @@ -78,11 +77,6 @@ template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INL } } - - - - - #pragma endregion // Common Building Blocks #pragma region BMI Cannon Intrinsics @@ -91,22 +85,22 @@ namespace Detail { template using unsigned_t = std::make_unsigned_t; -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t from_unsigned(const unsigned_t value) noexcept +template [[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) from_unsigned(const unsigned_t value) noexcept { return std::bit_cast(value); } -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr unsigned_t to_unsigned(const int_t value) noexcept +template [[nodiscard]] constexpr unsigned_t SIMD_FLAGS(Neither, ForceInline) to_unsigned(const int_t value) noexcept { return std::bit_cast>(value); } -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t portable_andn(const int_t lhs, const int_t rhs) noexcept +template [[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) portable_andn(const int_t lhs, const int_t rhs) noexcept { return from_unsigned(to_unsigned(rhs) & ~to_unsigned(lhs)); } -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t portable_bzhi(const int_t source, const unsigned index) noexcept +template [[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) portable_bzhi(const int_t source, const unsigned index) noexcept { using unsigned_type = unsigned_t; constexpr unsigned bit_count = static_cast(sizeof(int_t) * 8u); @@ -122,21 +116,21 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_ return from_unsigned(to_unsigned(source) & mask); } -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t portable_blsi(const int_t source) noexcept +template [[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) portable_blsi(const int_t source) noexcept { using unsigned_type = unsigned_t; const unsigned_type value = to_unsigned(source); return from_unsigned(value & (unsigned_type{0} - value)); } -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t portable_blsr(const int_t source) noexcept +template [[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) portable_blsr(const int_t source) noexcept { using unsigned_type = unsigned_t; const unsigned_type value = to_unsigned(source); return from_unsigned(value & (value - unsigned_type{1})); } -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t portable_blsmsk(const int_t source) noexcept +template [[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) portable_blsmsk(const int_t source) noexcept { using unsigned_type = unsigned_t; const unsigned_type value = to_unsigned(source); @@ -144,7 +138,7 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_ } template -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t portable_mulx(const int_t lhs, const int_t rhs, int_t &hi) noexcept +[[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) portable_mulx(const int_t lhs, const int_t rhs, int_t &hi) noexcept requires(!std::same_as, bool>) { using unsigned_type = unsigned_t; @@ -181,21 +175,21 @@ template } // namespace Detail /// @brief Compute the bitwise NOT of LHS and then AND with RHS. -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t andn(const int_t lhs, const int_t rhs) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) andn(const int_t lhs, const int_t rhs) noexcept { if constexpr (std::integral) { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_BMI1 if (!std::is_constant_evaluated()) { - #if SIMDLIB_TARGET_X64 +#if SIMDLIB_TARGET_X64 if constexpr (sizeof(int_t) == sizeof(std::uint64_t)) { return static_cast(_andn_u64(static_cast(lhs), static_cast(rhs))); } else - #endif - if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) +#endif + if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) { return static_cast(_andn_u32(static_cast(lhs), static_cast(rhs))); } @@ -210,21 +204,21 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati } /// @brief Copy all bits from source integer, and reset (set to 0) the high bits in output starting at index. -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t bzhi(const int_t source, unsigned index) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) bzhi(const int_t source, unsigned index) noexcept { if constexpr (std::integral) { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_BMI2 if (!std::is_constant_evaluated()) { - #if SIMDLIB_TARGET_X64 +#if SIMDLIB_TARGET_X64 if constexpr (sizeof(int_t) == sizeof(std::uint64_t)) { return static_cast(_bzhi_u64(static_cast(source), index)); } else - #endif - if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) +#endif + if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) { return static_cast(_bzhi_u32(static_cast(source), index)); } @@ -248,19 +242,19 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati /// @brief Extract the lowest set bit from source integer and set the corresponding bit in dst. All other bits in dst are zeroed, and all bits are zeroed if no /// bits are set in source. -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t blsi(const int_t source) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) blsi(const int_t source) noexcept { if constexpr (std::integral) { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_BMI1 if (!std::is_constant_evaluated()) { - #if SIMDLIB_TARGET_X64 +#if SIMDLIB_TARGET_X64 if constexpr (sizeof(int_t) == sizeof(std::uint64_t)) return static_cast(_blsi_u64(static_cast(source))); else - #endif - if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) +#endif + if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) return static_cast(_blsi_u32(static_cast(source))); } #endif @@ -271,19 +265,19 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati } /// @brief Copy all bits from source to dst, and reset (set to 0) the bit in dst that corresponds to the lowest set bit in source. -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t blsr(const int_t source) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) blsr(const int_t source) noexcept { if constexpr (std::integral) { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_BMI1 if (!std::is_constant_evaluated()) { - #if SIMDLIB_TARGET_X64 +#if SIMDLIB_TARGET_X64 if constexpr (sizeof(int_t) == sizeof(std::uint64_t)) return static_cast(_blsr_u64(static_cast(source))); else - #endif - if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) +#endif + if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) return static_cast(_blsr_u32(static_cast(source))); } #endif @@ -294,7 +288,7 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati } /// @brief Extract and reset the lowest set bit in source. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t blse(const int_t source, int_t &out_lsb) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) blse(const int_t source, int_t &out_lsb) noexcept { out_lsb = blsi(source); return source ^ out_lsb; @@ -302,7 +296,8 @@ template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLI /// @brief Extract and reset the lowest set bit in source. /// @return A tuple containing the source integer with the bits reset and the extracted bits. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static std::tuple blse(const int_t source) noexcept +template +[[nodiscard]] constexpr static std::tuple SIMD_FLAGS(Neither, ForceInline, Flatten) blse(const int_t source) noexcept { const int_t out_lsb = blsi(source); return {static_cast(source ^ out_lsb), out_lsb}; @@ -318,25 +313,26 @@ template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLI * @param starting_bit A one-hot bit defining the inclusive lower boundary. * @return The matching bit as a one-hot value, or zero if none exists. */ -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t blsioff(const int_t source, const int_t starting_bit) noexcept +template +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) blsioff(const int_t source, const int_t starting_bit) noexcept { return source & static_cast(~source + starting_bit); } /// @brief Set all the lower bits of dst up to and including the lowest set bit in source. -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t blsmsk(const int_t source) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) blsmsk(const int_t source) noexcept { if constexpr (std::integral) { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_BMI1 if (!std::is_constant_evaluated()) { - #if SIMDLIB_TARGET_X64 +#if SIMDLIB_TARGET_X64 if constexpr (sizeof(int_t) == sizeof(std::uint64_t)) return static_cast(_blsmsk_u64(static_cast(source))); else - #endif - if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) +#endif + if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) return static_cast(_blsmsk_u32(static_cast(source))); } #endif @@ -358,13 +354,13 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati * @return The low word of the full product. */ template -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t mulx(const int_t lhs, const int_t rhs, int_t &hi) noexcept +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) mulx(const int_t lhs, const int_t rhs, int_t &hi) noexcept requires(!std::same_as, bool>) { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_BMI2 if (!std::is_constant_evaluated()) { - #if SIMDLIB_TARGET_X64 +#if SIMDLIB_TARGET_X64 if constexpr (sizeof(int_t) == sizeof(std::uint64_t)) { unsigned long long intrinsic_hi = 0; @@ -373,8 +369,8 @@ template return static_cast(low); } else - #endif - if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) +#endif + if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) { #if !defined(__GNUC__) || defined(__clang__) || defined(__i386__) unsigned int intrinsic_hi = 0; @@ -393,35 +389,37 @@ template #pragma region Parallel Prefix/Suffix Operations /// @brief Computes a distance-1 parallel-prefix XOR stage by XORing each bit with its adjacent bit to the right (high-bits). [eg: pp_xor(0b01110) => 0b01001] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t pp_xor(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) pp_xor(const int_t value) noexcept { return (value >> 1) ^ value; } /// @brief Computes a distance-1 parallel-suffix XOR stage by XORing each bit with its adjacent bit to the left (low-bits). [eg: ps_xor(0b01110) => 0b10010] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t ps_xor(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) ps_xor(const int_t value) noexcept { return (value << 1) ^ value; } -/// @brief Computes the parallel-prefix OR of the given value, which is the result of or'ing each bit with all bits to the left (low-bits). [eg: 10100 => 11111 ] -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t pp_or(const int_t value) noexcept +/// @brief Computes the parallel-prefix OR of the given value, which is the result of or'ing each bit with all bits to the left (low-bits). [eg: 10100 => 11111 +/// ] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) pp_or(const int_t value) noexcept { using Bmi::bzhi; using std::bit_width; return bzhi(std::numeric_limits::max(), bit_width(value)); } -/// @brief Computes the parallel-suffix OR of the given value, which is the result of or'ing each bit with all bits to the right (high-bits). [eg: 010100 => 1...100 ] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t ps_or(const int_t value) noexcept +/// @brief Computes the parallel-suffix OR of the given value, which is the result of or'ing each bit with all bits to the right (high-bits). [eg: 010100 +/// => 1...100 ] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) ps_or(const int_t value) noexcept { return value | (int_t{0} - value); // return value | ((~value) + 1); } -/// @brief Computes the parallel-prefix-least-significant-OR of the given value, which is the result of clearing all bits to the right (high-bits) of the lsb and -/// then or'ing each bit with all bits to the left (low-bits). [eg: 10100 => 00111 ] -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t pp_lsor(const int_t value) noexcept +/// @brief Computes the parallel-prefix-least-significant-OR of the given value, which is the result of clearing all bits to the right (high-bits) of the lsb +/// and then or'ing each bit with all bits to the left (low-bits). [eg: 10100 => 00111 ] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) pp_lsor(const int_t value) noexcept { using Bmi::blsi; using Bmi::bzhi; @@ -429,41 +427,47 @@ template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLI return bzhi(std::numeric_limits::max(), bit_width(blsi(value))); } -/// @brief Computes a distance-1 parallel-prefix AND stage by ANDing each bit with its adjacent bit to the right (high-bits). [eg: pp_and(0b01101110) => 0b00100110] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t pp_and(const int_t value) noexcept +/// @brief Computes a distance-1 parallel-prefix AND stage by ANDing each bit with its adjacent bit to the right (high-bits). [eg: pp_and(0b01101110) => +/// 0b00100110] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) pp_and(const int_t value) noexcept { return value & (value >> 1); } -/// @brief Computes a distance-1 parallel-suffix AND stage by ANDing each bit with its adjacent bit to the left (low-bits). [eg: ps_and(0b01101110) => 0b01001100] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t ps_and(const int_t value) noexcept +/// @brief Computes a distance-1 parallel-suffix AND stage by ANDing each bit with its adjacent bit to the left (low-bits). [eg: ps_and(0b01101110) => +/// 0b01001100] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) ps_and(const int_t value) noexcept { return value & (value << 1); } -/// @brief Computes a distance-1 parallel-prefix AND-NOT stage, retaining set bits whose adjacent bit to the right (high-bits) is clear. [eg: pp_andn(0b01110) => 0b01000] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t pp_andn(const int_t value) noexcept +/// @brief Computes a distance-1 parallel-prefix AND-NOT stage, retaining set bits whose adjacent bit to the right (high-bits) is clear. [eg: pp_andn(0b01110) +/// => 0b01000] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) pp_andn(const int_t value) noexcept { using Bmi::andn; return andn(value >> 1, value); } -/// @brief Computes a distance-1 parallel-suffix AND-NOT stage, retaining set bits whose adjacent bit to the left (low-bits) is clear. [eg: ps_andn(0b01110) => 0b00010] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t ps_andn(const int_t value) noexcept +/// @brief Computes a distance-1 parallel-suffix AND-NOT stage, retaining set bits whose adjacent bit to the left (low-bits) is clear. [eg: ps_andn(0b01110) => +/// 0b00010] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) ps_andn(const int_t value) noexcept { using Bmi::andn; return andn(value << 1, value); } -/// @brief Computes an inverse distance-1 parallel-prefix AND-NOT stage, marking clear bits whose adjacent bit to the right (high-bits) is set. [eg: pp_andni(0b01110) => 0b00001] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t pp_andni(const int_t value) noexcept +/// @brief Computes an inverse distance-1 parallel-prefix AND-NOT stage, marking clear bits whose adjacent bit to the right (high-bits) is set. [eg: +/// pp_andni(0b01110) => 0b00001] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) pp_andni(const int_t value) noexcept { using Bmi::andn; return andn(value, value >> 1); } -/// @brief Computes an inverse distance-1 parallel-suffix AND-NOT stage, marking clear bits whose adjacent bit to the left (low-bits) is set. [eg: ps_andni(0b01110) => 0b10000] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t ps_andni(const int_t value) noexcept +/// @brief Computes an inverse distance-1 parallel-suffix AND-NOT stage, marking clear bits whose adjacent bit to the left (low-bits) is set. [eg: +/// ps_andni(0b01110) => 0b10000] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) ps_andni(const int_t value) noexcept { using Bmi::andn; return andn(value, value << 1); @@ -473,7 +477,7 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati #pragma region BMI Extended Operations /// @brief Extract the highest set bit from source integer and set the corresponding bit in dst. All other bits in dst are zeroed, and all bits are zeroed if no /// bits are set in source. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t bmsi(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) bmsi(const int_t value) noexcept { using std::bit_floor; return bit_floor(value); @@ -483,7 +487,7 @@ template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLI } /// @brief Copy all bits from source to dst, and reset (set to 0) the bit in dst that corresponds to the highest set bit in source. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t bmsr(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) bmsr(const int_t value) noexcept { using Bmi::bzhi; using std::bit_width; @@ -492,7 +496,8 @@ template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLI } /// @brief Copy all bits from source to dst, and reset (set to 0) the bit in dst that corresponds to the highest set bit in source. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t bmsr(const int_t value, int &out_msb_index) noexcept +template +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) bmsr(const int_t value, int &out_msb_index) noexcept { using std::bit_width; out_msb_index = bit_width(value) - 1; @@ -500,7 +505,7 @@ template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLI } /// @brief Extract and reset the highest set bit in source. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static int_t bmse(const int_t value, int_t &out_msb) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline, Flatten) bmse(const int_t value, int_t &out_msb) noexcept { using std::bit_floor; out_msb = bit_floor(value); @@ -509,7 +514,7 @@ template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLI /// @brief Extract and reset the highest set bit in source. /// @return A tuple containing the source integer with the bits reset and the extracted bits. -template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLINE constexpr static std::tuple bmse(const int_t value) noexcept +template [[nodiscard]] constexpr static std::tuple SIMD_FLAGS(Neither, ForceInline, Flatten) bmse(const int_t value) noexcept { using std::bit_floor; const int_t msb = bit_floor(value); @@ -526,20 +531,20 @@ template [[nodiscard]] [[msvc::flatten]] SIMDLIB_FORCE_INLI * @param index The exclusive upper bound of the cleared bit-index range. * @return The source value with bits in [0, index) cleared. */ -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t bzlo(const int_t source, unsigned index) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) bzlo(const int_t source, unsigned index) noexcept { return andn(bzhi(~int_t{0}, index), source); } /// @brief Set all the lower bits of dst up to and including the highest set bit in source. -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t bmsmsk(const int_t source) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) bmsmsk(const int_t source) noexcept { return pp_or(source); } #pragma endregion #pragma region Common Building Blocks -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t PartialSumBLSMSK(const int_t n) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) PartialSumBLSMSK(const int_t n) noexcept { int_t sum = n; sum += (n & 0xAAAAAAAA); @@ -550,7 +555,7 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati return sum; } -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t PartialSumBLSI(const int_t n) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) PartialSumBLSI(const int_t n) noexcept { int_t sum = n; sum += (n & 0xAAAAAAAA) >> 1; @@ -562,39 +567,39 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati } /// @brief Sets the least significant, leftmost (low-bits) unset bit. [eg: 01011 => 01111] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t flipr_unset(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) flipr_unset(const int_t value) noexcept { return value | (value + 1); } /// @brief Returns a single 1-bit at the position of the leftmost (low-bits) 0-bit, producing 0 if none. [eg: 01011 => 00100] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t maskr_unset(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) maskr_unset(const int_t value) noexcept { using Bmi::blsi; return blsi(~value); } /// @brief Returns a single 1-bit at the position of the rightmost (high-bits) trailing 1-bit, producing 0 if none. [eg: 010111 => 00100] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t maskl_trailing_one(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) maskl_trailing_one(const int_t value) noexcept { using Bmi::blsi; return blsi(~value) >> 1; } /// @brief Clears all least significant, leftmost (low-bits) trailing set bits. [eg: 1011 => 1000] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t clear_trailing_ones(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) clear_trailing_ones(const int_t value) noexcept { return value & (value + 1); } /// @brief Sets all least significant, leftmost (low-bits) trailing unset bits. [eg: 10100 => 10111] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t flip_trailing_zeros(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) flip_trailing_zeros(const int_t value) noexcept { return value | (value - 1); } /// @brief Returns a mask over the trailing 0-bits in the source integer. [eg: 10100 => 011] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t mask_trailing_zeros(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) mask_trailing_zeros(const int_t value) noexcept { using Bmi::blsi; return blsi(value) - 1; @@ -603,7 +608,7 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati /// @brief Returns a mask over the trailing 0-bits in the source integer. /// For value==0, returns 0 ("safe" variant; avoids the wraparound/all-ones behavior). /// [eg: 10100 => 00011] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t mask_trailing_zeros_or_zero(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) mask_trailing_zeros_or_zero(const int_t value) noexcept { return boolmask(value != 0) & mask_trailing_zeros(value); } @@ -611,7 +616,7 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati /// @brief Returns a mask of all bits strictly lower than the least-significant set bit (LSB). /// For value==0, returns 0. /// [eg: 101000 => 000111] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t mask_bits_lower_than_lsb(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) mask_bits_lower_than_lsb(const int_t value) noexcept { // `ps_or(value)` sets bits from the LSB up to MSB (and higher) to 1; inverting yields exactly the bits below the LSB. // For value==0, ps_or(0)==0, so ~ps_or(0) would be all-ones; mask it out. @@ -621,20 +626,21 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati /// @brief Returns a mask of all bits strictly lower than the least-significant set bit (LSB). /// For value==0, returns all-ones (useful as a "no constraint" mask). /// [eg: 101000 => 000111] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t mask_bits_lower_than_lsb_or_all_ones(const int_t value) noexcept +template +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) mask_bits_lower_than_lsb_or_all_ones(const int_t value) noexcept { return ~ps_or(value); } /// @brief Returns a mask over the trailing 1-bits in the source integer, producing 0 if none. [eg: 10111 => 00111] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t mask_trailing_ones(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) mask_trailing_ones(const int_t value) noexcept { using Bmi::blsi; return blsi(~value) - int_t{1}; } /// @brief Returns a mask over the leading zeros in the source integer. [eg: 000101 => 111000] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t mask_leading_zeros(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) mask_leading_zeros(const int_t value) noexcept { /*using Bmi::bzhi; using std::bit_width; @@ -643,45 +649,48 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati } /// @brief Returns a mask over the leading ones in the source integer. [eg: 111011 => 111000] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t mask_leading_ones(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) mask_leading_ones(const int_t value) noexcept { return ~pp_or(static_cast(~value)); } /// @brief Clears all most significant, rightmost (high-bits) leading set bits. [eg: 110101 => 000101] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t clear_leading_ones(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) clear_leading_ones(const int_t value) noexcept { return pp_or(static_cast(~value)) & value; } /// @brief Copy all bits from the source integer, and reset (set to 0) the leftmost (low-bits) string of contiguous set bits. [eg: 1011 => 1000] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t clear_lowest_set_bits(const int_t value) noexcept +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) clear_lowest_set_bits(const int_t value) noexcept { return value & ((value | (value - int_t{1})) + int_t{1}); } -/// @brief Copy all bits from the source integer, and reset (set to 0) the leftmost (low-bits) string of contiguous set bits after copying said bits into the provided -/// integer address. [eg: 1011 => 1000] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t clear_lowest_set_bits(const int_t value, int_t &out_consumed) noexcept +/// @brief Copy all bits from the source integer, and reset (set to 0) the leftmost (low-bits) string of contiguous set bits after copying said bits into the +/// provided integer address. [eg: 1011 => 1000] +template +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) clear_lowest_set_bits(const int_t value, int_t &out_consumed) noexcept { const int_t mask = ((value | (value - int_t{1})) + int_t{1}); out_consumed = value ^ mask; return value & mask; } -/// @brief Extracts and returns the leftmost (low-bits) string of contiguous set bits, said bits are also reset (set to 0) within the source integer. [eg: 1011 => 0011] +/// @brief Extracts and returns the leftmost (low-bits) string of contiguous set bits, said bits are also reset (set to 0) within the source integer. [eg: 1011 +/// => 0011] /// @return A tuple containing the source integer with the bits reset and the extracted bits. template -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static std::tuple consume_bit_sequence_right(const int_t value) noexcept +[[nodiscard]] constexpr static std::tuple SIMD_FLAGS(Neither, ForceInline) consume_bit_sequence_right(const int_t value) noexcept { const int_t mask = ((value | (value - int_t{1})) + int_t{1}); return {static_cast(value & mask), static_cast(value & ~mask)}; } -/// @brief Extracts and returns the rightmost (high-bits) string of contiguous set bits, said bits are also reset (set to 0) within the source integer. [eg: 0110111 => -/// 0110000] +/// @brief Extracts and returns the rightmost (high-bits) string of contiguous set bits, said bits are also reset (set to 0) within the source integer. [eg: +/// 0110111 => 0110000] /// @return A tuple containing the source integer with the bits reset and the extracted bits. -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static std::tuple consume_bit_sequence_left(const int_t value) noexcept +template +[[nodiscard]] constexpr static std::tuple SIMD_FLAGS(Neither, ForceInline) consume_bit_sequence_left(const int_t value) noexcept { using Bmi::andn; const int_t thresholds = ps_andn(value); @@ -689,8 +698,9 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati return {static_cast(value & seq_mask), andn(seq_mask, value)}; } -/// @brief Copy all bits from the source integer, and reset (set to 0) the trailing bits up-to but excluding the rightmost (high-bits) trailing set bit. [eg: 10111 => 10100] -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t left_collapse_trailing_bits(const int_t value) noexcept +/// @brief Copy all bits from the source integer, and reset (set to 0) the trailing bits up-to but excluding the rightmost (high-bits) trailing set bit. [eg: +/// 10111 => 10100] +template [[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) left_collapse_trailing_bits(const int_t value) noexcept { using Bmi::andn; return andn(mask_trailing_ones(value) >> 1, value); @@ -698,7 +708,7 @@ template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr stati /// @brief Clears all bits lower than (not including) the given target-bit from the source integer. [eg: (10111, 100) => 10100] template -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t clear_bits_lower_than(const int_t value, const int_t target_bit) noexcept +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) clear_bits_lower_than(const int_t value, const int_t target_bit) noexcept { using Bmi::andn; return andn(target_bit - int_t{1}, value); @@ -706,7 +716,7 @@ template /// @brief Clears all bits higher than (not including) the given target-bit from the source integer. [eg: (10111, 100) => 00111] template -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t clear_bits_higher_than(const int_t value, const int_t target_bit) noexcept +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) clear_bits_higher_than(const int_t value, const int_t target_bit) noexcept { using Bmi::blsmsk; return blsmsk(target_bit) & value; @@ -714,14 +724,14 @@ template /// @brief Extracts all bits lower than (not including) the given target-bit from the source integer. [eg: (10111, 100) => 00011] template -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t extract_bits_lower_than(const int_t value, const int_t target_bit) noexcept +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) extract_bits_lower_than(const int_t value, const int_t target_bit) noexcept { return value & (target_bit - int_t{1}); } /// @brief Extracts all bits higher than (not including) the given target-bit from the source integer. [eg: (10111, 001) => 10110] template -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static int_t extract_bits_higher_than(const int_t value, const int_t target_bit) noexcept +[[nodiscard]] constexpr static int_t SIMD_FLAGS(Neither, ForceInline) extract_bits_higher_than(const int_t value, const int_t target_bit) noexcept { using Bmi::andn; using Bmi::blsmsk; @@ -734,7 +744,7 @@ template namespace Detail { template -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t portable_bextr(const int_t source, const unsigned start, const unsigned len) noexcept +[[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) portable_bextr(const int_t source, const unsigned start, const unsigned len) noexcept { using unsigned_type = unsigned_t; constexpr unsigned bit_count = static_cast(sizeof(int_t) * 8u); @@ -756,19 +766,19 @@ template /// @brief Extract contiguous bits from source integer, and return them shifted to the LSB side of the output. Extract the number of bits specified by len, /// starting at the bit specified by start. template -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t bextr(const int_t source, const std::uint8_t len, const std::uint8_t start) noexcept +[[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) bextr(const int_t source, const std::uint8_t len, const std::uint8_t start) noexcept { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_BMI1 if (!std::is_constant_evaluated()) { - #if SIMDLIB_TARGET_X64 +#if SIMDLIB_TARGET_X64 if constexpr (sizeof(int_t) == sizeof(std::uint64_t)) { return static_cast(_bextr_u64(static_cast(source), start, len)); } else - #endif - if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) +#endif + if constexpr (sizeof(int_t) == sizeof(std::uint32_t)) { return static_cast(_bextr_u32(static_cast(source), start, len)); } @@ -779,7 +789,8 @@ template /// @brief Extract contiguous bits from source integer, and return them shifted to the LSB side of the output. Extract the number of bits specified by len, /// starting at the bit specified by start. -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t bextr(const int_t source, const std::uint8_t start) noexcept +template +[[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) bextr(const int_t source, const std::uint8_t start) noexcept { static_assert(len <= 255, "BMI bit-extract length must fit the intrinsic control field"); return bextr(source, static_cast(len), start); @@ -787,7 +798,8 @@ template [[nodiscard]] SIMDLIB_FORCE_INLI /// @brief Extract contiguous bits from source integer, and return them shifted to the LSB side of the output. Extract the number of bits specified by len, /// starting at the bit specified by start. -template [[nodiscard]] SIMDLIB_FORCE_INLINE constexpr int_t bextr(const int_t source) noexcept +template +[[nodiscard]] constexpr int_t SIMD_FLAGS(Neither, ForceInline) bextr(const int_t source) noexcept { static_assert(start <= 255 && len <= 255, "BMI bit-extract controls must fit the intrinsic control fields"); return bextr(source, static_cast(len), static_cast(start)); @@ -807,7 +819,7 @@ namespace Detail * @param mask The destination bit positions. * @return The deposited bit pattern. */ -template SIMDLIB_FORCE_INLINE constexpr int_t portable_pdep(int_t source, int_t mask) noexcept +template constexpr int_t SIMD_FLAGS(Neither, ForceInline) portable_pdep(int_t source, int_t mask) noexcept { using unsigned_type = std::make_unsigned_t; constexpr unsigned int_width = static_cast(sizeof(int_t) * 8u); @@ -836,7 +848,7 @@ template SIMDLIB_FORCE_INLINE constexpr int_t portable_pde } // namespace Detail /// @brief Note: This is a wrapper for the '_pdep_xxx' intrinsic providing compile-time emulation. -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static std::uint32_t pdep_u32(std::uint32_t source, std::uint32_t mask) noexcept +[[nodiscard]] constexpr static std::uint32_t SIMD_FLAGS(Neither, ForceInline) pdep_u32(std::uint32_t source, std::uint32_t mask) noexcept { #if SIMDLIB_TARGET_X64 && SIMDLIB_HAS_BMI2 if (!std::is_constant_evaluated()) @@ -846,7 +858,7 @@ template SIMDLIB_FORCE_INLINE constexpr int_t portable_pde } /// @brief Note: This is a wrapper for the '_pdep_xxx' intrinsic providing compile-time emulation. -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static std::uint64_t pdep_u64(std::uint64_t source, std::uint64_t mask) noexcept +[[nodiscard]] constexpr static std::uint64_t SIMD_FLAGS(Neither, ForceInline) pdep_u64(std::uint64_t source, std::uint64_t mask) noexcept { #if SIMDLIB_TARGET_X64 && SIMDLIB_HAS_BMI2 if (!std::is_constant_evaluated()) @@ -856,13 +868,13 @@ template SIMDLIB_FORCE_INLINE constexpr int_t portable_pde } /// @brief This is a "pdep, but from right (high-bits) to left (low-bits)" aka "expand left" -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static std::uint32_t pdepl_u32(std::uint32_t source, std::uint32_t mask) noexcept +[[nodiscard]] constexpr static std::uint32_t SIMD_FLAGS(Neither, ForceInline) pdepl_u32(std::uint32_t source, std::uint32_t mask) noexcept { return pdep_u32(source >> (std::popcount(~mask) & 31), mask); } /// @brief This is a "pdep, but from right (high-bits) to left (low-bits)" aka "expand left" -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static std::uint64_t pdepl_u64(std::uint64_t source, std::uint64_t mask) noexcept +[[nodiscard]] constexpr static std::uint64_t SIMD_FLAGS(Neither, ForceInline) pdepl_u64(std::uint64_t source, std::uint64_t mask) noexcept { return pdep_u64(source >> (std::popcount(~mask) & 63), mask); } @@ -875,11 +887,12 @@ namespace Detail /** * @brief Performs a portable parallel bit extraction for the width of int_t. * @tparam int_t The integral source, mask, and result type. - * @param source The source bits to extract. + * @param source + * The source bits to extract. * @param mask The source bit positions. * @return The extracted bits packed into the least-significant positions. */ -template SIMDLIB_FORCE_INLINE constexpr int_t portable_pext(int_t source, int_t mask) noexcept +template constexpr int_t SIMD_FLAGS(Neither, ForceInline) portable_pext(int_t source, int_t mask) noexcept { using unsigned_type = std::make_unsigned_t; constexpr unsigned int_width = static_cast(sizeof(int_t) * 8u); @@ -908,7 +921,7 @@ template SIMDLIB_FORCE_INLINE constexpr int_t portable_pex } // namespace Detail /// @brief Note: This is a wrapper for the '_pext_xxx' intrinsic, providing compile-time emulation. -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static std::uint32_t pext_u32(std::uint32_t source, std::uint32_t mask) noexcept +[[nodiscard]] constexpr static std::uint32_t SIMD_FLAGS(Neither, ForceInline) pext_u32(std::uint32_t source, std::uint32_t mask) noexcept { #if SIMDLIB_TARGET_X64 && SIMDLIB_HAS_BMI2 if (!std::is_constant_evaluated()) @@ -918,7 +931,7 @@ template SIMDLIB_FORCE_INLINE constexpr int_t portable_pex } /// @brief Note: This is a wrapper for the '_pext_xxx' intrinsic, providing compile-time emulation. -[[nodiscard]] SIMDLIB_FORCE_INLINE constexpr static std::uint64_t pext_u64(std::uint64_t source, std::uint64_t mask) noexcept +[[nodiscard]] constexpr static std::uint64_t SIMD_FLAGS(Neither, ForceInline) pext_u64(std::uint64_t source, std::uint64_t mask) noexcept { #if SIMDLIB_TARGET_X64 && SIMDLIB_HAS_BMI2 if (!std::is_constant_evaluated()) diff --git a/include/SimdLib/Config.h b/include/SimdLib/Config.h index fe6a010..35ea0f9 100644 --- a/include/SimdLib/Config.h +++ b/include/SimdLib/Config.h @@ -1,9 +1,9 @@ #pragma once -#include - -// All configuration macros are caller-overridable. Instruction-family values -// describe compiler-enabled code-generation features, not runtime CPU support. +// Configuration macros are caller-overridable except +// SIMDLIB_REGISTER_INTERFACE_AVAILABLE, which reports a language capability +// computed by SimdLib. Instruction-family values describe compiler-enabled +// code-generation features, not runtime CPU support. #ifndef SIMDLIB_COMPILER_CLANG #if defined(__clang__) @@ -29,6 +29,29 @@ #endif #endif +#if defined(SIMDLIB_REGISTER_INTERFACE_AVAILABLE) +#error "SIMDLIB_REGISTER_INTERFACE_AVAILABILITY_IS_COMPUTED: do not define SIMDLIB_REGISTER_INTERFACE_AVAILABLE" +#undef SIMDLIB_REGISTER_INTERFACE_AVAILABLE +#endif + +#if defined(__cpp_explicit_this_parameter) && __cpp_explicit_this_parameter >= 202110L +#define SIMDLIB_REGISTER_INTERFACE_AVAILABLE 1 +#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1944 && defined(_MSVC_LANG) && _MSVC_LANG > 202002L +#define SIMDLIB_REGISTER_INTERFACE_AVAILABLE 1 +#else +#define SIMDLIB_REGISTER_INTERFACE_AVAILABLE 0 +#endif + +// This caller-controlled signal requires the computed Register capability; it +// cannot enable or override that capability. +#ifndef SIMDLIB_REQUIRE_REGISTER_INTERFACE +#define SIMDLIB_REQUIRE_REGISTER_INTERFACE 0 +#endif + +#if SIMDLIB_REQUIRE_REGISTER_INTERFACE && !SIMDLIB_REGISTER_INTERFACE_AVAILABLE +#error "SIMDLIB_REGISTER_INTERFACE_UNAVAILABLE: SimdLib::Register requires C++23 explicit object parameter support" +#endif + #ifndef SIMDLIB_TARGET_X86 #if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) #define SIMDLIB_TARGET_X86 1 @@ -134,39 +157,176 @@ #endif #ifndef SIMDLIB_VECTORCALL_ENABLED -#if SIMDLIB_TARGET_X86 && (SIMDLIB_COMPILER_MSVC || SIMDLIB_COMPILER_CLANG) +#if SIMDLIB_TARGET_X86 && (SIMDLIB_COMPILER_MSVC || (SIMDLIB_COMPILER_CLANG && defined(_WIN32))) #define SIMDLIB_VECTORCALL_ENABLED 1 #else #define SIMDLIB_VECTORCALL_ENABLED 0 #endif #endif -// VECTORCALL is intentionally unprefixed: it is the library's externally -// configurable ABI-affecting calling convention. MSVC and Clang both accept -// the __vectorcall keyword in the same declarator positions. For a -// caller-supplied empty VECTORCALL also set -// SIMDLIB_VECTORCALL_ENABLED=0. -#ifndef VECTORCALL -#if SIMDLIB_VECTORCALL_ENABLED -#define VECTORCALL __vectorcall +/* + * Internal adapter: SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL + * @brief Reports whether the method-flags vector calling-convention adapter is active. + * @details A custom toolchain may override this capability together with + * SIMDLIB_METHOD_FLAGS_VECTORCALL before including this header. + */ +#ifndef SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL +#define SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL SIMDLIB_VECTORCALL_ENABLED +#endif + +/* + * Internal adapter: SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS + * @brief Reports whether RegisterOnly can suppress compiler stack-cookie instrumentation. + * @details A custom toolchain may override this capability together with + * SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS before including this header. + */ +#ifndef SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS +#define SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS SIMDLIB_COMPILER_MSVC +#endif + +/* + * Internal adapter: SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE + * @brief Reports whether ForceInline has an active compiler enforcement attribute. + * @details The adapter retains ordinary inline semantics when this capability is zero. + * A custom toolchain may override this capability together with + * SIMDLIB_METHOD_FLAGS_FORCE_INLINE before including this header. + */ +#ifndef SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE +#if SIMDLIB_COMPILER_MSVC || SIMDLIB_COMPILER_CLANG || SIMDLIB_COMPILER_GCC +#define SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE 1 +#else +#define SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE 0 +#endif +#endif + +/* + * Internal adapter: SIMDLIB_METHOD_FLAGS_HAS_FLATTEN + * @brief Reports whether Flatten has an active recursive-inlining attribute. + * @details A custom toolchain may override this capability together with + * SIMDLIB_METHOD_FLAGS_FLATTEN before including this header. + */ +#ifndef SIMDLIB_METHOD_FLAGS_HAS_FLATTEN +#if SIMDLIB_COMPILER_MSVC || SIMDLIB_COMPILER_CLANG || SIMDLIB_COMPILER_GCC +#define SIMDLIB_METHOD_FLAGS_HAS_FLATTEN 1 +#else +#define SIMDLIB_METHOD_FLAGS_HAS_FLATTEN 0 +#endif +#endif + +/* + * Internal adapter: SIMDLIB_METHOD_FLAGS_VECTORCALL + * @brief Placement-safe vector calling-convention adapter used by SIMD_FLAGS. + */ +#ifndef SIMDLIB_METHOD_FLAGS_VECTORCALL +#if SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL +#define SIMDLIB_METHOD_FLAGS_VECTORCALL __vectorcall +#else +#define SIMDLIB_METHOD_FLAGS_VECTORCALL +#endif +#endif + +/* + * Internal adapter: SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS + * @brief Placement-safe safe-buffer adapter used by the RegisterOnly flag. + */ +#ifndef SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS +#if SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS +#define SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS __declspec(safebuffers) +#else +#define SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS +#endif +#endif + +/* + * Internal adapter: SIMDLIB_METHOD_FLAGS_FORCE_INLINE + * @brief Placement-safe force-inline adapter used by the ForceInline flag. + */ +#ifndef SIMDLIB_METHOD_FLAGS_FORCE_INLINE +#if !SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE +#define SIMDLIB_METHOD_FLAGS_FORCE_INLINE inline +#elif SIMDLIB_COMPILER_MSVC +#define SIMDLIB_METHOD_FLAGS_FORCE_INLINE __forceinline +#elif SIMDLIB_COMPILER_CLANG || SIMDLIB_COMPILER_GCC +#define SIMDLIB_METHOD_FLAGS_FORCE_INLINE inline __attribute__((always_inline)) #else -#define VECTORCALL +#define SIMDLIB_METHOD_FLAGS_FORCE_INLINE inline #endif #endif -#ifndef SIMDLIB_FORCE_INLINE -#if SIMDLIB_COMPILER_MSVC -#define SIMDLIB_FORCE_INLINE [[msvc::forceinline]] inline -#elif SIMDLIB_COMPILER_CLANG -#define SIMDLIB_FORCE_INLINE [[clang::always_inline]] inline -#elif SIMDLIB_COMPILER_GCC -#define SIMDLIB_FORCE_INLINE [[gnu::always_inline]] inline +/* + * Internal adapter: SIMDLIB_METHOD_FLAGS_FLATTEN + * @brief Placement-safe recursive-inlining adapter used by the Flatten flag. + */ +#ifndef SIMDLIB_METHOD_FLAGS_FLATTEN +#if !SIMDLIB_METHOD_FLAGS_HAS_FLATTEN +#define SIMDLIB_METHOD_FLAGS_FLATTEN +#elif SIMDLIB_COMPILER_MSVC +#define SIMDLIB_METHOD_FLAGS_FLATTEN [[msvc::flatten]] +#elif SIMDLIB_COMPILER_CLANG || SIMDLIB_COMPILER_GCC +#define SIMDLIB_METHOD_FLAGS_FLATTEN __attribute__((flatten)) #else -#define SIMDLIB_FORCE_INLINE inline +#define SIMDLIB_METHOD_FLAGS_FLATTEN #endif #endif +#define SIMDLIB_DETAIL_FLAGS_CAT_RAW(left, right) left##right +#define SIMDLIB_DETAIL_FLAGS_CAT(left, right) SIMDLIB_DETAIL_FLAGS_CAT_RAW(left, right) + +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_ static_assert(false, "SIMDLIB_FLAGS_ERROR_EMPTY"); +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_Neither +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_In SIMDLIB_METHOD_FLAGS_VECTORCALL +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_Out SIMDLIB_METHOD_FLAGS_VECTORCALL +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_InOut SIMDLIB_METHOD_FLAGS_VECTORCALL + +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_RegisterOnly SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_ForceInline SIMDLIB_METHOD_FLAGS_FORCE_INLINE +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_Flatten SIMDLIB_METHOD_FLAGS_FLATTEN +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_RegisterOnly_ForceInline SIMDLIB_METHOD_FLAGS_FORCE_INLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_RegisterOnly_Flatten SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_ForceInline_Flatten SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_3_RegisterOnly_ForceInline_Flatten \ + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS + +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_RAW(mode) SIMDLIB_DETAIL_FLAGS_BOUNDARY_##mode +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY(mode) SIMDLIB_DETAIL_FLAGS_BOUNDARY_RAW(mode) +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_RAW(a) SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_##a +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1(a) SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_RAW(a) +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_RAW(a, b) SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_##a##_##b +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2(a, b) SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_RAW(a, b) +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_3_RAW(a, b, c) SIMDLIB_DETAIL_FLAGS_MODIFIERS_3_##a##_##b##_##c +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_3(a, b, c) SIMDLIB_DETAIL_FLAGS_MODIFIERS_3_RAW(a, b, c) + +#define SIMDLIB_DETAIL_FLAGS_1(boundary) SIMDLIB_DETAIL_FLAGS_BOUNDARY(boundary) +#define SIMDLIB_DETAIL_FLAGS_2(boundary, a) SIMDLIB_DETAIL_FLAGS_MODIFIERS_1(a) SIMDLIB_DETAIL_FLAGS_BOUNDARY(boundary) +#define SIMDLIB_DETAIL_FLAGS_3(boundary, a, b) SIMDLIB_DETAIL_FLAGS_MODIFIERS_2(a, b) SIMDLIB_DETAIL_FLAGS_BOUNDARY(boundary) +#define SIMDLIB_DETAIL_FLAGS_4(boundary, a, b, c) SIMDLIB_DETAIL_FLAGS_MODIFIERS_3(a, b, c) SIMDLIB_DETAIL_FLAGS_BOUNDARY(boundary) +#define SIMDLIB_DETAIL_FLAGS_5(...) static_assert(false, "SIMDLIB_FLAGS_ERROR_TOO_MANY"); + +#define SIMDLIB_DETAIL_FLAGS_ARITY_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, count, ...) count +#define SIMDLIB_DETAIL_FLAGS_ARITY_EXPAND(arguments) SIMDLIB_DETAIL_FLAGS_ARITY_IMPL arguments +#define SIMDLIB_DETAIL_FLAGS_ARITY(...) SIMDLIB_DETAIL_FLAGS_ARITY_EXPAND((__VA_ARGS__, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 3, 2, 1)) + +#define SIMDLIB_DETAIL_FLAGS_DISPATCH(count) SIMDLIB_DETAIL_FLAGS_CAT(SIMDLIB_DETAIL_FLAGS_, count) +#define SIMDLIB_DETAIL_FLAGS_EXPAND(...) __VA_ARGS__ + +/** + * @def SIMD_FLAGS + * @brief Declares a function's SIMD boundary and optimization promises. + * @param ... One required boundary mode followed by zero to three modifiers. + * @details The boundary is one of Neither, In, Out, or InOut. Modifiers are an + * ordered subsequence of RegisterOnly, ForceInline, and Flatten. Place the macro + * after the independently specified return type and immediately before the + * function name. Constructors, destructors, conversion operators, deduction + * guides, lambdas, virtual functions, explicit function-pointer types, + * coroutines, C-style variadic functions, extern-C functions, allocation + * functions, defaulted or deleted functions, and consteval functions are not + * supported. The macro records developer intent; it cannot inspect function + * signatures, bodies, template instantiations, or transitive callees. + */ +#define SIMD_FLAGS(...) SIMDLIB_DETAIL_FLAGS_EXPAND(SIMDLIB_DETAIL_FLAGS_DISPATCH(SIMDLIB_DETAIL_FLAGS_ARITY(__VA_ARGS__))(__VA_ARGS__)) + #ifndef SIMDLIB_PRECONDITION +#include #define SIMDLIB_PRECONDITION(condition, message) assert((condition) && (message)) #endif @@ -190,6 +350,10 @@ inline constexpr bool compiler_gcc = SIMDLIB_COMPILER_GCC != 0; inline constexpr bool target_x86 = SIMDLIB_TARGET_X86 != 0; inline constexpr bool target_x64 = SIMDLIB_TARGET_X64 != 0; inline constexpr bool vectorcall_enabled = SIMDLIB_VECTORCALL_ENABLED != 0; +inline constexpr bool method_flags_has_vectorcall = SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL != 0; +inline constexpr bool method_flags_has_safe_buffers = SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS != 0; +inline constexpr bool method_flags_has_force_inline = SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE != 0; +inline constexpr bool method_flags_has_flatten = SIMDLIB_METHOD_FLAGS_HAS_FLATTEN != 0; inline constexpr bool has_sse = SIMDLIB_HAS_SSE != 0; inline constexpr bool has_sse2 = SIMDLIB_HAS_SSE2 != 0; diff --git a/include/SimdLib/Detail/Extensions.h b/include/SimdLib/Detail/Extensions.h index 42053e0..676b07d 100644 --- a/include/SimdLib/Detail/Extensions.h +++ b/include/SimdLib/Detail/Extensions.h @@ -6,7 +6,12 @@ #include #include #include +#if SIMDLIB_TARGET_X86 +#include +#endif +#if SIMDLIB_COMPILER_MSVC && SIMDLIB_TARGET_X86 #include +#endif #include #include @@ -15,14 +20,84 @@ namespace SimdLib::Detail // This file contains SIMD extensions for 128-bit and 256-bit integer and floating-point types. // SEE: http://www.alfredklomp.com/programming/sse-intrinsics/ -/** Portable lane access for compiler-native x86 register types. - * MSVC exposes intrinsic registers as unions with named arrays, while Clang - * models them as - * vector types. Keep that compiler difference inside Detail. +/** + * @brief Reads one lane through the portable constant-evaluation representation. + * @tparam Element Scalar lane type. + * @tparam Vector Compiler-native register type. + * @param value Source register represented during constant evaluation. + * @param index Selected lane index. + * @return Selected scalar lane. + * @note This remains `constexpr`, rather than `consteval`, because C++20 + * runtime-callable `constexpr` wrappers pass their parameters through it. + */ +template + requires std::is_arithmetic_v && (sizeof(Vector) % sizeof(Element) == 0) +constexpr Element SIMD_FLAGS(Neither, ForceInline) register_get_constexpr(const Vector value, const std::size_t index) noexcept +{ +#if SIMDLIB_COMPILER_MSVC + if constexpr (sizeof(Vector) == 16) + { + if constexpr (std::is_integral_v && sizeof(Element) == 1 && std::is_unsigned_v) + return value.m128i_u8[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 1) + return value.m128i_i8[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 2 && std::is_unsigned_v) + return value.m128i_u16[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 2) + return value.m128i_i16[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 4 && std::is_unsigned_v) + return value.m128i_u32[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 4) + return value.m128i_i32[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 8 && std::is_unsigned_v) + return value.m128i_u64[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 8) + return value.m128i_i64[index]; + else if constexpr (std::same_as) + return value.m128_f32[index]; + else + return value.m128d_f64[index]; + } + else + { + if constexpr (std::is_integral_v && sizeof(Element) == 1 && std::is_unsigned_v) + return value.m256i_u8[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 1) + return value.m256i_i8[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 2 && std::is_unsigned_v) + return value.m256i_u16[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 2) + return value.m256i_i16[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 4 && std::is_unsigned_v) + return value.m256i_u32[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 4) + return value.m256i_i32[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 8 && std::is_unsigned_v) + return value.m256i_u64[index]; + else if constexpr (std::is_integral_v && sizeof(Element) == 8) + return value.m256i_i64[index]; + else if constexpr (std::same_as) + return value.m256_f32[index]; + else + return value.m256d_f64[index]; + } +#else + return std::bit_cast>(value)[index]; +#endif +} + +/** + * @brief Reads one runtime lane through the portable native-register representation. + * @tparam Element Scalar lane type. + * @tparam Vector Compiler-native register type. + * @param value Source register. + * @param index Selected lane index. + * @return Selected scalar lane. + * @note This fallback may materialize addressable storage and must not be used by register-only operation paths. */ template requires std::is_arithmetic_v && (sizeof(Vector) % sizeof(Element) == 0) -SIMDLIB_FORCE_INLINE constexpr Element register_get(const Vector value, const std::size_t index) noexcept +Element SIMD_FLAGS(Neither, ForceInline) register_get(const Vector value, const std::size_t index) noexcept { #if SIMDLIB_COMPILER_MSVC if constexpr (sizeof(Vector) == 16) @@ -76,9 +151,19 @@ SIMDLIB_FORCE_INLINE constexpr Element register_get(const Vector value, const st #endif } +/** + * @brief Replaces one lane through the portable constant-evaluation representation. + * @tparam Element Scalar lane type. + * @tparam Vector Compiler-native register type. + * @param value Register represented during constant evaluation. + * @param index Selected lane index. + * @param lane Replacement scalar lane. + * @note This remains `constexpr`, rather than `consteval`, because C++20 + * runtime-callable `constexpr` wrappers pass their parameters through it. + */ template requires std::is_arithmetic_v && (sizeof(Vector) % sizeof(Element) == 0) -SIMDLIB_FORCE_INLINE constexpr void register_set(Vector &value, const std::size_t index, const Element lane) noexcept +constexpr void SIMD_FLAGS(Neither, ForceInline) register_set_constexpr(Vector &value, const std::size_t index, const Element lane) noexcept { #if SIMDLIB_COMPILER_MSVC if constexpr (sizeof(Vector) == 16) @@ -136,86 +221,125 @@ SIMDLIB_FORCE_INLINE constexpr void register_set(Vector &value, const std::size_ template requires(sizeof(Vector) == sizeof(Element) * Count) -SIMDLIB_FORCE_INLINE constexpr Vector register_from_array(const std::array &lanes) noexcept +constexpr Vector SIMD_FLAGS(Neither, ForceInline) register_from_array(const std::array &lanes) noexcept { Vector result{}; for (std::size_t index = 0; index < Count; ++index) { - register_set(result, index, lanes[index]); + register_set_constexpr(result, index, lanes[index]); } return result; } template requires(sizeof(Vector) == sizeof(Element) * sizeof...(Args)) && (std::convertible_to && ...) -SIMDLIB_FORCE_INLINE constexpr Vector register_from_values(Args &&...values) noexcept +constexpr Vector SIMD_FLAGS(Neither, ForceInline) register_from_values(Args &&...values) noexcept { return register_from_array(std::array{static_cast(values)...}); } -template SIMDLIB_FORCE_INLINE constexpr auto register_to_array(const Vector value) noexcept +/** @brief Constructs a constant-evaluated native register with every lane set to one value. + * @tparam Vector Native register representation. + * @tparam Element Scalar lane type. + * @param value Value copied into every lane. + * @return Native register containing the repeated value. + */ +template + requires(sizeof(Vector) % sizeof(Element) == 0) +constexpr Vector SIMD_FLAGS(Neither, ForceInline) register_from_repeated_value(const Element value) noexcept +{ + std::array lanes{}; + lanes.fill(value); + return register_from_array(lanes); +} + +template constexpr auto SIMD_FLAGS(Neither, ForceInline) register_to_array(const Vector value) noexcept { std::array result{}; for (std::size_t index = 0; index < result.size(); ++index) { - result[index] = register_get(value, index); + result[index] = register_get_constexpr(value, index); } return result; } -template SIMDLIB_FORCE_INLINE Element *register_data(Vector &value) noexcept +template auto SIMD_FLAGS(Neither, ForceInline) register_data(Vector &value) noexcept -> Element * { return reinterpret_cast(&value); } -template SIMDLIB_FORCE_INLINE const Element *register_data(const Vector &value) noexcept +template auto SIMD_FLAGS(Neither, ForceInline) register_data(const Vector &value) noexcept -> const Element * { return reinterpret_cast(&value); } +/** + * @brief Returns a register with one lane replaced through the portable constant-evaluation representation. + * @tparam Element Scalar lane type. + * @tparam Vector Compiler-native register type. + * @tparam Value Replacement value type. + * @param value Source register represented during constant evaluation. + * @param lane Replacement lane value. + * @param index Selected lane index. + * @return Register with the selected lane replaced. + * @note This remains `constexpr`, rather than `consteval`, because C++20 + * runtime-callable `constexpr` wrappers pass their parameters through it. + */ template -SIMDLIB_FORCE_INLINE constexpr Vector register_insert(Vector value, const Value lane, const std::size_t index) noexcept +constexpr Vector SIMD_FLAGS(Neither, ForceInline) register_insert_constexpr(Vector value, const Value lane, const std::size_t index) noexcept { - register_set(value, index, static_cast(lane)); + register_set_constexpr(value, index, static_cast(lane)); return value; } -template SIMDLIB_FORCE_INLINE constexpr Vector register_blend(Vector lhs, const Vector rhs, const unsigned int mask) noexcept +/** @brief Emulates an immediate-controlled lane blend with a runtime scalar mask. + * @tparam Element Logical lane type. + * @tparam Vector Native register type. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param mask Runtime control byte. + * @return Register containing the selected lanes. + */ +template +constexpr Vector SIMD_FLAGS(Neither, ForceInline) register_blend_slow(Vector lhs, const Vector rhs, const unsigned int mask) noexcept { constexpr std::size_t count = sizeof(Vector) / sizeof(Element); for (std::size_t index = 0; index < count; ++index) { if ((mask & (1u << (index % 8))) != 0) - register_set(lhs, index, register_get(rhs, index)); + register_set_constexpr(lhs, index, register_get_constexpr(rhs, index)); } return lhs; } -template SIMDLIB_FORCE_INLINE constexpr Vector register_blend_bytes(Vector lhs, const Vector rhs, const Vector mask) noexcept -{ - constexpr std::size_t count = sizeof(Vector); - for (std::size_t index = 0; index < count; ++index) - { - if ((register_get(mask, index) & 0x80u) != 0) - register_set(lhs, index, register_get(rhs, index)); - } - return lhs; -} - -template SIMDLIB_FORCE_INLINE constexpr Vector register_insert_float(Vector lhs, const Vector rhs, const unsigned int control) noexcept +template +constexpr Vector SIMD_FLAGS(InOut, RegisterOnly, ForceInline) register_blend_bytes(Vector lhs, const Vector rhs, const Vector mask) noexcept { - auto lanes = register_to_array(lhs); - const auto source = register_to_array(rhs); - lanes[(control >> 4) & 0x3u] = source[(control >> 6) & 0x3u]; - for (std::size_t index = 0; index < lanes.size(); ++index) + if (std::is_constant_evaluated()) { - if ((control & (1u << index)) != 0) - lanes[index] = 0.0f; + constexpr std::size_t count = sizeof(Vector); + for (std::size_t index = 0; index < count; ++index) + { + if ((register_get_constexpr(mask, index) & 0x80u) != 0) + register_set_constexpr(lhs, index, register_get_constexpr(rhs, index)); + } + return lhs; } - return register_from_array(lanes); + if constexpr (sizeof(Vector) == 16) + return _mm_blendv_epi8(lhs, rhs, mask); + else + return _mm256_blendv_epi8(lhs, rhs, mask); } -template SIMDLIB_FORCE_INLINE constexpr Vector register_shuffle_float(const Vector lhs, const Vector rhs, const unsigned int control) noexcept +/** @brief Emulates a floating shuffle with a runtime control byte. + * @tparam Vector Native float register type. + * @param lhs Source for the lower selected lanes in each four-lane group. + * @param rhs Source for the upper selected lanes in each four-lane group. + * @param control Runtime control byte. + * @return Register containing the shuffled lanes. + */ +template +constexpr Vector SIMD_FLAGS(Neither, ForceInline) register_shuffle_float_slow(const Vector lhs, const Vector rhs, const unsigned int control) noexcept { const auto left = register_to_array(lhs); const auto right = register_to_array(rhs); @@ -230,7 +354,15 @@ template SIMDLIB_FORCE_INLINE constexpr Vector register_shuffle_f return register_from_array(result); } -template SIMDLIB_FORCE_INLINE constexpr Vector register_shuffle_double(const Vector lhs, const Vector rhs, const unsigned int control) noexcept +/** @brief Emulates a double shuffle with a runtime control byte. + * @tparam Vector Native double register type. + * @param lhs Source for the first selected lane in each pair. + * @param rhs Source for the second selected lane in each pair. + * @param control Runtime control byte. + * @return Register containing the shuffled lanes. + */ +template +constexpr Vector SIMD_FLAGS(Neither, ForceInline) register_shuffle_double_slow(const Vector lhs, const Vector rhs, const unsigned int control) noexcept { const auto left = register_to_array(lhs); const auto right = register_to_array(rhs); @@ -244,72 +376,655 @@ template SIMDLIB_FORCE_INLINE constexpr Vector register_shuffle_d return register_from_array(result); } -template SIMDLIB_FORCE_INLINE constexpr Vector register_shuffle_32(const Vector value, const unsigned int control) noexcept +/** @brief Emulates a 32-bit shuffle with a runtime control byte. + * @tparam Vector Native integer register type. + * @param value Source register. + * @param control Runtime control byte. + * @return Register with each four-lane group shuffled. + */ +template +constexpr Vector SIMD_FLAGS(InOut, RegisterOnly, ForceInline) register_shuffle_32_slow(const Vector value, const unsigned int control) noexcept { - const auto source = register_to_array(value); - std::array result{}; - for (std::size_t lane = 0; lane < result.size(); lane += 4) + if (std::is_constant_evaluated()) { - for (std::size_t index = 0; index < 4; ++index) - result[lane + index] = source[lane + ((control >> (index * 2)) & 0x3u)]; + const auto source = register_to_array(value); + std::array result{}; + for (std::size_t lane = 0; lane < result.size(); lane += 4) + { + for (std::size_t index = 0; index < 4; ++index) + result[lane + index] = source[lane + ((control >> (index * 2)) & 0x3u)]; + } + return register_from_array(result); } - return register_from_array(result); -} -template -SIMDLIB_FORCE_INLINE constexpr Vector register_shuffle_half_16(const Vector value, const unsigned int control, const bool high_half) noexcept -{ - const auto source = register_to_array(value); - auto result = source; - for (std::size_t lane = 0; lane < result.size(); lane += 8) + const int index0 = static_cast(control & 0x3u); + const int index1 = static_cast((control >> 2) & 0x3u); + const int index2 = static_cast((control >> 4) & 0x3u); + const int index3 = static_cast((control >> 6) & 0x3u); + if constexpr (sizeof(Vector) == 16) { - const std::size_t base = lane + (high_half ? 4 : 0); - for (std::size_t index = 0; index < 4; ++index) - result[base + index] = source[base + ((control >> (index * 2)) & 0x3u)]; + const __m128i dword_indices = _mm_set_epi32(index3 * 4, index2 * 4, index1 * 4, index0 * 4); + const __m128i byte_indices = + _mm_add_epi8(_mm_shuffle_epi8(dword_indices, _mm_set_epi32(0x0C0C0C0C, 0x08080808, 0x04040404, 0x00000000)), _mm_set1_epi32(0x03020100)); + return _mm_shuffle_epi8(value, byte_indices); + } + else + { + const __m256i dword_indices = _mm256_set_epi32(4 + index3, 4 + index2, 4 + index1, 4 + index0, index3, index2, index1, index0); + return _mm256_permutevar8x32_epi32(value, dword_indices); } - return register_from_array(result); } -template SIMDLIB_FORCE_INLINE constexpr Vector register_byte_shift_left(const Vector value, const int count) noexcept +/** @brief Emulates a low- or high-half 16-bit shuffle with a runtime control byte. + * @tparam Vector Native integer register type. + * @param value Source register. + * @param control Runtime control byte. + * @param high_half Whether to shuffle the high half instead of the low half. + * @return Register containing the shuffled half groups. + */ +template +constexpr Vector SIMD_FLAGS(InOut, RegisterOnly, ForceInline) + register_shuffle_half_16_slow(const Vector value, const unsigned int control, const bool high_half) noexcept { - if (count <= 0) - return value; - constexpr std::size_t size = sizeof(Vector); - if (static_cast(count) >= size) - return register_from_array(std::array{}); - const auto source = register_to_array(value); - std::array result{}; - for (std::size_t index = static_cast(count); index < size; ++index) - result[index] = source[index - static_cast(count)]; - return register_from_array(result); -} + if (std::is_constant_evaluated()) + { + const auto source = register_to_array(value); + auto result = source; + for (std::size_t lane = 0; lane < result.size(); lane += 8) + { + const std::size_t base = lane + (high_half ? 4 : 0); + for (std::size_t index = 0; index < 4; ++index) + result[base + index] = source[base + ((control >> (index * 2)) & 0x3u)]; + } + return register_from_array(result); + } -template SIMDLIB_FORCE_INLINE constexpr Vector register_byte_shift_right(const Vector value, const int count) noexcept -{ - if (count <= 0) - return value; - constexpr std::size_t size = sizeof(Vector); - if (static_cast(count) >= size) - return register_from_array(std::array{}); - const auto source = register_to_array(value); - std::array result{}; - for (std::size_t index = 0; index + static_cast(count) < size; ++index) - result[index] = source[index + static_cast(count)]; - return register_from_array(result); + const int index0 = static_cast(control & 0x3u); + const int index1 = static_cast((control >> 2) & 0x3u); + const int index2 = static_cast((control >> 4) & 0x3u); + const int index3 = static_cast((control >> 6) & 0x3u); + const int word0 = high_half ? 0 : index0; + const int word1 = high_half ? 1 : index1; + const int word2 = high_half ? 2 : index2; + const int word3 = high_half ? 3 : index3; + const int word4 = high_half ? 4 + index0 : 4; + const int word5 = high_half ? 4 + index1 : 5; + const int word6 = high_half ? 4 + index2 : 6; + const int word7 = high_half ? 4 + index3 : 7; + const int pair0 = (word0 * 2) | ((word0 * 2 + 1) << 8); + const int pair1 = (word1 * 2) | ((word1 * 2 + 1) << 8); + const int pair2 = (word2 * 2) | ((word2 * 2 + 1) << 8); + const int pair3 = (word3 * 2) | ((word3 * 2 + 1) << 8); + const int pair4 = (word4 * 2) | ((word4 * 2 + 1) << 8); + const int pair5 = (word5 * 2) | ((word5 * 2 + 1) << 8); + const int pair6 = (word6 * 2) | ((word6 * 2 + 1) << 8); + const int pair7 = (word7 * 2) | ((word7 * 2 + 1) << 8); + const __m128i byte_indices = _mm_set_epi32((pair7 << 16) | pair6, (pair5 << 16) | pair4, (pair3 << 16) | pair2, (pair1 << 16) | pair0); + if constexpr (sizeof(Vector) == 16) + return _mm_shuffle_epi8(value, byte_indices); + else + return _mm256_shuffle_epi8(value, _mm256_broadcastsi128_si256(byte_indices)); } template -SIMDLIB_FORCE_INLINE constexpr Vector register_transform_binary(const Vector lhs, const Vector rhs, Operation &&operation) noexcept +constexpr Vector SIMD_FLAGS(Neither, ForceInline) register_transform_binary(const Vector lhs, const Vector rhs, Operation &&operation) noexcept { constexpr std::size_t count = sizeof(Vector) / sizeof(Element); std::array result{}; for (std::size_t index = 0; index < count; ++index) - result[index] = static_cast(operation(register_get(lhs, index), register_get(rhs, index))); + result[index] = static_cast(operation(register_get_constexpr(lhs, index), register_get_constexpr(rhs, index))); return register_from_array(result); } #if SIMDLIB_HAS_SSE42 +#pragma region 128bit Complete-Register Byte Shift Extensions + +/** + * @brief Clamps a runtime byte-shift count to the complete 128-bit register. + * @param count Runtime byte count. + * @return A count in the inclusive range zero through sixteen. + */ +constexpr int SIMD_FLAGS(Neither, RegisterOnly, ForceInline) _ext128_clamp_shift_bytes_count(const int count) noexcept +{ + const int nonnegative = count < 0 ? 0 : count; + return nonnegative > 16 ? 16 : nonnegative; +} + +/** + * @brief Broadcasts a clamped byte-shift count into every byte lane. + * @param count Byte count in the inclusive range zero through sixteen. + * @return Register containing the count in every byte lane. + */ +__m128i SIMD_FLAGS(Out, RegisterOnly, ForceInline) _ext128_broadcast_shift_bytes_count(const int count) noexcept +{ + return _mm_set1_epi32(count * 0x01010101); +} + +/** + * @brief Shifts a complete 128-bit register toward higher byte indices. + * + * `PSLLDQ` accepts only an immediate count. This runtime path instead builds + * a variable `PSHUFB` control vector without materializing the register in + * addressable storage. + * + * @param lhs Source register. + * @param count Runtime byte count; nonpositive values are identity and values + * greater than or equal to sixteen produce zero. + * @return Shifted register with zero-filled low bytes. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_shift_bytes_left_slow(__m128i lhs, const int count) noexcept +{ + const __m128i indices = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const int boundedCount = _ext128_clamp_shift_bytes_count(count); + const __m128i counts = _ext128_broadcast_shift_bytes_count(boundedCount); + return _mm_shuffle_epi8(lhs, _mm_sub_epi8(indices, counts)); +} + +/** + * @brief Shifts a complete 128-bit register toward lower byte indices. + * + * `PSRLDQ` accepts only an immediate count. This runtime path instead builds + * a variable `PSHUFB` control vector without materializing the register in + * addressable storage. + * + * @param lhs Source register. + * @param count Runtime byte count; nonpositive values are identity and values + * greater than or equal to sixteen produce zero. + * @return Shifted register with zero-filled high bytes. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_shift_bytes_right_slow(__m128i lhs, const int count) noexcept +{ + const __m128i biasedIndices = _mm_setr_epi8(0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F); + const int boundedCount = _ext128_clamp_shift_bytes_count(count); + const __m128i counts = _ext128_broadcast_shift_bytes_count(boundedCount); + return _mm_shuffle_epi8(lhs, _mm_add_epi8(biasedIndices, counts)); +} + +#pragma endregion + +#pragma region 128bit Integer Division Extensions + +/** + * @brief Divides 16 signed 8-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The truncating integer quotient for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_div_epi8(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 0)) / static_cast(_mm_extract_epi8(rhs, 0)))), 0); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 1)) / static_cast(_mm_extract_epi8(rhs, 1)))), 1); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 2)) / static_cast(_mm_extract_epi8(rhs, 2)))), 2); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 3)) / static_cast(_mm_extract_epi8(rhs, 3)))), 3); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 4)) / static_cast(_mm_extract_epi8(rhs, 4)))), 4); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 5)) / static_cast(_mm_extract_epi8(rhs, 5)))), 5); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 6)) / static_cast(_mm_extract_epi8(rhs, 6)))), 6); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 7)) / static_cast(_mm_extract_epi8(rhs, 7)))), 7); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 8)) / static_cast(_mm_extract_epi8(rhs, 8)))), 8); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 9)) / static_cast(_mm_extract_epi8(rhs, 9)))), 9); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 10)) / static_cast(_mm_extract_epi8(rhs, 10)))), + 10); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 11)) / static_cast(_mm_extract_epi8(rhs, 11)))), + 11); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 12)) / static_cast(_mm_extract_epi8(rhs, 12)))), + 12); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 13)) / static_cast(_mm_extract_epi8(rhs, 13)))), + 13); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 14)) / static_cast(_mm_extract_epi8(rhs, 14)))), + 14); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 15)) / static_cast(_mm_extract_epi8(rhs, 15)))), + 15); + return result; +} + +/** + * @brief Divides 16 unsigned 8-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The truncating integer quotient for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_div_epu8(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 0)) / static_cast(_mm_extract_epi8(rhs, 0)))), + 0); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 1)) / static_cast(_mm_extract_epi8(rhs, 1)))), + 1); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 2)) / static_cast(_mm_extract_epi8(rhs, 2)))), + 2); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 3)) / static_cast(_mm_extract_epi8(rhs, 3)))), + 3); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 4)) / static_cast(_mm_extract_epi8(rhs, 4)))), + 4); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 5)) / static_cast(_mm_extract_epi8(rhs, 5)))), + 5); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 6)) / static_cast(_mm_extract_epi8(rhs, 6)))), + 6); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 7)) / static_cast(_mm_extract_epi8(rhs, 7)))), + 7); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 8)) / static_cast(_mm_extract_epi8(rhs, 8)))), + 8); + result = _mm_insert_epi8( + result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 9)) / static_cast(_mm_extract_epi8(rhs, 9)))), + 9); + result = _mm_insert_epi8(result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 10)) / + static_cast(_mm_extract_epi8(rhs, 10)))), + 10); + result = _mm_insert_epi8(result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 11)) / + static_cast(_mm_extract_epi8(rhs, 11)))), + 11); + result = _mm_insert_epi8(result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 12)) / + static_cast(_mm_extract_epi8(rhs, 12)))), + 12); + result = _mm_insert_epi8(result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 13)) / + static_cast(_mm_extract_epi8(rhs, 13)))), + 13); + result = _mm_insert_epi8(result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 14)) / + static_cast(_mm_extract_epi8(rhs, 14)))), + 14); + result = _mm_insert_epi8(result, + static_cast(static_cast(static_cast(_mm_extract_epi8(lhs, 15)) / + static_cast(_mm_extract_epi8(rhs, 15)))), + 15); + return result; +} + +/** + * @brief Divides 8 signed 16-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The truncating integer quotient for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_div_epi16(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 0)) / + static_cast(_mm_extract_epi16(rhs, 0)))), + 0); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 1)) / + static_cast(_mm_extract_epi16(rhs, 1)))), + 1); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 2)) / + static_cast(_mm_extract_epi16(rhs, 2)))), + 2); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 3)) / + static_cast(_mm_extract_epi16(rhs, 3)))), + 3); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 4)) / + static_cast(_mm_extract_epi16(rhs, 4)))), + 4); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 5)) / + static_cast(_mm_extract_epi16(rhs, 5)))), + 5); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 6)) / + static_cast(_mm_extract_epi16(rhs, 6)))), + 6); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 7)) / + static_cast(_mm_extract_epi16(rhs, 7)))), + 7); + return result; +} + +/** + * @brief Divides 8 unsigned 16-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The truncating integer quotient for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_div_epu16(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 0)) / + static_cast(_mm_extract_epi16(rhs, 0)))), + 0); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 1)) / + static_cast(_mm_extract_epi16(rhs, 1)))), + 1); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 2)) / + static_cast(_mm_extract_epi16(rhs, 2)))), + 2); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 3)) / + static_cast(_mm_extract_epi16(rhs, 3)))), + 3); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 4)) / + static_cast(_mm_extract_epi16(rhs, 4)))), + 4); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 5)) / + static_cast(_mm_extract_epi16(rhs, 5)))), + 5); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 6)) / + static_cast(_mm_extract_epi16(rhs, 6)))), + 6); + result = _mm_insert_epi16(result, + static_cast(static_cast(static_cast(_mm_extract_epi16(lhs, 7)) / + static_cast(_mm_extract_epi16(rhs, 7)))), + 7); + return result; +} + +/** + * @brief Divides 4 signed 32-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The truncating integer quotient for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_div_epi32(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi32(result, static_cast(_mm_extract_epi32(lhs, 0)) / static_cast(_mm_extract_epi32(rhs, 0)), 0); + result = _mm_insert_epi32(result, static_cast(_mm_extract_epi32(lhs, 1)) / static_cast(_mm_extract_epi32(rhs, 1)), 1); + result = _mm_insert_epi32(result, static_cast(_mm_extract_epi32(lhs, 2)) / static_cast(_mm_extract_epi32(rhs, 2)), 2); + result = _mm_insert_epi32(result, static_cast(_mm_extract_epi32(lhs, 3)) / static_cast(_mm_extract_epi32(rhs, 3)), 3); + return result; +} + +/** + * @brief Divides 4 unsigned 32-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The truncating integer quotient for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_div_epu32(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi32( + result, std::bit_cast(static_cast(_mm_extract_epi32(lhs, 0)) / static_cast(_mm_extract_epi32(rhs, 0))), 0); + result = _mm_insert_epi32( + result, std::bit_cast(static_cast(_mm_extract_epi32(lhs, 1)) / static_cast(_mm_extract_epi32(rhs, 1))), 1); + result = _mm_insert_epi32( + result, std::bit_cast(static_cast(_mm_extract_epi32(lhs, 2)) / static_cast(_mm_extract_epi32(rhs, 2))), 2); + result = _mm_insert_epi32( + result, std::bit_cast(static_cast(_mm_extract_epi32(lhs, 3)) / static_cast(_mm_extract_epi32(rhs, 3))), 3); + return result; +} + +/** + * @brief Divides 2 signed 64-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The truncating integer quotient for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_div_epi64(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi64(result, static_cast(_mm_extract_epi64(lhs, 0)) / static_cast(_mm_extract_epi64(rhs, 0)), 0); + result = _mm_insert_epi64(result, static_cast(_mm_extract_epi64(lhs, 1)) / static_cast(_mm_extract_epi64(rhs, 1)), 1); + return result; +} + +/** + * @brief Divides 2 unsigned 64-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The truncating integer quotient for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_div_epu64(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi64( + result, std::bit_cast(static_cast(_mm_extract_epi64(lhs, 0)) / static_cast(_mm_extract_epi64(rhs, 0))), 0); + result = _mm_insert_epi64( + result, std::bit_cast(static_cast(_mm_extract_epi64(lhs, 1)) / static_cast(_mm_extract_epi64(rhs, 1))), 1); + return result; +} + +#pragma endregion + +#pragma region 128bit Integer Remainder Extensions + +/** + * @brief Computes remainders for 16 signed 8-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The scalar signed remainder for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_rem_epi8(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 0)) % static_cast(_mm_extract_epi8(rhs, 0)), 0); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 1)) % static_cast(_mm_extract_epi8(rhs, 1)), 1); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 2)) % static_cast(_mm_extract_epi8(rhs, 2)), 2); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 3)) % static_cast(_mm_extract_epi8(rhs, 3)), 3); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 4)) % static_cast(_mm_extract_epi8(rhs, 4)), 4); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 5)) % static_cast(_mm_extract_epi8(rhs, 5)), 5); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 6)) % static_cast(_mm_extract_epi8(rhs, 6)), 6); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 7)) % static_cast(_mm_extract_epi8(rhs, 7)), 7); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 8)) % static_cast(_mm_extract_epi8(rhs, 8)), 8); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 9)) % static_cast(_mm_extract_epi8(rhs, 9)), 9); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 10)) % static_cast(_mm_extract_epi8(rhs, 10)), 10); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 11)) % static_cast(_mm_extract_epi8(rhs, 11)), 11); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 12)) % static_cast(_mm_extract_epi8(rhs, 12)), 12); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 13)) % static_cast(_mm_extract_epi8(rhs, 13)), 13); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 14)) % static_cast(_mm_extract_epi8(rhs, 14)), 14); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 15)) % static_cast(_mm_extract_epi8(rhs, 15)), 15); + return result; +} + +/** + * @brief Computes remainders for 16 unsigned 8-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The scalar unsigned remainder for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_rem_epu8(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 0)) % static_cast(_mm_extract_epi8(rhs, 0)), 0); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 1)) % static_cast(_mm_extract_epi8(rhs, 1)), 1); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 2)) % static_cast(_mm_extract_epi8(rhs, 2)), 2); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 3)) % static_cast(_mm_extract_epi8(rhs, 3)), 3); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 4)) % static_cast(_mm_extract_epi8(rhs, 4)), 4); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 5)) % static_cast(_mm_extract_epi8(rhs, 5)), 5); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 6)) % static_cast(_mm_extract_epi8(rhs, 6)), 6); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 7)) % static_cast(_mm_extract_epi8(rhs, 7)), 7); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 8)) % static_cast(_mm_extract_epi8(rhs, 8)), 8); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 9)) % static_cast(_mm_extract_epi8(rhs, 9)), 9); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 10)) % static_cast(_mm_extract_epi8(rhs, 10)), 10); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 11)) % static_cast(_mm_extract_epi8(rhs, 11)), 11); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 12)) % static_cast(_mm_extract_epi8(rhs, 12)), 12); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 13)) % static_cast(_mm_extract_epi8(rhs, 13)), 13); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 14)) % static_cast(_mm_extract_epi8(rhs, 14)), 14); + result = _mm_insert_epi8(result, static_cast(_mm_extract_epi8(lhs, 15)) % static_cast(_mm_extract_epi8(rhs, 15)), 15); + return result; +} + +/** + * @brief Computes remainders for 8 signed 16-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The scalar signed remainder for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_rem_epi16(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 0)) % static_cast(_mm_extract_epi16(rhs, 0)), 0); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 1)) % static_cast(_mm_extract_epi16(rhs, 1)), 1); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 2)) % static_cast(_mm_extract_epi16(rhs, 2)), 2); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 3)) % static_cast(_mm_extract_epi16(rhs, 3)), 3); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 4)) % static_cast(_mm_extract_epi16(rhs, 4)), 4); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 5)) % static_cast(_mm_extract_epi16(rhs, 5)), 5); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 6)) % static_cast(_mm_extract_epi16(rhs, 6)), 6); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 7)) % static_cast(_mm_extract_epi16(rhs, 7)), 7); + return result; +} + +/** + * @brief Computes remainders for 8 unsigned 16-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The scalar unsigned remainder for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_rem_epu16(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 0)) % static_cast(_mm_extract_epi16(rhs, 0)), 0); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 1)) % static_cast(_mm_extract_epi16(rhs, 1)), 1); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 2)) % static_cast(_mm_extract_epi16(rhs, 2)), 2); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 3)) % static_cast(_mm_extract_epi16(rhs, 3)), 3); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 4)) % static_cast(_mm_extract_epi16(rhs, 4)), 4); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 5)) % static_cast(_mm_extract_epi16(rhs, 5)), 5); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 6)) % static_cast(_mm_extract_epi16(rhs, 6)), 6); + result = _mm_insert_epi16(result, static_cast(_mm_extract_epi16(lhs, 7)) % static_cast(_mm_extract_epi16(rhs, 7)), 7); + return result; +} + +/** + * @brief Computes remainders for 4 signed 32-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The scalar signed remainder for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_rem_epi32(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi32(result, static_cast(_mm_extract_epi32(lhs, 0)) % static_cast(_mm_extract_epi32(rhs, 0)), 0); + result = _mm_insert_epi32(result, static_cast(_mm_extract_epi32(lhs, 1)) % static_cast(_mm_extract_epi32(rhs, 1)), 1); + result = _mm_insert_epi32(result, static_cast(_mm_extract_epi32(lhs, 2)) % static_cast(_mm_extract_epi32(rhs, 2)), 2); + result = _mm_insert_epi32(result, static_cast(_mm_extract_epi32(lhs, 3)) % static_cast(_mm_extract_epi32(rhs, 3)), 3); + return result; +} + +/** + * @brief Computes remainders for 4 unsigned 32-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The scalar unsigned remainder for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_rem_epu32(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi32( + result, std::bit_cast(static_cast(_mm_extract_epi32(lhs, 0)) % static_cast(_mm_extract_epi32(rhs, 0))), 0); + result = _mm_insert_epi32( + result, std::bit_cast(static_cast(_mm_extract_epi32(lhs, 1)) % static_cast(_mm_extract_epi32(rhs, 1))), 1); + result = _mm_insert_epi32( + result, std::bit_cast(static_cast(_mm_extract_epi32(lhs, 2)) % static_cast(_mm_extract_epi32(rhs, 2))), 2); + result = _mm_insert_epi32( + result, std::bit_cast(static_cast(_mm_extract_epi32(lhs, 3)) % static_cast(_mm_extract_epi32(rhs, 3))), 3); + return result; +} + +/** + * @brief Computes remainders for 2 signed 64-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The scalar signed remainder for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_rem_epi64(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi64(result, static_cast(_mm_extract_epi64(lhs, 0)) % static_cast(_mm_extract_epi64(rhs, 0)), 0); + result = _mm_insert_epi64(result, static_cast(_mm_extract_epi64(lhs, 1)) % static_cast(_mm_extract_epi64(rhs, 1)), 1); + return result; +} + +/** + * @brief Computes remainders for 2 unsigned 64-bit lanes using constant-index intrinsic extraction and insertion. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The scalar unsigned remainder for every lane. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext128_rem_epu64(__m128i lhs, __m128i rhs) noexcept +{ + __m128i result = _mm_setzero_si128(); + result = _mm_insert_epi64( + result, std::bit_cast(static_cast(_mm_extract_epi64(lhs, 0)) % static_cast(_mm_extract_epi64(rhs, 0))), 0); + result = _mm_insert_epi64( + result, std::bit_cast(static_cast(_mm_extract_epi64(lhs, 1)) % static_cast(_mm_extract_epi64(rhs, 1))), 1); + return result; +} + +#pragma endregion + #pragma region 128bit int8_t Extensions /** @@ -319,7 +1034,7 @@ SIMDLIB_FORCE_INLINE constexpr Vector register_transform_binary(const Vector lhs * @param rhs The second byte-lane register. * @return The low byte of each lane product. */ -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_mul_epi8(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_mul_epi8(__m128i lhs, __m128i rhs) noexcept { // unpack and multiply const __m128i dst_even = _mm_mullo_epi16(lhs, rhs); @@ -329,13 +1044,13 @@ SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_mul_epi8(__m128i lhs, __m128i rhs) return _mm_blendv_epi8(dst_odd, dst_even, mask); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_slli_epx8(__m128i lhs, const int count) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_slli_epx8(__m128i lhs, const int count) noexcept { const __m128i mask = _mm_set1_epi8(0xFF << count); return _mm_and_si128(_mm_slli_epi16(lhs, count), mask); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_srli_epx8(__m128i lhs, const int count) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_srli_epx8(__m128i lhs, const int count) noexcept { const __m128i mask = _mm_set1_epi8(0xFF >> count); return _mm_and_si128(_mm_srli_epi16(lhs, count), mask); @@ -348,7 +1063,7 @@ SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_srli_epx8(__m128i lhs, const int co * @param count The per-lane shift count. * @return The arithmetic-right-shifted byte lanes. */ -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_srai_epx8(__m128i lhs, const int count) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_srai_epx8(__m128i lhs, const int count) noexcept { __m128i aeven = _mm_slli_epi16(lhs, 8); // even numbered elements get sign bit in position aeven = _mm_sra_epi16(aeven, _mm_cvtsi32_si128(count + 8)); // shift arithmetic, back to position @@ -362,24 +1077,24 @@ SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_srai_epx8(__m128i lhs, const int co #pragma region 128bit uint8_t Extensions -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_mul_epu8(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_mul_epu8(__m128i lhs, __m128i rhs) noexcept { return _ext_mul_epi8(lhs, rhs); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_cmpgt_epu8(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_cmpgt_epu8(__m128i lhs, __m128i rhs) noexcept { // Returns 0xFF where x > y: return _mm_andnot_si128(_mm_cmpeq_epi8(lhs, rhs), _mm_cmpeq_epi8(_mm_max_epu8(lhs, rhs), lhs)); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_cmplt_epu8(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_cmplt_epu8(__m128i lhs, __m128i rhs) noexcept { // Returns 0xFF where x < y: return _ext_cmpgt_epu8(rhs, lhs); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_set1_epu8(const std::uint8_t value) noexcept +__m128i SIMD_FLAGS(Out, ForceInline) _ext_set1_epu8(const std::uint8_t value) noexcept { return _mm_set1_epi8(std::bit_cast(value)); } @@ -388,32 +1103,32 @@ SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_set1_epu8(const std::uint8_t value) #pragma region 128bit uint16_t Extensions -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_cmple_epu16(__m128i x, __m128i y) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_cmple_epu16(__m128i x, __m128i y) noexcept { // Returns 0xFFFF where x <= y: return _mm_cmpeq_epi16(_mm_subs_epu16(x, y), _mm_setzero_si128()); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_cmpgt_epu16(__m128i x, __m128i y) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_cmpgt_epu16(__m128i x, __m128i y) noexcept { // Returns 0xFFFF where x > y: return _mm_andnot_si128(_mm_cmpeq_epi16(x, y), _ext_cmple_epu16(y, x)); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_cmplt_epu16(__m128i x, __m128i y) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_cmplt_epu16(__m128i x, __m128i y) noexcept { // Returns 0xFFFF where x < y: return _ext_cmpgt_epu16(y, x); } // Return x where x <= y, else y. -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_min_epu16(__m128i x, __m128i y) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_min_epu16(__m128i x, __m128i y) noexcept { return _mm_sub_epi16(x, _mm_subs_epu16(x, y)); } // Return x where x >= y, else y. -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_max_epu16(__m128i x, __m128i y) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_max_epu16(__m128i x, __m128i y) noexcept { return _mm_add_epi16(x, _mm_subs_epu16(y, x)); } @@ -421,16 +1136,11 @@ SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_max_epu16(__m128i x, __m128i y) noe #pragma region 128bit int32_t Extensions -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_div_epi32(__m128i lhs, __m128i rhs) noexcept -{ - return _mm_cvttps_epi32(_mm_div_ps(_mm_cvtepi32_ps(lhs), _mm_cvtepi32_ps(rhs))); -} - #pragma endregion #pragma region 128bit uint32_t Extensions -SIMDLIB_FORCE_INLINE __m128 VECTORCALL _ext_cvtepu32_ps(__m128i lhs) noexcept +__m128 SIMD_FLAGS(InOut, ForceInline) _ext_cvtepu32_ps(__m128i lhs) noexcept { const __m128 signedFloats = _mm_cvtepi32_ps(lhs); const __m128i highBitMask = _mm_cmpgt_epi32(_mm_setzero_si128(), lhs); @@ -438,12 +1148,7 @@ SIMDLIB_FORCE_INLINE __m128 VECTORCALL _ext_cvtepu32_ps(__m128i lhs) noexcept return _mm_add_ps(signedFloats, correction); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_div_epu32(__m128i lhs, __m128i rhs) noexcept -{ - return _mm_cvttps_epi32(_mm_div_ps(_ext_cvtepu32_ps(lhs), _ext_cvtepu32_ps(rhs))); -} - -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_cmpgt_epu32(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_cmpgt_epu32(__m128i lhs, __m128i rhs) noexcept { // Returns 0xFFFFFFFF where x > y: return _mm_andnot_si128(_mm_cmpeq_epi32(lhs, rhs), _mm_cmpeq_epi32(_mm_max_epu32(lhs, rhs), lhs)); @@ -455,9 +1160,281 @@ SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_cmpgt_epu32(__m128i lhs, __m128i rh #if SIMDLIB_HAS_AVX2 && SIMDLIB_HAS_SSE42 +#pragma region 256bit Integer Division Extensions + +/** + * @brief Divides 32 signed 8-bit lanes through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The truncating integer quotient for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_div_epi8(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i lhsLow = _mm256_castsi256_si128(lhs); + const __m128i rhsLow = _mm256_castsi256_si128(rhs); + const __m128i lhsHigh = _mm256_extracti128_si256(lhs, 1); + const __m128i rhsHigh = _mm256_extracti128_si256(rhs, 1); + + const __m128i resultLow = _ext128_div_epi8(lhsLow, rhsLow); + const __m128i resultHigh = _ext128_div_epi8(lhsHigh, rhsHigh); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Divides 32 unsigned 8-bit lanes through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The truncating integer quotient for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_div_epu8(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i lhsLow = _mm256_castsi256_si128(lhs); + const __m128i rhsLow = _mm256_castsi256_si128(rhs); + const __m128i lhsHigh = _mm256_extracti128_si256(lhs, 1); + const __m128i rhsHigh = _mm256_extracti128_si256(rhs, 1); + + const __m128i resultLow = _ext128_div_epu8(lhsLow, rhsLow); + const __m128i resultHigh = _ext128_div_epu8(lhsHigh, rhsHigh); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Divides 16 signed 16-bit lanes through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The truncating integer quotient for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_div_epi16(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i lhsLow = _mm256_castsi256_si128(lhs); + const __m128i rhsLow = _mm256_castsi256_si128(rhs); + const __m128i lhsHigh = _mm256_extracti128_si256(lhs, 1); + const __m128i rhsHigh = _mm256_extracti128_si256(rhs, 1); + + const __m128i resultLow = _ext128_div_epi16(lhsLow, rhsLow); + const __m128i resultHigh = _ext128_div_epi16(lhsHigh, rhsHigh); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Divides 16 unsigned 16-bit lanes through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The truncating integer quotient for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_div_epu16(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i lhsLow = _mm256_castsi256_si128(lhs); + const __m128i rhsLow = _mm256_castsi256_si128(rhs); + const __m128i lhsHigh = _mm256_extracti128_si256(lhs, 1); + const __m128i rhsHigh = _mm256_extracti128_si256(rhs, 1); + + const __m128i resultLow = _ext128_div_epu16(lhsLow, rhsLow); + const __m128i resultHigh = _ext128_div_epu16(lhsHigh, rhsHigh); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Divides 8 signed 32-bit lanes through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The truncating integer quotient for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_div_epi32(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i lhsLow = _mm256_castsi256_si128(lhs); + const __m128i rhsLow = _mm256_castsi256_si128(rhs); + const __m128i lhsHigh = _mm256_extracti128_si256(lhs, 1); + const __m128i rhsHigh = _mm256_extracti128_si256(rhs, 1); + + const __m128i resultLow = _ext128_div_epi32(lhsLow, rhsLow); + const __m128i resultHigh = _ext128_div_epi32(lhsHigh, rhsHigh); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Divides 8 unsigned 32-bit lanes through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The truncating integer quotient for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_div_epu32(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i lhsLow = _mm256_castsi256_si128(lhs); + const __m128i rhsLow = _mm256_castsi256_si128(rhs); + const __m128i lhsHigh = _mm256_extracti128_si256(lhs, 1); + const __m128i rhsHigh = _mm256_extracti128_si256(rhs, 1); + + const __m128i resultLow = _ext128_div_epu32(lhsLow, rhsLow); + const __m128i resultHigh = _ext128_div_epu32(lhsHigh, rhsHigh); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Divides 4 signed 64-bit lanes through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The truncating integer quotient for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_div_epi64(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i lhsLow = _mm256_castsi256_si128(lhs); + const __m128i rhsLow = _mm256_castsi256_si128(rhs); + const __m128i lhsHigh = _mm256_extracti128_si256(lhs, 1); + const __m128i rhsHigh = _mm256_extracti128_si256(rhs, 1); + + const __m128i resultLow = _ext128_div_epi64(lhsLow, rhsLow); + const __m128i resultHigh = _ext128_div_epi64(lhsHigh, rhsHigh); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Divides 4 unsigned 64-bit lanes through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The truncating integer quotient for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_div_epu64(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i lhsLow = _mm256_castsi256_si128(lhs); + const __m128i rhsLow = _mm256_castsi256_si128(rhs); + const __m128i lhsHigh = _mm256_extracti128_si256(lhs, 1); + const __m128i rhsHigh = _mm256_extracti128_si256(rhs, 1); + + const __m128i resultLow = _ext128_div_epu64(lhsLow, rhsLow); + const __m128i resultHigh = _ext128_div_epu64(lhsHigh, rhsHigh); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +#pragma endregion + +#pragma region 256bit Integer Remainder Extensions + +/** + * @brief Computes signed 8-bit lane remainders through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The scalar-equivalent remainder for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_rem_epi8(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i resultLow = _ext128_rem_epi8(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); + const __m128i resultHigh = _ext128_rem_epi8(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Computes unsigned 8-bit lane remainders through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The scalar-equivalent remainder for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_rem_epu8(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i resultLow = _ext128_rem_epu8(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); + const __m128i resultHigh = _ext128_rem_epu8(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Computes signed 16-bit lane remainders through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The scalar-equivalent remainder for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_rem_epi16(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i resultLow = _ext128_rem_epi16(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); + const __m128i resultHigh = _ext128_rem_epi16(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Computes unsigned 16-bit lane remainders through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The scalar-equivalent remainder for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_rem_epu16(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i resultLow = _ext128_rem_epu16(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); + const __m128i resultHigh = _ext128_rem_epu16(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Computes signed 32-bit lane remainders through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The scalar-equivalent remainder for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_rem_epi32(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i resultLow = _ext128_rem_epi32(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); + const __m128i resultHigh = _ext128_rem_epi32(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Computes unsigned 32-bit lane remainders through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The scalar-equivalent remainder for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_rem_epu32(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i resultLow = _ext128_rem_epu32(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); + const __m128i resultHigh = _ext128_rem_epu32(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Computes signed 64-bit lane remainders through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero and no dividend-minimum lane is divided by negative one. + * @return The scalar-equivalent remainder for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_rem_epi64(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i resultLow = _ext128_rem_epi64(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); + const __m128i resultHigh = _ext128_rem_epi64(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +/** + * @brief Computes unsigned 64-bit lane remainders through the matching 128-bit extension. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @pre Every lane in rhs is nonzero. + * @return The scalar-equivalent remainder for every lane. + */ +__m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) _ext256_rem_epu64(__m256i lhs, __m256i rhs) noexcept +{ + const __m128i resultLow = _ext128_rem_epu64(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); + const __m128i resultHigh = _ext128_rem_epu64(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); + return _mm256_inserti128_si256(_mm256_zextsi128_si256(resultLow), resultHigh, 1); +} + +#pragma endregion + #pragma region 256bit uint32_t Extensions -SIMDLIB_FORCE_INLINE __m256 VECTORCALL _ext256_cvtepu32_ps(__m256i lhs) noexcept +__m256 SIMD_FLAGS(InOut, ForceInline) _ext256_cvtepu32_ps(__m256i lhs) noexcept { const __m256 signedFloats = _mm256_cvtepi32_ps(lhs); const __m256i highBitMask = _mm256_cmpgt_epi32(_mm256_setzero_si256(), lhs); @@ -473,12 +1450,12 @@ SIMDLIB_FORCE_INLINE __m256 VECTORCALL _ext256_cvtepu32_ps(__m256i lhs) noexcept #pragma region 128bit int64_t Extensions -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_cmpgt_epi64(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_cmpgt_epi64(__m128i lhs, __m128i rhs) noexcept { return _mm_cmpgt_epi64(lhs, rhs); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_mullo_epi64(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_mullo_epi64(__m128i lhs, __m128i rhs) noexcept { const __m128i productLow = _mm_mul_epu32(lhs, rhs); const __m128i lhsHigh = _mm_srli_epi64(lhs, 32); @@ -487,26 +1464,26 @@ SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_mullo_epi64(__m128i lhs, __m128i rh return _mm_add_epi64(productLow, _mm_slli_epi64(cross, 32)); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_abs_epi64(__m128i lhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_abs_epi64(__m128i lhs) noexcept { const __m128i zero = _mm_setzero_si128(); const __m128i sign = _mm_cmpgt_epi64(zero, lhs); return _mm_sub_epi64(_mm_xor_si128(lhs, sign), sign); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_min_epi64(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_min_epi64(__m128i lhs, __m128i rhs) noexcept { const __m128i mask = _mm_cmpgt_epi64(lhs, rhs); return _mm_blendv_epi8(lhs, rhs, mask); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_max_epi64(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_max_epi64(__m128i lhs, __m128i rhs) noexcept { const __m128i mask = _mm_cmpgt_epi64(lhs, rhs); return _mm_blendv_epi8(rhs, lhs, mask); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_srai_epi64(__m128i lhs, const int count) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_srai_epi64(__m128i lhs, const int count) noexcept { if (count <= 0) { @@ -527,51 +1504,23 @@ SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_srai_epi64(__m128i lhs, const int c return _mm_or_si128(logical, fill); } -// AVX2 has no efficient exact variable u64/s64 vector divide. For general-purpose -// per-lane divisors, unpacking to scalar hardware division is faster than a bit-serial -// SIMD long-division loop and preserves exact integer semantics. - -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_div_epu64(__m128i lhs, __m128i rhs) noexcept -{ - return register_from_values<__m128i, std::uint64_t>(register_get(lhs, 0) / register_get(rhs, 0), - register_get(lhs, 1) / register_get(rhs, 1)); -} - -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_rem_epu64(__m128i lhs, __m128i rhs) noexcept -{ - return register_from_values<__m128i, std::uint64_t>(register_get(lhs, 0) % register_get(rhs, 0), - register_get(lhs, 1) % register_get(rhs, 1)); -} - -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_div_epi64(__m128i lhs, __m128i rhs) noexcept -{ - return register_from_values<__m128i, std::int64_t>(register_get(lhs, 0) / register_get(rhs, 0), - register_get(lhs, 1) / register_get(rhs, 1)); -} - -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_rem_epi64(__m128i lhs, __m128i rhs) noexcept -{ - return register_from_values<__m128i, std::int64_t>(register_get(lhs, 0) % register_get(rhs, 0), - register_get(lhs, 1) % register_get(rhs, 1)); -} - #pragma endregion #pragma region 128bit uint64_t Extensions -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_cmpgt_epu64(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_cmpgt_epu64(__m128i lhs, __m128i rhs) noexcept { const __m128i signBit = _mm_set1_epi64x(std::numeric_limits::min()); return _mm_cmpgt_epi64(_mm_xor_si128(lhs, signBit), _mm_xor_si128(rhs, signBit)); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_min_epu64(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_min_epu64(__m128i lhs, __m128i rhs) noexcept { const __m128i mask = _ext_cmpgt_epu64(lhs, rhs); return _mm_blendv_epi8(lhs, rhs, mask); } -SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_max_epu64(__m128i lhs, __m128i rhs) noexcept +__m128i SIMD_FLAGS(InOut, ForceInline) _ext_max_epu64(__m128i lhs, __m128i rhs) noexcept { const __m128i mask = _ext_cmpgt_epu64(lhs, rhs); return _mm_blendv_epi8(rhs, lhs, mask); @@ -581,46 +1530,80 @@ SIMDLIB_FORCE_INLINE __m128i VECTORCALL _ext_max_epu64(__m128i lhs, __m128i rhs) #pragma region 128bit uint128_t Extentions -SIMDLIB_FORCE_INLINE constexpr __m128i VECTORCALL _ext128_shift_left_bits_dynamic(__m128i lhs, int shift) noexcept +/** + * @brief Shifts a complete 128-bit register left by a runtime bit count. + * @param lhs Source register interpreted as one unsigned 128-bit bit string. + * @param shift Runtime count; nonpositive counts are identity and counts of at least 128 produce zero. + * @return Shifted register with zero-filled low bits. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) _ext128_shift_bits_left_slow(const __m128i lhs, const int shift) noexcept { - if (shift <= 0) - return lhs; - if (shift >= 128) - return register_from_values<__m128i, std::uint64_t>(0, 0); - - const auto lanes = register_to_array(lhs); - if (shift == 64) - return register_from_values<__m128i, std::uint64_t>(0, lanes[0]); - if (shift < 64) - return register_from_values<__m128i, std::uint64_t>(lanes[0] << shift, (lanes[1] << shift) | (lanes[0] >> (64 - shift))); - return register_from_values<__m128i, std::uint64_t>(0, lanes[0] << (shift - 64)); + const __m128i count = _mm_min_epi32(_mm_max_epi32(_mm_cvtsi32_si128(shift), _mm_setzero_si128()), _mm_cvtsi32_si128(128)); + const __m128i midpoint = _mm_cvtsi32_si128(64); + const __m128i complement = _mm_sub_epi64(midpoint, count); + const __m128i excess = _mm_sub_epi64(count, midpoint); + const __m128i low_range = _mm_or_si128(_mm_sll_epi64(lhs, count), _mm_slli_si128(_mm_srl_epi64(lhs, complement), 8)); + const __m128i high_range = _mm_sll_epi64(_mm_slli_si128(lhs, 8), excess); + return _mm_or_si128(low_range, high_range); } -template SIMDLIB_FORCE_INLINE constexpr __m128i VECTORCALL _ext128_shift_left_bits_static(__m128i lhs) noexcept +/** + * @brief Shifts a complete 128-bit register left by a compile-time bit count. + * @tparam shift Nonnegative bit count; counts of at least 128 produce zero. + * @param lhs Source register interpreted as one unsigned 128-bit bit string. + * @return Shifted register with zero-filled low bits. + */ +template __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) _ext128_shift_bits_left_static(const __m128i lhs) noexcept { static_assert(shift >= 0, "Whole-register shifts require a non-negative count."); - return _ext128_shift_left_bits_dynamic(lhs, shift); + if constexpr (shift == 0) + return lhs; + else if constexpr (shift >= 128) + return _mm_setzero_si128(); + else if constexpr (shift < 64) + return _mm_or_si128(_mm_slli_epi64(lhs, shift), _mm_slli_si128(_mm_srli_epi64(lhs, 64 - shift), 8)); + else if constexpr (shift == 64) + return _mm_slli_si128(lhs, 8); + else + return _mm_slli_epi64(_mm_slli_si128(lhs, 8), shift - 64); } -SIMDLIB_FORCE_INLINE constexpr __m128i VECTORCALL _ext128_shift_right_bits_dynamic(__m128i lhs, int shift) noexcept +/** + * @brief Shifts a complete 128-bit register right by a runtime bit count. + * @param lhs Source register interpreted as one unsigned 128-bit bit string. + * @param shift Runtime count; nonpositive counts are identity and counts of at least 128 produce zero. + * @return Shifted register with zero-filled high bits. + */ +__m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) _ext128_shift_bits_right_slow(const __m128i lhs, const int shift) noexcept { - if (shift <= 0) - return lhs; - if (shift >= 128) - return register_from_values<__m128i, std::uint64_t>(0, 0); - - const auto lanes = register_to_array(lhs); - if (shift == 64) - return register_from_values<__m128i, std::uint64_t>(lanes[1], 0); - if (shift < 64) - return register_from_values<__m128i, std::uint64_t>((lanes[0] >> shift) | (lanes[1] << (64 - shift)), lanes[1] >> shift); - return register_from_values<__m128i, std::uint64_t>(lanes[1] >> (shift - 64), 0); + const __m128i count = _mm_min_epi32(_mm_max_epi32(_mm_cvtsi32_si128(shift), _mm_setzero_si128()), _mm_cvtsi32_si128(128)); + const __m128i midpoint = _mm_cvtsi32_si128(64); + const __m128i complement = _mm_sub_epi64(midpoint, count); + const __m128i excess = _mm_sub_epi64(count, midpoint); + const __m128i low_range = _mm_or_si128(_mm_srl_epi64(lhs, count), _mm_srli_si128(_mm_sll_epi64(lhs, complement), 8)); + const __m128i high_range = _mm_srl_epi64(_mm_srli_si128(lhs, 8), excess); + return _mm_or_si128(low_range, high_range); } -template SIMDLIB_FORCE_INLINE constexpr __m128i VECTORCALL _ext128_shift_right_bits_static(__m128i lhs) noexcept +/** + * @brief Shifts a complete 128-bit register right by a compile-time bit count. + * @tparam shift Nonnegative bit count; counts of at least 128 produce zero. + * @param lhs Source register interpreted as one unsigned 128-bit bit string. + * @return Shifted register with zero-filled high bits. + */ +template __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) _ext128_shift_bits_right_static(const __m128i lhs) noexcept { static_assert(shift >= 0, "Whole-register shifts require a non-negative count."); - return _ext128_shift_right_bits_dynamic(lhs, shift); + if constexpr (shift == 0) + return lhs; + else if constexpr (shift >= 128) + return _mm_setzero_si128(); + else if constexpr (shift < 64) + return _mm_or_si128(_mm_srli_epi64(lhs, shift), _mm_srli_si128(_mm_slli_epi64(lhs, 64 - shift), 8)); + else if constexpr (shift == 64) + return _mm_srli_si128(lhs, 8); + else + return _mm_srli_epi64(_mm_srli_si128(lhs, 8), shift - 64); } #pragma endregion @@ -633,19 +1616,18 @@ template SIMDLIB_FORCE_INLINE constexpr __m128i VECTORCALL _ext128_s * @param lhs The floating-point lanes. * @return The per-lane absolute values. */ -SIMDLIB_FORCE_INLINE __m128 VECTORCALL _ext_abs_ps(const __m128 lhs) noexcept +__m128 SIMD_FLAGS(InOut, ForceInline) _ext_abs_ps(const __m128 lhs) noexcept { return _mm_and_ps(lhs, _mm_castsi128_ps(_mm_set1_epi32(0x7FFFFFFF))); } - /** * @brief Clears the sign bit of each 64-bit floating-point lane. * * @param lhs The floating-point lanes. * @return The per-lane absolute values. */ -SIMDLIB_FORCE_INLINE __m128d VECTORCALL _ext_abs_pd(const __m128d lhs) noexcept +__m128d SIMD_FLAGS(InOut, ForceInline) _ext_abs_pd(const __m128d lhs) noexcept { return _mm_and_pd(lhs, _mm_castsi128_pd(_mm_set1_epi64x(0x7FFF'FFFF'FFFF'FFFFLL))); } @@ -664,7 +1646,7 @@ SIMDLIB_FORCE_INLINE __m128d VECTORCALL _ext_abs_pd(const __m128d lhs) noexcept * @param rhs The second byte-lane register. * @return The low byte of each lane product. */ -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_mul_epi8(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_mul_epi8(__m256i lhs, __m256i rhs) noexcept { // unpack and multiply const auto dst_even = _mm256_mullo_epi16(lhs, rhs); @@ -674,19 +1656,19 @@ SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_mul_epi8(__m256i lhs, __m256i rh return _mm256_blendv_epi8(dst_odd, dst_even, mask); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_cmplt_epi8(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_cmplt_epi8(__m256i lhs, __m256i rhs) noexcept { // Compare (b > a) which is effectively (a < b) return _mm256_cmpgt_epi8(rhs, lhs); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_slli_epx8(__m256i lhs, const int count) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_slli_epx8(__m256i lhs, const int count) noexcept { const __m256i mask = _mm256_set1_epi8(0xFF << count); return _mm256_and_si256(_mm256_slli_epi16(lhs, count), mask); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_srli_epx8(__m256i lhs, const int count) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_srli_epx8(__m256i lhs, const int count) noexcept { const __m256i mask = _mm256_set1_epi8(0xFF >> count); return _mm256_and_si256(_mm256_srli_epi16(lhs, count), mask); @@ -699,7 +1681,7 @@ SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_srli_epx8(__m256i lhs, const int * @param count The per-lane shift count. * @return The arithmetic-right-shifted byte lanes. */ -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_srai_epx8(__m256i lhs, const int count) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_srai_epx8(__m256i lhs, const int count) noexcept { __m256i aeven = _mm256_slli_epi16(lhs, 8); // even numbered elements get sign bit in position aeven = _mm256_sra_epi16(aeven, _mm_cvtsi32_si128(count + 8)); // shift arithmetic, back to position @@ -713,17 +1695,17 @@ SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_srai_epx8(__m256i lhs, const int #pragma region 256bit uint8_t Extensions -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_mul_epu8(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_mul_epu8(__m256i lhs, __m256i rhs) noexcept { return _ext256_mul_epi8(lhs, rhs); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_set1_epu8(std::uint8_t value) noexcept +__m256i SIMD_FLAGS(Out, ForceInline) _ext256_set1_epu8(std::uint8_t value) noexcept { return _mm256_set1_epi8(static_cast(value)); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_cmpgt_epu8(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_cmpgt_epu8(__m256i lhs, __m256i rhs) noexcept { // Returns 0xFF where x > y: return _mm256_andnot_si256(_mm256_cmpeq_epi8(lhs, rhs), _mm256_cmpeq_epi8(_mm256_max_epu8(lhs, rhs), lhs)); @@ -733,7 +1715,7 @@ SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_cmpgt_epu8(__m256i lhs, __m256i #pragma region 256bit uint16_t Extensions -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_cmpgt_epu16(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_cmpgt_epu16(__m256i lhs, __m256i rhs) noexcept { // Returns 0xFF where x > y: return _mm256_andnot_si256(_mm256_cmpeq_epi16(lhs, rhs), _mm256_cmpeq_epi16(_mm256_max_epu16(lhs, rhs), lhs)); @@ -743,7 +1725,7 @@ SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_cmpgt_epu16(__m256i lhs, __m256i #pragma region 256bit uint32_t Extensions -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_cmpgt_epu32(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_cmpgt_epu32(__m256i lhs, __m256i rhs) noexcept { // Returns 0xFF where x > y: return _mm256_andnot_si256(_mm256_cmpeq_epi32(lhs, rhs), _mm256_cmpeq_epi32(_mm256_max_epu32(lhs, rhs), lhs)); @@ -753,13 +1735,13 @@ SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_cmpgt_epu32(__m256i lhs, __m256i #pragma region 256bit uint64_t Extensions -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_cmpgt_epu64(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_cmpgt_epu64(__m256i lhs, __m256i rhs) noexcept { const __m256i signBit = _mm256_set1_epi64x(std::numeric_limits::min()); return _mm256_cmpgt_epi64(_mm256_xor_si256(lhs, signBit), _mm256_xor_si256(rhs, signBit)); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_mullo_epi64(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_mullo_epi64(__m256i lhs, __m256i rhs) noexcept { const __m256i productLow = _mm256_mul_epu32(lhs, rhs); const __m256i lhsHigh = _mm256_srli_epi64(lhs, 32); @@ -768,38 +1750,38 @@ SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_mullo_epi64(__m256i lhs, __m256i return _mm256_add_epi64(productLow, _mm256_slli_epi64(cross, 32)); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_abs_epi64(__m256i lhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_abs_epi64(__m256i lhs) noexcept { const __m256i zero = _mm256_setzero_si256(); const __m256i sign = _mm256_cmpgt_epi64(zero, lhs); return _mm256_sub_epi64(_mm256_xor_si256(lhs, sign), sign); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_min_epi64(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_min_epi64(__m256i lhs, __m256i rhs) noexcept { const __m256i mask = _mm256_cmpgt_epi64(lhs, rhs); return _mm256_blendv_epi8(lhs, rhs, mask); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_max_epi64(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_max_epi64(__m256i lhs, __m256i rhs) noexcept { const __m256i mask = _mm256_cmpgt_epi64(lhs, rhs); return _mm256_blendv_epi8(rhs, lhs, mask); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_min_epu64(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_min_epu64(__m256i lhs, __m256i rhs) noexcept { const __m256i mask = _ext256_cmpgt_epu64(lhs, rhs); return _mm256_blendv_epi8(lhs, rhs, mask); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_max_epu64(__m256i lhs, __m256i rhs) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_max_epu64(__m256i lhs, __m256i rhs) noexcept { const __m256i mask = _ext256_cmpgt_epu64(lhs, rhs); return _mm256_blendv_epi8(rhs, lhs, mask); } -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_srai_epi64(__m256i lhs, const int count) noexcept +__m256i SIMD_FLAGS(InOut, ForceInline) _ext256_srai_epi64(__m256i lhs, const int count) noexcept { if (count <= 0) { @@ -820,38 +1802,6 @@ SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_srai_epi64(__m256i lhs, const in return _mm256_or_si256(logical, fill); } -// AVX2 has no efficient exact variable u64/s64 vector divide. For general-purpose -// per-lane divisors, unpacking to scalar hardware division is faster than a bit-serial -// SIMD long-division loop and preserves exact integer semantics. - -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_div_epu64(__m256i lhs, __m256i rhs) noexcept -{ - return register_from_values<__m256i, std::uint64_t>( - register_get(lhs, 0) / register_get(rhs, 0), register_get(lhs, 1) / register_get(rhs, 1), - register_get(lhs, 2) / register_get(rhs, 2), register_get(lhs, 3) / register_get(rhs, 3)); -} - -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_rem_epu64(__m256i lhs, __m256i rhs) noexcept -{ - return register_from_values<__m256i, std::uint64_t>( - register_get(lhs, 0) % register_get(rhs, 0), register_get(lhs, 1) % register_get(rhs, 1), - register_get(lhs, 2) % register_get(rhs, 2), register_get(lhs, 3) % register_get(rhs, 3)); -} - -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_div_epi64(__m256i lhs, __m256i rhs) noexcept -{ - return register_from_values<__m256i, std::int64_t>( - register_get(lhs, 0) / register_get(rhs, 0), register_get(lhs, 1) / register_get(rhs, 1), - register_get(lhs, 2) / register_get(rhs, 2), register_get(lhs, 3) / register_get(rhs, 3)); -} - -SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_rem_epi64(__m256i lhs, __m256i rhs) noexcept -{ - return register_from_values<__m256i, std::int64_t>( - register_get(lhs, 0) % register_get(rhs, 0), register_get(lhs, 1) % register_get(rhs, 1), - register_get(lhs, 2) % register_get(rhs, 2), register_get(lhs, 3) % register_get(rhs, 3)); -} - #pragma endregion #pragma region 256bit float Extensions @@ -862,7 +1812,7 @@ SIMDLIB_FORCE_INLINE __m256i VECTORCALL _ext256_rem_epi64(__m256i lhs, __m256i r * @param lhs The floating-point lanes. * @return The per-lane absolute values. */ -SIMDLIB_FORCE_INLINE __m256 VECTORCALL _ext256_abs_ps(const __m256 lhs) noexcept +__m256 SIMD_FLAGS(InOut, ForceInline) _ext256_abs_ps(const __m256 lhs) noexcept { return _mm256_and_ps(lhs, _mm256_castsi256_ps(_mm256_set1_epi32(0x7FFFFFFF))); } @@ -873,17 +1823,17 @@ SIMDLIB_FORCE_INLINE __m256 VECTORCALL _ext256_abs_ps(const __m256 lhs) noexcept * @param lhs The floating-point lanes. * @return The per-lane absolute values. */ -SIMDLIB_FORCE_INLINE __m256d VECTORCALL _ext256_abs_pd(const __m256d lhs) noexcept +__m256d SIMD_FLAGS(InOut, ForceInline) _ext256_abs_pd(const __m256d lhs) noexcept { return _mm256_and_pd(lhs, _mm256_castsi256_pd(_mm256_set1_epi64x(0x7FFF'FFFF'FFFF'FFFFLL))); } -SIMDLIB_FORCE_INLINE __m256 VECTORCALL _ext256_cmpeq_ps(__m256 lhs, __m256 rhs) noexcept +__m256 SIMD_FLAGS(InOut, ForceInline) _ext256_cmpeq_ps(__m256 lhs, __m256 rhs) noexcept { return _mm256_cmp_ps(lhs, rhs, _CMP_EQ_OQ); } -SIMDLIB_FORCE_INLINE __m256 VECTORCALL _ext256_cmpgt_ps(__m256 lhs, __m256 rhs) noexcept +__m256 SIMD_FLAGS(InOut, ForceInline) _ext256_cmpgt_ps(__m256 lhs, __m256 rhs) noexcept { return _mm256_cmp_ps(lhs, rhs, _CMP_GT_OQ); } @@ -894,7 +1844,7 @@ SIMDLIB_FORCE_INLINE __m256 VECTORCALL _ext256_cmpgt_ps(__m256 lhs, __m256 rhs) * @param rhs The second floating-point register. * @return An all-ones lane mask where corresponding lanes are equal. */ -SIMDLIB_FORCE_INLINE __m256d VECTORCALL _ext256_cmpeq_pd(const __m256d lhs, const __m256d rhs) noexcept +__m256d SIMD_FLAGS(InOut, ForceInline) _ext256_cmpeq_pd(const __m256d lhs, const __m256d rhs) noexcept { return _mm256_cmp_pd(lhs, rhs, _CMP_EQ_OQ); } @@ -906,17 +1856,12 @@ SIMDLIB_FORCE_INLINE __m256d VECTORCALL _ext256_cmpeq_pd(const __m256d lhs, cons * @param rhs The second floating-point register. * @return An all-ones lane mask where lhs is greater than rhs. */ -SIMDLIB_FORCE_INLINE __m256d VECTORCALL _ext256_cmpgt_pd(const __m256d lhs, const __m256d rhs) noexcept +__m256d SIMD_FLAGS(InOut, ForceInline) _ext256_cmpgt_pd(const __m256d lhs, const __m256d rhs) noexcept { return _mm256_cmp_pd(lhs, rhs, _CMP_GT_OQ); } -SIMDLIB_FORCE_INLINE __m256 VECTORCALL _ext256_extract_ps(__m256 lhs, const int imm8) noexcept -{ - return _mm256_permutevar8x32_ps(lhs, _mm256_set1_epi32(imm8)); -} - -// SIMDLIB_FORCE_INLINE VECTORCALL __m256 _ext256_insert_ps(__m256 lhs, __m128 rhs, const int imm8) noexcept +// __m256 SIMD_FLAGS(InOut, ForceInline) _ext256_insert_ps(__m256 lhs, __m128 rhs, const int imm8) noexcept //{ // return _mm256_insertf128_ps(lhs, rhs, imm8); // } diff --git a/include/SimdLib/Detail/Implementations.h b/include/SimdLib/Detail/Implementations.h index 7811a72..f6c256b 100644 --- a/include/SimdLib/Detail/Implementations.h +++ b/include/SimdLib/Detail/Implementations.h @@ -1,12 +1,24 @@ #pragma once #include #include +#include #include #include +#include +#include #include +#if SIMDLIB_COMPILER_MSVC && SIMDLIB_TARGET_X86 #include +#endif #include - +#include + +/* + * This file contains SIMD operation abstractions for 128-bit & 256-bit register types across all integer and floating-point numeric types. + * SEE: http://www.alfredklomp.com/programming/sse-intrinsics/ + * SEE: https://agner.org/optimize/optimizing_assembly.pdf + * SEE: https://software.intel.com/sites/landingpage/IntrinsicsGuide/ + */ namespace SimdLib::Detail { /// Provides a common interface of standard SIMD method alias names for different integer types. @@ -32,6 +44,103 @@ template requires std::is_integral_v using promoted_unsigned_t = SimdLib::select_unsigned_integer_t>; +/** + * @brief Rounds the square root of a 64-bit square sum and corrects the floating estimate exactly. + * + * @param total The nonnegative square sum. + * @param maximum The greatest magnitude representable by the destination element type. + * @return The nearest integer square root, saturated to `maximum`. + */ +std::uint64_t SIMD_FLAGS(Neither, RegisterOnly, ForceInline) magnitude_round_sqrt_u64(const std::uint64_t total, const std::uint64_t maximum) noexcept +{ + const __m128d totalValue = _mm_set_sd(static_cast(total)); + const double root = _mm_cvtsd_f64(_mm_sqrt_sd(_mm_setzero_pd(), totalValue)); + std::uint64_t candidate = static_cast(root + 0.5); + if (candidate > maximum) + candidate = maximum; + if (candidate > 0 && total < candidate * candidate - candidate + 1) + --candidate; + if (candidate < maximum && total > candidate * candidate + candidate) + ++candidate; + return candidate; +} + +/** + * @brief Packs one checked magnitude and its canonical overflow mask into the first two lanes. + * + * @tparam element_t The signed or unsigned integer lane type. + * @param magnitude The saturated magnitude stored in lane zero. + * @param overflow Whether lane one should contain an all-ones mask. + * @return A native register whose remaining lanes are unspecified. + */ +template + requires std::is_integral_v +__m128i SIMD_FLAGS(Out, RegisterOnly, ForceInline) magnitude_checked_result(const std::uint64_t magnitude, const bool overflow) noexcept +{ + constexpr std::uint64_t laneMask = []() constexpr + { + if constexpr (sizeof(element_t) == 8) + return ~std::uint64_t{0}; + else + return (std::uint64_t{1} << (sizeof(element_t) * 8)) - 1; + }(); + const std::uint64_t low = magnitude & laneMask; + if constexpr (sizeof(element_t) == 8) + return _mm_set_epi64x(overflow ? -1 : 0, static_cast(low)); + else + return _mm_cvtsi64_si128(static_cast(low | ((overflow ? laneMask : 0) << (sizeof(element_t) * 8)))); +} +/** + * @brief Squares one unsigned 64-bit value into low and high 64-bit register lanes. + * + * @param value The unsigned scalar value. + * @return A register containing the 128-bit product as `[low, high]`. + */ +__m128i SIMD_FLAGS(Out, RegisterOnly, ForceInline) magnitude_square_u64(const std::uint64_t value) noexcept +{ +#if SIMDLIB_COMPILER_MSVC + std::uint64_t high = 0; + const std::uint64_t low = _umul128(value, value, &high); + return _mm_set_epi64x(static_cast(high), static_cast(low)); +#else + const std::uint64_t lowHalf = static_cast(value); + const std::uint64_t highHalf = value >> 32; + const std::uint64_t lowSquare = lowHalf * lowHalf; + const std::uint64_t cross = highHalf * lowHalf; + const std::uint64_t low = lowSquare + (cross << 33); + const std::uint64_t high = highHalf * highHalf + (cross >> 31) + static_cast(low < lowSquare); + return _mm_set_epi64x(static_cast(high), static_cast(low)); +#endif +} + +/** + * @brief Converts an exact 128-bit square sum into a rounded, bounded 64-bit magnitude. + * + * @param low The low 64 bits of the square sum. + * @param high The high 64 bits of the square sum. + * @param maximum The greatest representable destination magnitude. + * @return The floating estimate rounded to the nearest integer and bounded by `maximum`. + */ +std::uint64_t SIMD_FLAGS(Neither, RegisterOnly, ForceInline) + magnitude_round_sqrt_u128(const std::uint64_t low, const std::uint64_t high, const std::uint64_t maximum) noexcept +{ + constexpr double twoTo64 = 18'446'744'073'709'551'616.0; + constexpr double twoTo63 = 9'223'372'036'854'775'808.0; + const __m128d highValue = _mm_set_sd(static_cast(high)); + const __m128d lowValue = _mm_set_sd(static_cast(low)); + const __m128d total = _mm_add_sd(_mm_mul_sd(highValue, _mm_set_sd(twoTo64)), lowValue); + const double root = _mm_cvtsd_f64(_mm_sqrt_sd(_mm_setzero_pd(), total)); + const double maximumAsDouble = static_cast(maximum); + if (root >= maximumAsDouble) + return maximum; + const double rounded = root + 0.5; + if (rounded >= maximumAsDouble) + return maximum; + if (rounded < twoTo63) + return static_cast(rounded); + return static_cast(rounded - twoTo63) + (std::uint64_t{1} << 63); +} + #if SIMDLIB_HAS_SSE42 #pragma region 128-bit Implementations @@ -42,14 +151,92 @@ struct SimdImpl128 { }; +/** + * @brief Encodes four logical 32-bit selectors in low-to-high result-lane order. + * @tparam index0 Source lane for result lane zero. + * @tparam index1 Source lane for result lane one. + * @tparam index2 Source lane for result lane two. + * @tparam index3 Source lane for result lane three. + * @return Immediate accepted by the 128-bit four-lane shuffle intrinsics. + */ +template +[[nodiscard]] consteval int encode_logical_shuffle_32_immediate() noexcept +{ + return static_cast(index0 | (index1 << 2) | (index2 << 4) | (index3 << 6)); +} + +/** + * @brief Encodes one byte of a selected logical 16-bit lane. + * @param index Logical 16-bit source-lane selector. + * @param byte Byte position within the selected lane. + * @return Byte selector accepted by the 128-bit byte-shuffle intrinsic. + */ +[[nodiscard]] consteval int encode_logical_shuffle_16_byte(const std::size_t index, const std::size_t byte) noexcept +{ + return static_cast((index * 2) + byte); +} + +/** + * @brief Builds the constant byte-control register for a logical 16-bit shuffle. + * @tparam indices Logical 16-bit lane selectors. + * @tparam byte_positions Byte positions in the resulting control register. + * @return Native byte-control register for the 128-bit byte-shuffle intrinsic. + */ +template +static __m128i SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) make_logical_shuffle_16_control(std::index_sequence) noexcept +{ + return _mm_setr_epi8(encode_logical_shuffle_16_byte(indices[byte_positions / 2], byte_positions % 2)...); +} + +/** + * @brief Encodes two logical 64-bit selectors as inseparable 32-bit pairs. + * @tparam index0 Source lane for result lane zero. + * @tparam index1 Source lane for result lane one. + * @return Immediate accepted by the 128-bit 32-bit-lane shuffle intrinsic. + */ +template [[nodiscard]] consteval int encode_logical_shuffle_64_immediate() noexcept +{ + return encode_logical_shuffle_32_immediate<(index0 * 2), (index0 * 2) + 1, (index1 * 2), (index1 * 2) + 1>(); +} + +/** + * @brief Encodes two logical 64-bit floating-point selectors in low-to-high result-lane order. + * @tparam index0 Source lane for result lane zero. + * @tparam index1 Source lane for result lane one. + * @return Immediate accepted by the 128-bit two-lane floating-point shuffle intrinsic. + */ +template [[nodiscard]] consteval int encode_logical_shuffle_double_immediate() noexcept +{ + return static_cast(index0 | (index1 << 1)); +} + template <> struct SimdImpl128 { + /** @brief Selects bytes from two registers using a canonical predicate register. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128i condition, __m128i when_true, __m128i when_false) noexcept + { + return _mm_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical signed-byte lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 16 && ((indices < 16) && ...)) + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128i lhs) noexcept + { + return _mm_shuffle_epi8(lhs, _mm_setr_epi8(static_cast(indices)...)); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m128i lhsWideLo = _mm_cvtepi8_epi16(lhs); const __m128i rhsWideLo = _mm_cvtepi8_epi16(rhs); @@ -57,27 +244,31 @@ template <> struct SimdImpl128 const __m128i rhsWideHi = _mm_cvtepi8_epi16(_mm_srli_si128(rhs, 8)); return _mm_hadd_epi16(_mm_mullo_epi16(lhsWideLo, rhsWideLo), _mm_mullo_epi16(lhsWideHi, rhsWideHi)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _ext_mul_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding signed 8-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext128_div_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes corresponding signed 8-bit remainders with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext128_rem_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { auto sqrt16 = [](__m128i values) noexcept { @@ -92,7 +283,37 @@ template <> struct SimdImpl128 const __m128i hi16 = sqrt16(_mm_cvtepi8_epi16(_mm_srli_si128(lhs, 8))); return _mm_packs_epi16(lo16, hi16); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes the unchecked group magnitude in lane zero; all other lanes are unspecified. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i low = _mm_cvtepi8_epi16(lhs); + const __m128i high = _mm_cvtepi8_epi16(_mm_srli_si128(lhs, 8)); + __m128i total = _mm_add_epi16(_mm_mullo_epi16(low, low), _mm_mullo_epi16(high, high)); + total = _mm_hadd_epi16(total, total); + total = _mm_hadd_epi16(total, total); + total = _mm_hadd_epi16(total, total); + const __m128 squareSum = _mm_cvtepi32_ps(_mm_cvtepu16_epi32(total)); + return _mm_cvtps_epi32(_mm_sqrt_ss(squareSum)); + } + + /** @brief Computes a saturated magnitude in lane zero and a canonical overflow mask in lane one. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + constexpr std::uint64_t maximum = static_cast(std::numeric_limits::max()); + constexpr std::uint64_t threshold = maximum * maximum + maximum + 1; + const __m128i low = _mm_cvtepi8_epi16(lhs); + const __m128i high = _mm_cvtepi8_epi16(_mm_srli_si128(lhs, 8)); + __m128i pairSums = _mm_add_epi32(_mm_madd_epi16(low, low), _mm_madd_epi16(high, high)); + pairSums = _mm_hadd_epi32(pairSums, pairSums); + pairSums = _mm_hadd_epi32(pairSums, pairSums); + const std::uint64_t total = static_cast(_mm_cvtsi128_si32(pairSums)); + const bool overflow = total >= threshold; + const std::uint64_t result = overflow ? maximum : magnitude_round_sqrt_u64(total, maximum); + return magnitude_checked_result(result, overflow); + } + + /** @brief Computes minimum-value position metadata for this native register specialization. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { constexpr __m128i indices = register_from_values<__m128i, std::int8_t>(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); __m128i values = lhs; @@ -109,92 +330,96 @@ template <> struct SimdImpl128 reduce.template operator()<2>(); reduce.template operator()<4>(); reduce.template operator()<8>(); - alignas(16) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(output.data()), values); - output[1] = static_cast(_mm_extract_epi8(positions, 0)); - return _mm_load_si128(reinterpret_cast(output.data())); + return _mm_insert_epi8(values, _mm_extract_epi8(positions, 0), 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m128i lhs, __m128i rhs) noexcept { return _mm_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm_abs_epi8(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm_sub_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm_min_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm_max_epi8(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _ext_slli_epx8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _ext_srli_epx8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _ext_srai_epx8(lhs, rhs); } // arithmetic (saturated) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_saturated(auto lhs, auto rhs) noexcept + /** @brief Adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(auto lhs, auto rhs) noexcept { return _mm_adds_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(auto lhs, auto rhs) noexcept { return _mm_subs_epi8(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm_set1_epi8(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm_set_epi8(static_cast(args)...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm_setr_epi8(static_cast(args)...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm_cmpgt_epi8(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm_cvtepi8_epi16(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static typename target_simd::vector_t VECTORCALL widen(auto lhs) noexcept + template static typename target_simd::vector_t SIMD_FLAGS(InOut, ForceInline) widen(auto lhs) noexcept { using target_element_t = typename target_simd::element_type; if constexpr (target_simd::register_width == 128) @@ -228,39 +453,105 @@ template <> struct SimdImpl128 } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm_extract_epi8(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected signed 8-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 16)`. + * @return Selected scalar lane. + */ + static int8_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 16, "Signed 8-bit extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + case 2: + return extract<2>(lhs); + case 3: + return extract<3>(lhs); + case 4: + return extract<4>(lhs); + case 5: + return extract<5>(lhs); + case 6: + return extract<6>(lhs); + case 7: + return extract<7>(lhs); + case 8: + return extract<8>(lhs); + case 9: + return extract<9>(lhs); + case 10: + return extract<10>(lhs); + case 11: + return extract<11>(lhs); + case 12: + return extract<12>(lhs); + case 13: + return extract<13>(lhs); + case 14: + return extract<14>(lhs); + case 15: + return extract<15>(lhs); + default: + return extract<0>(lhs); + } + } + /** @brief Replaces the compile-time-selected signed 8-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const int8_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** @brief Replaces the compile-time-selected signed 8-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const int8_t rhs) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + return _mm_insert_epi8(lhs, static_cast(rhs), index); + } + /** + * @brief Replaces one runtime-selected signed 8-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 16)`. + * @return Register with the selected lane replaced. + */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128i lhs, const int8_t rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 16, "Signed 8-bit insertion requires a valid 128-bit lane index"); + const __m128i lane_indices = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const __m128i selected_lane = _mm_cmpeq_epi8(lane_indices, _mm_set1_epi8(static_cast(index))); + return _mm_blendv_epi8(lhs, _mm_set1_epi8(rhs), selected_lane); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm_unpacklo_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_epi8(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(auto lhs, auto rhs) noexcept + /** @brief Shuffles bytes through the native runtime selector-register instruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle(auto lhs, auto rhs) noexcept + requires(std::same_as && std::same_as) { return _mm_shuffle_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, auto mask) noexcept + /** @brief Selects bytes through the native runtime mask-register operation. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) blend(const __m128i lhs, const __m128i rhs, const __m128i mask) noexcept { return register_blend_bytes(lhs, rhs, mask); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL movemask(auto lhs) noexcept + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) movemask(auto lhs) noexcept { return _mm_movemask_epi8(lhs); } @@ -268,12 +559,31 @@ template <> struct SimdImpl128 template <> struct SimdImpl128 { + /** @brief Selects bytes from two registers using a canonical predicate register. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128i condition, __m128i when_true, __m128i when_false) noexcept + { + return _mm_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical unsigned-byte lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 16 && ((indices < 16) && ...)) + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128i lhs) noexcept + { + return _mm_shuffle_epi8(lhs, _mm_setr_epi8(static_cast(indices)...)); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m128i lhsWideLo = _mm_cvtepu8_epi16(lhs); const __m128i rhsWideLo = _mm_cvtepu8_epi16(rhs); @@ -281,27 +591,31 @@ template <> struct SimdImpl128 const __m128i rhsWideHi = _mm_cvtepu8_epi16(_mm_srli_si128(rhs, 8)); return _mm_hadd_epi16(_mm_mullo_epi16(lhsWideLo, rhsWideLo), _mm_mullo_epi16(lhsWideHi, rhsWideHi)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _ext_mul_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding unsigned 8-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext128_div_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes corresponding unsigned 8-bit remainders with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext128_rem_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { auto sqrt16 = [](__m128i values) noexcept { @@ -316,7 +630,37 @@ template <> struct SimdImpl128 const __m128i hi16 = sqrt16(_mm_cvtepu8_epi16(_mm_srli_si128(lhs, 8))); return _mm_packus_epi16(lo16, hi16); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes the unchecked group magnitude in lane zero; all other lanes are unspecified. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i low = _mm_cvtepu8_epi16(lhs); + const __m128i high = _mm_cvtepu8_epi16(_mm_srli_si128(lhs, 8)); + __m128i total = _mm_add_epi16(_mm_mullo_epi16(low, low), _mm_mullo_epi16(high, high)); + total = _mm_hadd_epi16(total, total); + total = _mm_hadd_epi16(total, total); + total = _mm_hadd_epi16(total, total); + const __m128 squareSum = _mm_cvtepi32_ps(_mm_cvtepu16_epi32(total)); + return _mm_cvtps_epi32(_mm_sqrt_ss(squareSum)); + } + + /** @brief Computes a saturated magnitude in lane zero and a canonical overflow mask in lane one. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + constexpr std::uint64_t maximum = static_cast(std::numeric_limits::max()); + constexpr std::uint64_t threshold = maximum * maximum + maximum + 1; + const __m128i low = _mm_cvtepu8_epi16(lhs); + const __m128i high = _mm_cvtepu8_epi16(_mm_srli_si128(lhs, 8)); + __m128i pairSums = _mm_add_epi32(_mm_madd_epi16(low, low), _mm_madd_epi16(high, high)); + pairSums = _mm_hadd_epi32(pairSums, pairSums); + pairSums = _mm_hadd_epi32(pairSums, pairSums); + const std::uint64_t total = static_cast(_mm_cvtsi128_si32(pairSums)); + const bool overflow = total >= threshold; + const std::uint64_t result = overflow ? maximum : magnitude_round_sqrt_u64(total, maximum); + return magnitude_checked_result(result, overflow); + } + + /** @brief Computes minimum-value position metadata for this native register specialization. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { constexpr __m128i indices = register_from_values<__m128i, std::uint8_t>(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); const __m128i signBit = _mm_set1_epi8(static_cast(0x80)); @@ -334,99 +678,104 @@ template <> struct SimdImpl128 reduce.template operator()<2>(); reduce.template operator()<4>(); reduce.template operator()<8>(); - alignas(16) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(output.data()), values); - output[1] = static_cast(_mm_extract_epi8(positions, 0)); - return _mm_load_si128(reinterpret_cast(output.data())); + return _mm_insert_epi8(values, _mm_extract_epi8(positions, 0), 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m128i lhs, __m128i rhs) noexcept { return _mm_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm_abs_epi8(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm_sub_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm_min_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm_max_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL avg(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise averages for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) avg(auto lhs, auto rhs) noexcept { return _mm_avg_epu8(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _ext_slli_epx8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _ext_srli_epx8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _ext_srai_epx8(lhs, rhs); } // arithmetic (horizontal) - // static SIMDLIB_FORCE_INLINE auto VECTORCALL hadd (auto lhs, auto rhs) noexcept { return _mm_hadd_epi8(lhs, rhs); } - // static SIMDLIB_FORCE_INLINE auto VECTORCALL hsub (auto lhs, auto rhs) noexcept { return _mm_hsub_epi8(lhs, rhs); } + // static auto SIMD_FLAGS(InOut, ForceInline) hadd (auto lhs, auto rhs) noexcept { return _mm_hadd_epi8(lhs, rhs); } + // static auto SIMD_FLAGS(InOut, ForceInline) hsub (auto lhs, auto rhs) noexcept { return _mm_hsub_epi8(lhs, rhs); } // arithmetic (saturated) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_saturated(auto lhs, auto rhs) noexcept + /** @brief Adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(auto lhs, auto rhs) noexcept { return _mm_adds_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(auto lhs, auto rhs) noexcept { return _mm_subs_epu8(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _ext_set1_epu8(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args &&...args) noexcept { return _mm_set_epi8(static_cast(args)...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args &&...args) noexcept { return _mm_setr_epi8(static_cast(args)...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext_cmpgt_epu8(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm_cvtepu8_epi16(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static typename target_simd::vector_t VECTORCALL widen(auto lhs) noexcept + template static typename target_simd::vector_t SIMD_FLAGS(InOut, ForceInline) widen(auto lhs) noexcept { using target_element_t = typename target_simd::element_type; if constexpr (target_simd::register_width == 128) @@ -460,35 +809,105 @@ template <> struct SimdImpl128 } // extract / insert - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept + { + return static_cast(_mm_extract_epi8(lhs, index)); + } + /** + * @brief Extracts one runtime-selected unsigned 8-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 16)`. + * @return Selected scalar lane. + */ + static uint8_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 16, "Unsigned 8-bit extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + case 2: + return extract<2>(lhs); + case 3: + return extract<3>(lhs); + case 4: + return extract<4>(lhs); + case 5: + return extract<5>(lhs); + case 6: + return extract<6>(lhs); + case 7: + return extract<7>(lhs); + case 8: + return extract<8>(lhs); + case 9: + return extract<9>(lhs); + case 10: + return extract<10>(lhs); + case 11: + return extract<11>(lhs); + case 12: + return extract<12>(lhs); + case 13: + return extract<13>(lhs); + case 14: + return extract<14>(lhs); + case 15: + return extract<15>(lhs); + default: + return extract<0>(lhs); + } + } + /** @brief Replaces the compile-time-selected unsigned 8-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const uint8_t rhs) noexcept + { + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected unsigned 8-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const uint8_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return _mm_insert_epi8(lhs, static_cast(rhs), index); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** + * @brief Replaces one runtime-selected unsigned 8-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 16)`. + * @return Register with the selected lane replaced. + */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128i lhs, const uint8_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + SIMDLIB_PRECONDITION(index >= 0 && index < 16, "Unsigned 8-bit insertion requires a valid 128-bit lane index"); + const __m128i lane_indices = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const __m128i selected_lane = _mm_cmpeq_epi8(lane_indices, _mm_set1_epi8(static_cast(index))); + return _mm_blendv_epi8(lhs, _mm_set1_epi8(std::bit_cast(rhs)), selected_lane); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm_unpacklo_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_epi8(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(auto lhs, auto rhs) noexcept + /** @brief Shuffles bytes through the native runtime selector-register instruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle(auto lhs, auto rhs) noexcept + requires(std::same_as && std::same_as) { return _mm_shuffle_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, auto mask) noexcept + /** @brief Selects bytes through the native runtime mask-register operation. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) blend(const __m128i lhs, const __m128i rhs, const __m128i mask) noexcept { return register_blend_bytes(lhs, rhs, mask); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL movemask(auto lhs) noexcept + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) movemask(auto lhs) noexcept { return _mm_movemask_epi8(lhs); } @@ -496,36 +915,59 @@ template <> struct SimdImpl128 template <> struct SimdImpl128 { + /** @brief Selects 16-bit lanes from two registers using a canonical predicate register. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128i condition, __m128i when_true, __m128i when_false) noexcept + { + return _mm_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical signed 16-bit lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 8 && ((indices < 8) && ...)) + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128i lhs) noexcept + { + return _mm_shuffle_epi8(lhs, make_logical_shuffle_16_control(std::make_index_sequence<16>{})); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { return _mm_madd_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm_mullo_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding signed 16-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext128_div_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes corresponding signed 16-bit remainders with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext128_rem_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { const __m128i lo32 = _mm_cvtepi16_epi32(lhs); const __m128i hi32 = _mm_cvtepi16_epi32(_mm_srli_si128(lhs, 8)); @@ -533,7 +975,36 @@ template <> struct SimdImpl128 const __m128i hiRoots = _mm_cvtps_epi32(_mm_sqrt_ps(_mm_cvtepi32_ps(hi32))); return _mm_packs_epi32(loRoots, hiRoots); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes the unchecked group magnitude in lane zero; all other lanes are unspecified. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + __m128i total = _mm_madd_epi16(lhs, lhs); + total = _mm_hadd_epi32(total, total); + total = _mm_hadd_epi32(total, total); + return _mm_cvtpd_epi32(_mm_sqrt_sd(_mm_cvtepi32_pd(total), _mm_cvtepi32_pd(total))); + } + + /** @brief Computes a saturated magnitude in lane zero and a canonical overflow mask in lane one. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + constexpr std::uint64_t maximum = static_cast(std::numeric_limits::max()); + constexpr std::uint64_t threshold = maximum * maximum + maximum + 1; + const __m128i minimum = _mm_set1_epi16(std::numeric_limits::min()); + if (_mm_movemask_epi8(_mm_cmpeq_epi16(lhs, minimum)) != 0) + return magnitude_checked_result(maximum, true); + const __m128i pairSquares = _mm_madd_epi16(lhs, lhs); + const __m128i lowPairs = _mm_cvtepu32_epi64(pairSquares); + const __m128i highPairs = _mm_cvtepu32_epi64(_mm_srli_si128(pairSquares, 8)); + const __m128i pairTotals = _mm_add_epi64(lowPairs, highPairs); + const __m128i totalVector = _mm_add_epi64(pairTotals, _mm_srli_si128(pairTotals, 8)); + const std::uint64_t total = static_cast(_mm_cvtsi128_si64(totalVector)); + const bool overflow = total >= threshold; + const std::uint64_t result = overflow ? maximum : magnitude_round_sqrt_u64(total, maximum); + return magnitude_checked_result(result, overflow); + } + + /** @brief Computes minimum-value position metadata for this native register specialization. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { constexpr __m128i indices = register_from_values<__m128i, std::int16_t>(0, 1, 2, 3, 4, 5, 6, 7); __m128i values = lhs; @@ -552,79 +1023,87 @@ template <> struct SimdImpl128 reduce.template operator()<1>(); reduce.template operator()<2>(); reduce.template operator()<4>(); - alignas(16) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(output.data()), values); - output[1] = static_cast(_mm_extract_epi16(positions, 0)); - return _mm_load_si128(reinterpret_cast(output.data())); + return _mm_insert_epi16(values, _mm_extract_epi16(positions, 0), 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m128i lhs, __m128i rhs) noexcept { return _mm_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm_abs_epi16(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm_sub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm_min_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm_max_epi16(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm_slli_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm_srli_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _mm_srai_epi16(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm_hadd_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm_hsub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL hadd_saturated(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hadd_saturated(auto lhs, auto rhs) noexcept { return _mm_hadds_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL hsubtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hsubtract_saturated(auto lhs, auto rhs) noexcept { return _mm_hsubs_epi16(lhs, rhs); } // arithmetic (saturated) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_saturated(auto lhs, auto rhs) noexcept + /** @brief Adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(auto lhs, auto rhs) noexcept { return _mm_adds_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(auto lhs, auto rhs) noexcept { return _mm_subs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_saturated(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) multiply_saturated(auto lhs, auto rhs) noexcept { const __m128i lhsLo = _mm_cvtepi16_epi32(lhs); const __m128i rhsLo = _mm_cvtepi16_epi32(rhs); @@ -634,35 +1113,35 @@ template <> struct SimdImpl128 } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm_set1_epi16(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args &&...args) noexcept { return _mm_set_epi16(static_cast(args)...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args &&...args) noexcept { return _mm_setr_epi16(static_cast(args)...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm_cmpgt_epi16(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm_cvtepi16_epi32(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static typename target_simd::vector_t VECTORCALL widen(auto lhs) noexcept + template static typename target_simd::vector_t SIMD_FLAGS(InOut, ForceInline) widen(auto lhs) noexcept { using target_element_t = typename target_simd::element_type; if constexpr (target_simd::register_width == 128) @@ -690,58 +1169,157 @@ template <> struct SimdImpl128 static_assert(dependent_false_v, "No direct widen mapping exists for SimdImpl128 and the requested destination SIMD shape."); } } - SIMDLIB_FORCE_INLINE static auto VECTORCALL compress(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) compress(auto lhs, auto rhs) noexcept { return _mm_packs_epi16(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm_extract_epi16(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected signed 16-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 8)`. + * @return Selected scalar lane. + */ + static int16_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "Signed 16-bit extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + case 2: + return extract<2>(lhs); + case 3: + return extract<3>(lhs); + case 4: + return extract<4>(lhs); + case 5: + return extract<5>(lhs); + case 6: + return extract<6>(lhs); + case 7: + return extract<7>(lhs); + default: + return extract<0>(lhs); + } + } + /** @brief Replaces the compile-time-selected signed 16-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const int16_t rhs) noexcept + { + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected signed 16-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const int16_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return _mm_insert_epi16(lhs, static_cast(rhs), index); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** + * @brief Replaces one runtime-selected signed 16-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 8)`. + * @return Register with the selected lane replaced. + */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128i lhs, const int16_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "Signed 16-bit insertion requires a valid 128-bit lane index"); + const __m128i lane_indices = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7); + const __m128i selected_lane = _mm_cmpeq_epi16(lane_indices, _mm_set1_epi16(static_cast(index))); + return _mm_blendv_epi8(lhs, _mm_set1_epi16(rhs), selected_lane); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm_unpacklo_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_epi16(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_lo(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled low-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each low four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_lo_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_half_16_slow(lhs, static_cast(rhs), false); + } + /** @brief Shuffles the low four 16-bit lanes in each 128-bit group with an immediate control. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_lo(__m128i lhs) noexcept + { + return _mm_shufflelo_epi16(lhs, imm8); + } + /** @brief Emulates an immediate-controlled high-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each high four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_hi_slow(auto lhs, auto rhs) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), false); + return register_shuffle_half_16_slow(lhs, static_cast(rhs), true); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_hi(auto lhs, auto rhs) noexcept + /** @brief Shuffles the high four 16-bit lanes in each 128-bit group with an immediate control. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_hi(__m128i lhs) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), true); + return _mm_shufflehi_epi16(lhs, imm8); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept + { + return register_blend_slow(lhs, rhs, static_cast(imm8)); + } + /** @brief Selects signed 16-bit lanes from two registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m128i lhs, const __m128i rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm_blend_epi16(lhs, rhs, imm8); } }; template <> struct SimdImpl128 { + /** @brief Selects 16-bit lanes from two registers using a canonical predicate register. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128i condition, __m128i when_true, __m128i when_false) noexcept + { + return _mm_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical unsigned 16-bit lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 8 && ((indices < 8) && ...)) + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128i lhs) noexcept + { + return _mm_shuffle_epi8(lhs, make_logical_shuffle_16_control(std::make_index_sequence<16>{})); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m128i lhsLo = _mm_cvtepu16_epi32(lhs); const __m128i rhsLo = _mm_cvtepu16_epi32(rhs); @@ -749,31 +1327,70 @@ template <> struct SimdImpl128 const __m128i rhsHi = _mm_cvtepu16_epi32(_mm_srli_si128(rhs, 8)); return _mm_hadd_epi32(_mm_mullo_epi32(lhsLo, rhsLo), _mm_mullo_epi32(lhsHi, rhsHi)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes the unchecked group magnitude in lane zero; all other lanes are unspecified. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i lowProducts = _mm_mullo_epi16(lhs, lhs); + const __m128i highProducts = _mm_mulhi_epu16(lhs, lhs); + const __m128i lowSquares = _mm_unpacklo_epi16(lowProducts, highProducts); + const __m128i highSquares = _mm_unpackhi_epi16(lowProducts, highProducts); + __m128i total = _mm_add_epi32(lowSquares, highSquares); + total = _mm_hadd_epi32(total, total); + total = _mm_hadd_epi32(total, total); + const std::uint32_t squareSum = static_cast(_mm_cvtsi128_si32(total)); + const __m128d root = _mm_sqrt_sd(_mm_setzero_pd(), _mm_set_sd(static_cast(squareSum))); + return _mm_cvtsi32_si128(static_cast(_mm_cvtsd_si32(root))); + } + + /** @brief Computes a saturated magnitude in lane zero and a canonical overflow mask in lane one. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + constexpr std::uint64_t maximum = static_cast(std::numeric_limits::max()); + constexpr std::uint64_t threshold = maximum * maximum + maximum + 1; + const __m128i lowProducts = _mm_mullo_epi16(lhs, lhs); + const __m128i highProducts = _mm_mulhi_epu16(lhs, lhs); + const __m128i lowSquares = _mm_unpacklo_epi16(lowProducts, highProducts); + const __m128i highSquares = _mm_unpackhi_epi16(lowProducts, highProducts); + const __m128i lowPairs = _mm_add_epi64(_mm_cvtepu32_epi64(lowSquares), _mm_cvtepu32_epi64(_mm_srli_si128(lowSquares, 8))); + const __m128i highPairs = _mm_add_epi64(_mm_cvtepu32_epi64(highSquares), _mm_cvtepu32_epi64(_mm_srli_si128(highSquares, 8))); + const __m128i pairTotals = _mm_add_epi64(lowPairs, highPairs); + const __m128i totalVector = _mm_add_epi64(pairTotals, _mm_srli_si128(pairTotals, 8)); + const std::uint64_t total = static_cast(_mm_cvtsi128_si64(totalVector)); + const bool overflow = total >= threshold; + const std::uint64_t result = overflow ? maximum : magnitude_round_sqrt_u64(total, maximum); + return magnitude_checked_result(result, overflow); + } + + /** @brief Computes minimum-value position metadata for this native register specialization. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { return _mm_minpos_epu16(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm_mullo_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding unsigned 16-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext128_div_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes corresponding unsigned 16-bit remainders with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext128_rem_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { const __m128i lo32 = _mm_cvtepu16_epi32(lhs); const __m128i hi32 = _mm_cvtepu16_epi32(_mm_srli_si128(lhs, 8)); @@ -781,78 +1398,96 @@ template <> struct SimdImpl128 const __m128i hiRoots = _mm_cvtps_epi32(_mm_sqrt_ps(_ext_cvtepu32_ps(hi32))); return _mm_packus_epi32(loRoots, hiRoots); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m128i lhs, __m128i rhs) noexcept { return _mm_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm_abs_epi16(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm_sub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm_min_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm_max_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL avg(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise averages for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) avg(auto lhs, auto rhs) noexcept { return _mm_avg_epu16(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm_slli_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm_srli_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _mm_srai_epi16(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm_hadd_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm_hsub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL hadd_saturated(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds unsigned 16-bit lanes with unsigned saturation. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hadd_saturated(auto lhs, auto rhs) noexcept { - return _mm_hadds_epi16(lhs, rhs); + const __m128i zero = _mm_setzero_si128(); + const __m128i lhsPairs = _mm_adds_epu16(lhs, _mm_srli_epi32(lhs, 16)); + const __m128i rhsPairs = _mm_adds_epu16(rhs, _mm_srli_epi32(rhs, 16)); + return _mm_packus_epi32(_mm_blend_epi16(lhsPairs, zero, 0xAA), _mm_blend_epi16(rhsPairs, zero, 0xAA)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL hsubtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts unsigned 16-bit lanes with unsigned saturation. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hsubtract_saturated(auto lhs, auto rhs) noexcept { - return _mm_hsubs_epi16(lhs, rhs); + const __m128i zero = _mm_setzero_si128(); + const __m128i lhsPairs = _mm_subs_epu16(lhs, _mm_srli_epi32(lhs, 16)); + const __m128i rhsPairs = _mm_subs_epu16(rhs, _mm_srli_epi32(rhs, 16)); + return _mm_packus_epi32(_mm_blend_epi16(lhsPairs, zero, 0xAA), _mm_blend_epi16(rhsPairs, zero, 0xAA)); } // arithmetic (saturated) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_saturated(auto lhs, auto rhs) noexcept + /** @brief Adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(auto lhs, auto rhs) noexcept { return _mm_adds_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(auto lhs, auto rhs) noexcept { return _mm_subs_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_saturated(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) multiply_saturated(auto lhs, auto rhs) noexcept { const __m128i lhsLo = _mm_cvtepu16_epi32(lhs); const __m128i rhsLo = _mm_cvtepu16_epi32(rhs); @@ -862,35 +1497,35 @@ template <> struct SimdImpl128 } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm_set1_epi16(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args &&...args) noexcept { return _mm_set_epi16(static_cast(args)...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args &&...args) noexcept { return _mm_setr_epi16(static_cast(args)...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext_cmpgt_epu16(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm_cvtepu16_epi32(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static typename target_simd::vector_t VECTORCALL widen(auto lhs) noexcept + template static typename target_simd::vector_t SIMD_FLAGS(InOut, ForceInline) widen(auto lhs) noexcept { using target_element_t = typename target_simd::element_type; if constexpr (target_simd::register_width == 128) @@ -918,89 +1553,222 @@ template <> struct SimdImpl128 static_assert(dependent_false_v, "No direct widen mapping exists for SimdImpl128 and the requested destination SIMD shape."); } } - SIMDLIB_FORCE_INLINE static auto VECTORCALL compress(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) compress(auto lhs, auto rhs) noexcept { return _mm_packus_epi16(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm_extract_epi16(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected unsigned 16-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 8)`. + * @return Selected scalar lane. + */ + static uint16_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "Unsigned 16-bit extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + case 2: + return extract<2>(lhs); + case 3: + return extract<3>(lhs); + case 4: + return extract<4>(lhs); + case 5: + return extract<5>(lhs); + case 6: + return extract<6>(lhs); + case 7: + return extract<7>(lhs); + default: + return extract<0>(lhs); + } + } + /** @brief Replaces the compile-time-selected unsigned 16-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const uint16_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** @brief Replaces the compile-time-selected unsigned 16-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const uint16_t rhs) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + return _mm_insert_epi16(lhs, static_cast(rhs), index); + } + /** + * @brief Replaces one runtime-selected unsigned 16-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 8)`. + * @return Register with the selected lane replaced. + */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128i lhs, const uint16_t rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "Unsigned 16-bit insertion requires a valid 128-bit lane index"); + const __m128i lane_indices = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7); + const __m128i selected_lane = _mm_cmpeq_epi16(lane_indices, _mm_set1_epi16(static_cast(index))); + return _mm_blendv_epi8(lhs, _mm_set1_epi16(std::bit_cast(rhs)), selected_lane); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm_unpacklo_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_epi16(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_lo(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled low-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each low four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_lo_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_half_16_slow(lhs, static_cast(rhs), false); + } + /** @brief Shuffles the low four unsigned 16-bit lanes in each 128-bit group with an immediate control. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_lo(__m128i lhs) noexcept + { + return _mm_shufflelo_epi16(lhs, imm8); + } + /** @brief Emulates an immediate-controlled high-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each high four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_hi_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_half_16_slow(lhs, static_cast(rhs), true); + } + /** @brief Shuffles the high four unsigned 16-bit lanes in each 128-bit group with an immediate control. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_hi(__m128i lhs) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), false); + return _mm_shufflehi_epi16(lhs, imm8); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_hi(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), true); + return register_blend_slow(lhs, rhs, static_cast(imm8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects unsigned 16-bit lanes from two registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m128i lhs, const __m128i rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm_blend_epi16(lhs, rhs, imm8); } }; template <> struct SimdImpl128 { + /** @brief Selects 32-bit lanes from two registers using a canonical predicate register. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128i condition, __m128i when_true, __m128i when_false) noexcept + { + return _mm_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical signed 32-bit lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 4 && ((indices < 4) && ...)) + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128i lhs) noexcept + { + return _mm_shuffle_epi32(lhs, encode_logical_shuffle_32_immediate()); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m128i evenProducts = _mm_mul_epi32(lhs, rhs); const __m128i oddProducts = _mm_mul_epi32(_mm_srli_si128(lhs, 4), _mm_srli_si128(rhs, 4)); return _mm_add_epi64(evenProducts, oddProducts); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm_mullo_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding signed 32-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return _ext_div_epi32(lhs, rhs); + return _ext128_div_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes corresponding signed 32-bit remainders with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext128_rem_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { const __m128 roots = _mm_sqrt_ps(_mm_cvtepi32_ps(lhs)); return _mm_cvtps_epi32(roots); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes the unchecked group magnitude in lane zero; all other lanes are unspecified. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i pairSums = multiply_add_adjacent(lhs, lhs); + const __m128i totalVector = _mm_add_epi64(pairSums, _mm_srli_si128(pairSums, 8)); + const std::int64_t total = _mm_cvtsi128_si64(totalVector); + const __m128d root = _mm_sqrt_sd(_mm_setzero_pd(), _mm_cvtsi64_sd(_mm_setzero_pd(), total)); + return _mm_cvtsi64_si128(_mm_cvtsd_si64(root)); + } + + /** @brief Computes a saturated magnitude in lane zero and a canonical overflow mask in lane one. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + constexpr std::uint64_t maximum = static_cast(std::numeric_limits::max()); + constexpr std::uint64_t threshold = maximum * maximum + maximum + 1; + const __m128i minimum = _mm_set1_epi32(std::numeric_limits::min()); + if (_mm_movemask_epi8(_mm_cmpeq_epi32(lhs, minimum)) != 0) + return magnitude_checked_result(maximum, true); + const __m128i evenSquares = _mm_mul_epi32(lhs, lhs); + const __m128i shifted = _mm_srli_si128(lhs, 4); + const __m128i oddSquares = _mm_mul_epi32(shifted, shifted); + const __m128i pairTotals = _mm_add_epi64(evenSquares, oddSquares); + const __m128i totalVector = _mm_add_epi64(pairTotals, _mm_srli_si128(pairTotals, 8)); + const std::uint64_t total = static_cast(_mm_cvtsi128_si64(totalVector)); + const bool overflow = total >= threshold; + const std::uint64_t result = overflow ? maximum : magnitude_round_sqrt_u64(total, maximum); + return magnitude_checked_result(result, overflow); + } + + /** @brief Computes minimum-value position metadata for this native register specialization. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { constexpr __m128i indices = register_from_values<__m128i, std::int32_t>(0, 1, 2, 3); __m128i values = lhs; @@ -1015,91 +1783,95 @@ template <> struct SimdImpl128 const __m128i less2 = _mm_cmpgt_epi32(values, shifted2Values); values = _mm_blendv_epi8(values, shifted2Values, less2); positions = _mm_blendv_epi8(positions, shifted2Indices, less2); - alignas(16) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(output.data()), values); - output[1] = _mm_extract_epi32(positions, 0); - return _mm_load_si128(reinterpret_cast(output.data())); + return _mm_insert_epi32(values, _mm_extract_epi32(positions, 0), 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m128i lhs, __m128i rhs) noexcept { return _mm_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm_abs_epi32(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm_sub_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm_min_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm_max_epi32(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm_slli_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm_srli_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _mm_srai_epi32(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm_hadd_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm_hsub_epi32(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm_set1_epi32(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args &&...args) noexcept { return _mm_set_epi32(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args &&...args) noexcept { return _mm_setr_epi32(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm_cmpgt_epi32(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm_cvtepi32_epi64(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static typename target_simd::vector_t VECTORCALL widen(auto lhs) noexcept + template static typename target_simd::vector_t SIMD_FLAGS(InOut, ForceInline) widen(auto lhs) noexcept { using target_element_t = typename target_simd::element_type; if constexpr (sizeof(target_element_t) == sizeof(int64_t)) @@ -1117,54 +1889,134 @@ template <> struct SimdImpl128 static_assert(dependent_false_v, "No direct widen mapping exists for SimdImpl128 and the requested destination SIMD shape."); } } - SIMDLIB_FORCE_INLINE static auto VECTORCALL compress(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) compress(auto lhs, auto rhs) noexcept { return _mm_packs_epi32(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm_extract_epi32(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected signed 32-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 4)`. + * @return Selected scalar lane. + */ + static int32_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "Signed 32-bit extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + case 2: + return extract<2>(lhs); + case 3: + return extract<3>(lhs); + default: + return extract<0>(lhs); + } + } + /** @brief Replaces the compile-time-selected signed 32-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const int32_t rhs) noexcept + { + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected signed 32-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const int32_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return _mm_insert_epi32(lhs, rhs, index); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** + * @brief Replaces one runtime-selected signed 32-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 4)`. + * @return Register with the selected lane replaced. + */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128i lhs, const int32_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "Signed 32-bit insertion requires a valid 128-bit lane index"); + const __m128i lane_indices = _mm_setr_epi32(0, 1, 2, 3); + const __m128i selected_lane = _mm_cmpeq_epi32(lane_indices, _mm_set1_epi32(index)); + return _mm_blendv_epi8(lhs, _mm_set1_epi32(rhs), selected_lane); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm_unpacklo_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_epi32(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_lo(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled low-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each low four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_lo_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_half_16_slow(lhs, static_cast(rhs), false); + } + /** @brief Emulates an immediate-controlled high-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each high four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_hi_slow(auto lhs, auto rhs) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), false); + return register_shuffle_half_16_slow(lhs, static_cast(rhs), true); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_hi(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), true); + return register_blend_slow(lhs, rhs, static_cast(imm8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects signed 32-bit lanes from two registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m128i lhs, const __m128i rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm_castps_si128(_mm_blend_ps(_mm_castsi128_ps(lhs), _mm_castsi128_ps(rhs), imm8 & 0x0F)); } }; template <> struct SimdImpl128 { + /** @brief Selects 32-bit lanes from two registers using a canonical predicate register. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128i condition, __m128i when_true, __m128i when_false) noexcept + { + return _mm_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical unsigned 32-bit lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 4 && ((indices < 4) && ...)) + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128i lhs) noexcept + { + return _mm_shuffle_epi32(lhs, encode_logical_shuffle_32_immediate()); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_epi32(lhs, rhs); } @@ -1174,25 +2026,27 @@ template <> struct SimdImpl128 * @param lhs The unsigned integer lanes. * @return The converted floating-point lanes. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL convert_to_float(const __m128i lhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) convert_to_float(const __m128i lhs) noexcept { return _ext_cvtepu32_ps(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m128i evenProducts = _mm_mul_epu32(lhs, rhs); const __m128i oddProducts = _mm_mul_epu32(_mm_srli_si128(lhs, 4), _mm_srli_si128(rhs, 4)); return _mm_add_epi64(evenProducts, oddProducts); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm_mullo_epi32(lhs, rhs); } @@ -1203,20 +2057,52 @@ template <> struct SimdImpl128 * @param rhs The nonzero divisor lanes. * @return The truncating integer quotients. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext128_div_epu32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes corresponding unsigned 32-bit remainders with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext128_rem_epu32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { const __m128 roots = _mm_sqrt_ps(_ext_cvtepu32_ps(lhs)); return _mm_cvtps_epi32(roots); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes the unchecked group magnitude in lane zero; all other lanes are unspecified. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i pairSums = multiply_add_adjacent(lhs, lhs); + const __m128i totalVector = _mm_add_epi64(pairSums, _mm_srli_si128(pairSums, 8)); + const std::uint64_t total = static_cast(_mm_cvtsi128_si64(totalVector)); + const __m128d root = _mm_sqrt_sd(_mm_setzero_pd(), _mm_set_sd(static_cast(total))); + return _mm_cvtsi64_si128(_mm_cvtsd_si64(root)); + } + + /** @brief Computes a saturated magnitude in lane zero and a canonical overflow mask in lane one. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + constexpr std::uint64_t maximum = static_cast(std::numeric_limits::max()); + constexpr std::uint64_t threshold = maximum * maximum + maximum + 1; + const __m128i evenSquares = _mm_mul_epu32(lhs, lhs); + const __m128i shifted = _mm_srli_si128(lhs, 4); + const __m128i oddSquares = _mm_mul_epu32(shifted, shifted); + const __m128i pairTotals = _mm_add_epi64(evenSquares, oddSquares); + const __m128i signBit = _mm_set1_epi64x(std::numeric_limits::min()); + const __m128i pairCarries = _mm_cmpgt_epi64(_mm_xor_si128(evenSquares, signBit), _mm_xor_si128(pairTotals, signBit)); + const std::uint64_t lowTotal = static_cast(_mm_cvtsi128_si64(pairTotals)); + const std::uint64_t highTotal = static_cast(_mm_extract_epi64(pairTotals, 1)); + const std::uint64_t total = lowTotal + highTotal; + const bool overflow = _mm_movemask_pd(_mm_castsi128_pd(pairCarries)) != 0 || total < lowTotal || total >= threshold; + const std::uint64_t result = overflow ? maximum : magnitude_round_sqrt_u64(total, maximum); + return magnitude_checked_result(result, overflow); + } + + /** @brief Computes minimum-value position metadata for this native register specialization. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { constexpr __m128i indices = register_from_values<__m128i, std::uint32_t>(0u, 1u, 2u, 3u); const __m128i signBit = _mm_set1_epi32(static_cast(0x80000000u)); @@ -1232,91 +2118,95 @@ template <> struct SimdImpl128 const __m128i less2 = _mm_cmpgt_epi32(_mm_xor_si128(values, signBit), _mm_xor_si128(shifted2Values, signBit)); values = _mm_blendv_epi8(values, shifted2Values, less2); positions = _mm_blendv_epi8(positions, shifted2Indices, less2); - alignas(16) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(output.data()), values); - output[1] = static_cast(_mm_extract_epi32(positions, 0)); - return _mm_load_si128(reinterpret_cast(output.data())); + return _mm_insert_epi32(values, _mm_extract_epi32(positions, 0), 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m128i lhs, __m128i rhs) noexcept { return _mm_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm_abs_epi32(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm_sub_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm_min_epu32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm_max_epu32(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm_slli_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm_srli_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _mm_srai_epi32(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm_hadd_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm_hsub_epi32(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm_set1_epi32(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm_set_epi32(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm_setr_epi32(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext_cmpgt_epu32(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm_cvtepu32_epi64(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static typename target_simd::vector_t VECTORCALL widen(auto lhs) noexcept + template static typename target_simd::vector_t SIMD_FLAGS(InOut, ForceInline) widen(auto lhs) noexcept { using target_element_t = typename target_simd::element_type; if constexpr (sizeof(target_element_t) == sizeof(uint64_t)) @@ -1334,58 +2224,139 @@ template <> struct SimdImpl128 static_assert(dependent_false_v, "No direct widen mapping exists for SimdImpl128 and the requested destination SIMD shape."); } } - SIMDLIB_FORCE_INLINE static auto VECTORCALL compress(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) compress(auto lhs, auto rhs) noexcept { return _mm_packus_epi32(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm_extract_epi32(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected unsigned 32-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 4)`. + * @return Selected scalar lane. + */ + static uint32_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "Unsigned 32-bit extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + case 2: + return extract<2>(lhs); + case 3: + return extract<3>(lhs); + default: + return extract<0>(lhs); + } + } + /** @brief Replaces the compile-time-selected unsigned 32-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const uint32_t rhs) noexcept + { + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected unsigned 32-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const uint32_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return _mm_insert_epi32(lhs, std::bit_cast(rhs), index); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** + * @brief Replaces one runtime-selected unsigned 32-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 4)`. + * @return Register with the selected lane replaced. + */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128i lhs, const uint32_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "Unsigned 32-bit insertion requires a valid 128-bit lane index"); + const __m128i lane_indices = _mm_setr_epi32(0, 1, 2, 3); + const __m128i selected_lane = _mm_cmpeq_epi32(lane_indices, _mm_set1_epi32(index)); + return _mm_blendv_epi8(lhs, _mm_set1_epi32(std::bit_cast(rhs)), selected_lane); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm_unpacklo_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_epi32(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_lo(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled low-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each low four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_lo_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_half_16_slow(lhs, static_cast(rhs), false); + } + /** @brief Emulates an immediate-controlled high-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each high four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_hi_slow(auto lhs, auto rhs) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), false); + return register_shuffle_half_16_slow(lhs, static_cast(rhs), true); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_hi(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), true); + return register_blend_slow(lhs, rhs, static_cast(imm8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects unsigned 32-bit lanes from two registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m128i lhs, const __m128i rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm_castps_si128(_mm_blend_ps(_mm_castsi128_ps(lhs), _mm_castsi128_ps(rhs), imm8 & 0x0F)); } }; template <> struct SimdImpl128 { + /** @brief Selects 64-bit lanes from two registers using a canonical predicate register. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128i condition, __m128i when_true, __m128i when_false) noexcept + { + return _mm_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical signed 64-bit lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 2 && ((indices < 2) && ...)) + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128i lhs) noexcept + { + return _mm_shuffle_epi32(lhs, encode_logical_shuffle_64_immediate()); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m128i productLow = _mm_mul_epu32(lhs, rhs); const __m128i lhsHigh = _mm_srli_epi64(lhs, 32); @@ -1393,39 +2364,91 @@ template <> struct SimdImpl128 const __m128i cross = _mm_add_epi64(_mm_mul_epu32(lhsHigh, rhs), _mm_mul_epu32(lhs, rhsHigh)); const __m128i products = _mm_add_epi64(productLow, _mm_slli_epi64(cross, 32)); const __m128i shifted = _mm_bsrli_si128(products, 8); - alignas(16) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(output.data()), _mm_add_epi64(products, shifted)); - output[1] = 0; - return _mm_load_si128(reinterpret_cast(output.data())); + const __m128i sum = _mm_add_epi64(products, shifted); + return _mm_unpacklo_epi64(sum, _mm_setzero_si128()); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _ext_mullo_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding signed 64-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return _ext_div_epi64(lhs, rhs); + return _ext128_div_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes corresponding signed 64-bit remainders with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return _ext_rem_epi64(lhs, rhs); + return _ext128_rem_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { const __m128d roots = _mm_sqrt_pd(_mm_setr_pd(static_cast(_mm_cvtsi128_si64(lhs)), static_cast(_mm_extract_epi64(lhs, 1)))); - alignas(16) double values[2]; - _mm_storeu_pd(values, roots); - return register_from_values<__m128i, std::int64_t>(static_cast(values[0]), static_cast(values[1])); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + const auto lowRoot = static_cast(_mm_cvtsd_f64(roots)); + const auto highRoot = static_cast(_mm_cvtsd_f64(_mm_unpackhi_pd(roots, roots))); + return _mm_set_epi64x(highRoot, lowRoot); + } + /** @brief Computes the unchecked group magnitude in lane zero; lane one is unspecified. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const std::uint64_t rawLow = static_cast(_mm_cvtsi128_si64(lhs)); + const std::uint64_t rawHigh = static_cast(_mm_extract_epi64(lhs, 1)); + const std::uint64_t lowSign = rawLow >> 63; + const std::uint64_t highSign = rawHigh >> 63; + const std::uint64_t lowValue = (rawLow ^ (std::uint64_t{0} - lowSign)) + lowSign; + const std::uint64_t highValue = (rawHigh ^ (std::uint64_t{0} - highSign)) + highSign; + const __m128i lowSquare = magnitude_square_u64(lowValue); + const __m128i highSquare = magnitude_square_u64(highValue); + const std::uint64_t lowWord0 = static_cast(_mm_cvtsi128_si64(lowSquare)); + const std::uint64_t lowWord1 = static_cast(_mm_cvtsi128_si64(highSquare)); + const std::uint64_t highWord0 = static_cast(_mm_extract_epi64(lowSquare, 1)); + const std::uint64_t highWord1 = static_cast(_mm_extract_epi64(highSquare, 1)); + const std::uint64_t lowWord = lowWord0 + lowWord1; + const std::uint64_t highWord = highWord0 + highWord1 + static_cast(lowWord < lowWord0); + const std::uint64_t result = magnitude_round_sqrt_u128(lowWord, highWord, static_cast(std::numeric_limits::max())); + return _mm_cvtsi64_si128(static_cast(result)); + } + + /** @brief Computes a saturated magnitude in lane zero and a canonical overflow mask in lane one. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + constexpr std::uint64_t maximum = static_cast(std::numeric_limits::max()); + constexpr std::uint64_t thresholdLow = 0x8000'0000'0000'0001ULL; + constexpr std::uint64_t thresholdHigh = 0x3FFF'FFFF'FFFF'FFFFULL; + const std::uint64_t rawLow = static_cast(_mm_cvtsi128_si64(lhs)); + const std::uint64_t rawHigh = static_cast(_mm_extract_epi64(lhs, 1)); + const std::uint64_t lowSign = rawLow >> 63; + const std::uint64_t highSign = rawHigh >> 63; + const std::uint64_t lowValue = (rawLow ^ (std::uint64_t{0} - lowSign)) + lowSign; + const std::uint64_t highValue = (rawHigh ^ (std::uint64_t{0} - highSign)) + highSign; + const __m128i lowSquare = magnitude_square_u64(lowValue); + const __m128i highSquare = magnitude_square_u64(highValue); + const std::uint64_t lowWord0 = static_cast(_mm_cvtsi128_si64(lowSquare)); + const std::uint64_t lowWord1 = static_cast(_mm_cvtsi128_si64(highSquare)); + const std::uint64_t highWord0 = static_cast(_mm_extract_epi64(lowSquare, 1)); + const std::uint64_t highWord1 = static_cast(_mm_extract_epi64(highSquare, 1)); + const std::uint64_t lowWord = lowWord0 + lowWord1; + const std::uint64_t carry = static_cast(lowWord < lowWord0); + const std::uint64_t highPartial = highWord0 + highWord1; + const bool highOverflow = highPartial < highWord0 || highPartial + carry < highPartial; + const std::uint64_t highWord = highPartial + carry; + const bool overflow = highOverflow || highWord > thresholdHigh || (highWord == thresholdHigh && lowWord >= thresholdLow); + const std::uint64_t result = overflow ? maximum : magnitude_round_sqrt_u128(lowWord, highWord, maximum); + return magnitude_checked_result(result, overflow); + } + + /** @brief Computes minimum-value position metadata for this native register specialization. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { constexpr __m128i indices = register_from_values<__m128i, std::int64_t>(0, 1); const __m128i shiftedValues = _mm_bsrli_si128(lhs, 8); @@ -1433,95 +2456,138 @@ template <> struct SimdImpl128 const __m128i less = _mm_cmpgt_epi64(lhs, shiftedValues); const __m128i values = _mm_blendv_epi8(lhs, shiftedValues, less); const __m128i positions = _mm_blendv_epi8(indices, shiftedIndices, less); - alignas(16) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(output.data()), values); - output[1] = _mm_extract_epi64(positions, 0); - return _mm_load_si128(reinterpret_cast(output.data())); + return _mm_insert_epi64(values, _mm_extract_epi64(positions, 0), 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m128i lhs, __m128i rhs) noexcept { return _mm_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _ext_abs_epi64(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm_sub_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _ext_min_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _ext_max_epi64(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm_slli_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm_srli_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _ext_srai_epi64(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm_set1_epi64x(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm_set_epi64x(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + /** + * @brief Constructs two signed 64-bit lanes in low-to-high logical order. + * @param low Value for lane zero. + * @param high Value for lane one. + * @return Register containing low followed by high. + */ + static __m128i SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(const std::int64_t low, const std::int64_t high) noexcept { - return register_from_values<__m128i, std::int64_t>(args...); + return _mm_set_epi64x(high, low); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm_cmpgt_epi64(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm_extract_epi64(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected signed 64-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 2)`. + * @return Selected scalar lane. + */ + static int64_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 2, "Signed 64-bit extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + default: + return extract<0>(lhs); + } + } + /** @brief Replaces the compile-time-selected signed 64-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const int64_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, int index) noexcept + /** @brief Replaces the compile-time-selected signed 64-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const int64_t rhs) noexcept + { + return _mm_insert_epi64(lhs, rhs, index); + } + /** + * @brief Replaces one runtime-selected signed 64-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 2)`. + * @return Register with the selected lane replaced. + */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128i lhs, const int64_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + SIMDLIB_PRECONDITION(index >= 0 && index < 2, "Signed 64-bit insertion requires a valid 128-bit lane index"); + const __m128i lane_indices = _mm_set_epi64x(1, 0); + const __m128i selected_lane = _mm_cmpeq_epi64(lane_indices, _mm_set1_epi64x(index)); + return _mm_blendv_epi8(lhs, _mm_set1_epi64x(rhs), selected_lane); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm_unpacklo_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_epi64(lhs, rhs); } @@ -1529,12 +2595,31 @@ template <> struct SimdImpl128 template <> struct SimdImpl128 { + /** @brief Selects 64-bit lanes from two registers using a canonical predicate register. */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128i condition, __m128i when_true, __m128i when_false) noexcept + { + return _mm_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical unsigned 64-bit lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 2 && ((indices < 2) && ...)) + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128i lhs) noexcept + { + return _mm_shuffle_epi32(lhs, encode_logical_shuffle_64_immediate()); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m128i productLow = _mm_mul_epu32(lhs, rhs); const __m128i lhsHigh = _mm_srli_epi64(lhs, 32); @@ -1542,40 +2627,84 @@ template <> struct SimdImpl128 const __m128i cross = _mm_add_epi64(_mm_mul_epu32(lhsHigh, rhs), _mm_mul_epu32(lhs, rhsHigh)); const __m128i products = _mm_add_epi64(productLow, _mm_slli_epi64(cross, 32)); const __m128i shifted = _mm_bsrli_si128(products, 8); - alignas(16) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(output.data()), _mm_add_epi64(products, shifted)); - output[1] = 0; - return _mm_load_si128(reinterpret_cast(output.data())); + const __m128i sum = _mm_add_epi64(products, shifted); + return _mm_unpacklo_epi64(sum, _mm_setzero_si128()); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _ext_mullo_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding unsigned 64-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return _ext_div_epu64(lhs, rhs); + return _ext128_div_epu64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes corresponding unsigned 64-bit remainders with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return _ext_rem_epu64(lhs, rhs); + return _ext128_rem_epu64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { const __m128d roots = _mm_sqrt_pd(_mm_setr_pd(static_cast(static_cast(_mm_cvtsi128_si64(lhs))), static_cast(static_cast(_mm_extract_epi64(lhs, 1))))); - alignas(16) double values[2]; - _mm_storeu_pd(values, roots); - return register_from_values<__m128i, std::int64_t>(static_cast(values[0]), static_cast(values[1])); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + const auto lowRoot = static_cast(_mm_cvtsd_f64(roots)); + const auto highRoot = static_cast(_mm_cvtsd_f64(_mm_unpackhi_pd(roots, roots))); + return _mm_set_epi64x(highRoot, lowRoot); + } + /** @brief Computes the unchecked group magnitude in lane zero; lane one is unspecified. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const std::uint64_t lowValue = static_cast(_mm_cvtsi128_si64(lhs)); + const std::uint64_t highValue = static_cast(_mm_extract_epi64(lhs, 1)); + const __m128i lowSquare = magnitude_square_u64(lowValue); + const __m128i highSquare = magnitude_square_u64(highValue); + const std::uint64_t lowWord0 = static_cast(_mm_cvtsi128_si64(lowSquare)); + const std::uint64_t lowWord1 = static_cast(_mm_cvtsi128_si64(highSquare)); + const std::uint64_t highWord0 = static_cast(_mm_extract_epi64(lowSquare, 1)); + const std::uint64_t highWord1 = static_cast(_mm_extract_epi64(highSquare, 1)); + const std::uint64_t lowWord = lowWord0 + lowWord1; + const std::uint64_t highWord = highWord0 + highWord1 + static_cast(lowWord < lowWord0); + const std::uint64_t result = magnitude_round_sqrt_u128(lowWord, highWord, std::numeric_limits::max()); + return _mm_cvtsi64_si128(static_cast(result)); + } + + /** @brief Computes a saturated magnitude in lane zero and a canonical overflow mask in lane one. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + constexpr std::uint64_t maximum = std::numeric_limits::max(); + constexpr std::uint64_t thresholdLow = 1; + constexpr std::uint64_t thresholdHigh = maximum; + const std::uint64_t lowValue = static_cast(_mm_cvtsi128_si64(lhs)); + const std::uint64_t highValue = static_cast(_mm_extract_epi64(lhs, 1)); + const __m128i lowSquare = magnitude_square_u64(lowValue); + const __m128i highSquare = magnitude_square_u64(highValue); + const std::uint64_t lowWord0 = static_cast(_mm_cvtsi128_si64(lowSquare)); + const std::uint64_t lowWord1 = static_cast(_mm_cvtsi128_si64(highSquare)); + const std::uint64_t highWord0 = static_cast(_mm_extract_epi64(lowSquare, 1)); + const std::uint64_t highWord1 = static_cast(_mm_extract_epi64(highSquare, 1)); + const std::uint64_t lowWord = lowWord0 + lowWord1; + const std::uint64_t carry = static_cast(lowWord < lowWord0); + const std::uint64_t highPartial = highWord0 + highWord1; + const bool highOverflow = highPartial < highWord0 || highPartial + carry < highPartial; + const std::uint64_t highWord = highPartial + carry; + const bool overflow = highOverflow || highWord > thresholdHigh || (highWord == thresholdHigh && lowWord >= thresholdLow); + const std::uint64_t result = overflow ? maximum : magnitude_round_sqrt_u128(lowWord, highWord, maximum); + return magnitude_checked_result(result, overflow); + } + + /** @brief Computes minimum-value position metadata for this native register specialization. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { constexpr __m128i indices = register_from_values<__m128i, std::uint64_t>(0ull, 1ull); const __m128i signBit = _mm_set1_epi64x(std::numeric_limits::min()); @@ -1584,95 +2713,138 @@ template <> struct SimdImpl128 const __m128i less = _mm_cmpgt_epi64(_mm_xor_si128(lhs, signBit), _mm_xor_si128(shiftedValues, signBit)); const __m128i values = _mm_blendv_epi8(lhs, shiftedValues, less); const __m128i positions = _mm_blendv_epi8(indices, shiftedIndices, less); - alignas(16) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(output.data()), values); - output[1] = static_cast(_mm_extract_epi64(positions, 0)); - return _mm_load_si128(reinterpret_cast(output.data())); + return _mm_insert_epi64(values, _mm_extract_epi64(positions, 0), 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m128i lhs, __m128i rhs) noexcept { return _mm_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return lhs; } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm_sub_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _ext_min_epu64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _ext_max_epu64(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm_slli_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm_srli_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _ext_srai_epi64(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm_set1_epi64x(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args &&...args) noexcept { return _mm_set_epi64x(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args &&...args) noexcept + /** + * @brief Constructs two unsigned 64-bit lanes in low-to-high logical order. + * @param low Value for lane zero. + * @param high Value for lane one. + * @return Register containing the exact low and high lane bit patterns. + */ + static __m128i SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(const std::uint64_t low, const std::uint64_t high) noexcept { - return register_from_values<__m128i, std::int64_t>(args...); + return _mm_set_epi64x(std::bit_cast(high), std::bit_cast(low)); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext_cmpgt_epu64(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept + { + return static_cast(_mm_extract_epi64(lhs, index)); + } + /** + * @brief Extracts one runtime-selected unsigned 64-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 2)`. + * @return Selected scalar lane. + */ + static uint64_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 2, "Unsigned 64-bit extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + default: + return extract<0>(lhs); + } + } + /** @brief Replaces the compile-time-selected unsigned 64-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const uint64_t rhs) noexcept { - return register_get(lhs, static_cast(index)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** @brief Replaces the compile-time-selected unsigned 64-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const uint64_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return _mm_insert_epi64(lhs, std::bit_cast(rhs), index); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, int index) noexcept + /** + * @brief Replaces one runtime-selected unsigned 64-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 2)`. + * @return Register with the selected lane replaced. + */ + static __m128i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128i lhs, const uint64_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + SIMDLIB_PRECONDITION(index >= 0 && index < 2, "Unsigned 64-bit insertion requires a valid 128-bit lane index"); + const __m128i lane_indices = _mm_set_epi64x(1, 0); + const __m128i selected_lane = _mm_cmpeq_epi64(lane_indices, _mm_set1_epi64x(index)); + return _mm_blendv_epi8(lhs, _mm_set1_epi64x(std::bit_cast(rhs)), selected_lane); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm_unpacklo_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_epi64(lhs, rhs); } @@ -1680,32 +2852,58 @@ template <> struct SimdImpl128 template <> struct SimdImpl128 { + /** @brief Selects float lanes from two registers using a canonical predicate register. */ + static __m128 SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128 condition, __m128 when_true, __m128 when_false) noexcept + { + return _mm_blendv_ps(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical floating-point lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 4 && ((indices < 4) && ...)) + static __m128 SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128 lhs) noexcept + { + return _mm_shuffle_ps(lhs, lhs, encode_logical_shuffle_32_immediate()); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_subtract(auto lhs, auto rhs) noexcept + /** @brief Alternates lane subtraction and addition for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_subtract(auto lhs, auto rhs) noexcept { return _mm_addsub_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm_mul_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { return _mm_div_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { return _mm_sqrt_ps(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add(auto lhs, auto rhs, auto addend) noexcept + /** @brief Computes and broadcasts the 128-bit floating-point magnitude. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + return _mm_sqrt_ps(_mm_dp_ps(lhs, lhs, 0xFF)); + } + /** @brief Multiplies lanes and adds a third register for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add(auto lhs, auto rhs, auto addend) noexcept { #if SIMDLIB_HAS_FMA return _mm_fmadd_ps(lhs, rhs, addend); @@ -1713,100 +2911,175 @@ template <> struct SimdImpl128 return _mm_add_ps(_mm_mul_ps(lhs, rhs), addend); #endif } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL dot_product(auto lhs, auto rhs) noexcept + /** @brief Computes an immediate-controlled dot product for this native register specialization. */ + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) dot_product(__m128 lhs, __m128 rhs) noexcept { return _mm_dp_ps(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _ext_abs_ps(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm_min_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm_max_ps(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm_hadd_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm_hsub_ps(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm_set_ps1(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm_set_ps(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm_setr_ps(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm_cmpgt_ps(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm_cvtps_epi32(lhs, rhs); } - // static SIMDLIB_FORCE_INLINE auto VECTORCALL compress (auto lhs, auto rhs) noexcept { return _mm_cvtepi32_ps(lhs, rhs); } + // static auto SIMD_FLAGS(InOut, ForceInline) compress (auto lhs, auto rhs) noexcept { return _mm_cvtepi32_ps(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { - return static_cast(_mm_extract_ps(lhs, index)); + return _mm_cvtss_f32(_mm_shuffle_ps(lhs, lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected 32-bit floating-point lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 4)`. + * @return Selected scalar lane. + */ + static float SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128 lhs, const int index) noexcept { - return register_get(lhs, static_cast(rhs)); + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "32-bit floating-point extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + case 2: + return extract<2>(lhs); + case 3: + return extract<3>(lhs); + default: + return extract<0>(lhs); + } } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** @brief Replaces the compile-time-selected 32-bit floating-point lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const float rhs) noexcept { - return register_insert_float(lhs, rhs, static_cast(index)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); } - - // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + /** @brief Replaces the compile-time-selected 32-bit floating-point lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const float rhs) noexcept { - return _mm_unpacklo_ps(lhs, rhs); + return _mm_insert_ps(lhs, _mm_set_ss(rhs), index << 4); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + /** + * @brief Replaces one runtime-selected 32-bit floating-point lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane index. + * @return Register with the selected lane replaced. + */ + static __m128 SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128 lhs, const float rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "32-bit floating-point insertion requires a valid 128-bit lane index"); + switch (index) + { + case 1: + return insert<1>(lhs, rhs); + case 2: + return insert<2>(lhs, rhs); + case 3: + return insert<3>(lhs, rhs); + default: + return insert<0>(lhs, rhs); + } + } + + // unpack / pack + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept + { + return _mm_unpacklo_ps(lhs, rhs); + } + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_ps(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(auto lhs, auto rhs, unsigned int imm8) noexcept + /** @brief Emulates an immediate-controlled floating shuffle with a runtime scalar control. + * @param lhs Source for the lower selected lanes in each group. + * @param rhs Source for the upper selected lanes in each group. + * @param imm8 Runtime control byte. + * @return Register containing the shuffled lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) shuffle_slow(auto lhs, auto rhs, unsigned int imm8) noexcept + { + return register_shuffle_float_slow(lhs, rhs, imm8); + } + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_float(lhs, rhs, imm8); + const unsigned int control = static_cast(imm8); + const __m128 mask = _mm_castsi128_ps(_mm_set_epi32(-static_cast((control >> 3) & 0x1u), -static_cast((control >> 2) & 0x1u), + -static_cast((control >> 1) & 0x1u), -static_cast(control & 0x1u))); + return _mm_blendv_ps(lhs, rhs, mask); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects 32-bit floating-point lanes from two registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m128 lhs, const __m128 rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm_blend_ps(lhs, rhs, imm8 & 0x0F); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL movemask(auto lhs) noexcept + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) movemask(auto lhs) noexcept { return _mm_movemask_ps(lhs); } @@ -1814,32 +3087,58 @@ template <> struct SimdImpl128 template <> struct SimdImpl128 { + /** @brief Selects double lanes from two registers using a canonical predicate register. */ + static __m128d SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m128d condition, __m128d when_true, __m128d when_false) noexcept + { + return _mm_blendv_pd(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical double-precision floating-point lanes using compile-time source selectors. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 2 && ((indices < 2) && ...)) + static __m128d SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m128d lhs) noexcept + { + return _mm_shuffle_pd(lhs, lhs, encode_logical_shuffle_double_immediate()); + } // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm_add_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_subtract(auto lhs, auto rhs) noexcept + /** @brief Alternates lane subtraction and addition for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_subtract(auto lhs, auto rhs) noexcept { return _mm_addsub_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm_sub_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm_mul_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { return _mm_div_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { return _mm_sqrt_pd(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add(auto lhs, auto rhs, auto addend) noexcept + /** @brief Computes and broadcasts the 128-bit floating-point magnitude. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + return _mm_sqrt_pd(_mm_dp_pd(lhs, lhs, 0x33)); + } + /** @brief Multiplies lanes and adds a third register for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add(auto lhs, auto rhs, auto addend) noexcept { #if SIMDLIB_HAS_FMA return _mm_fmadd_pd(lhs, rhs, addend); @@ -1847,100 +3146,173 @@ template <> struct SimdImpl128 return _mm_add_pd(_mm_mul_pd(lhs, rhs), addend); #endif } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL dot_product(auto lhs, auto rhs) noexcept + /** @brief Computes an immediate-controlled dot product for this native register specialization. */ + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) dot_product(__m128d lhs, __m128d rhs) noexcept { return _mm_dp_pd(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _ext_abs_pd(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm_min_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm_max_pd(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm_hadd_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm_hsub_pd(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm_set1_pd(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm_set_pd(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm_setr_pd(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm_cmpeq_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm_cmpgt_pd(lhs, rhs); } - // static SIMDLIB_FORCE_INLINE auto VECTORCALL cmplt (auto lhs, auto rhs) noexcept { return _mm_cmplt_pd(lhs, rhs); } + // static auto SIMD_FLAGS(InOut, ForceInline) cmplt (auto lhs, auto rhs) noexcept { return _mm_cmplt_pd(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm_cvtps_epi32(lhs, rhs); } - // static SIMDLIB_FORCE_INLINE auto VECTORCALL compress (auto lhs, auto rhs) noexcept { return _mm_cvtepi32_pd(lhs, rhs); } + // static auto SIMD_FLAGS(InOut, ForceInline) compress (auto lhs, auto rhs) noexcept { return _mm_cvtepi32_pd(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { - return register_get(lhs, static_cast(index)); + if constexpr (index == 0) + return _mm_cvtsd_f64(lhs); + else + return _mm_cvtsd_f64(_mm_unpackhi_pd(lhs, lhs)); + } + /** + * @brief Extracts one runtime-selected 64-bit floating-point lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 2)`. + * @return Selected scalar lane. + */ + static double SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m128d lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 2, "64-bit floating-point extraction requires a valid 128-bit lane index"); + switch (index) + { + case 0: + return extract<0>(lhs); + case 1: + return extract<1>(lhs); + default: + return extract<0>(lhs); + } } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** @brief Replaces the compile-time-selected 64-bit floating-point lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const double rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** @brief Replaces the compile-time-selected 64-bit floating-point lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const double rhs) noexcept { - return register_insert(lhs, register_get(rhs, (static_cast(index) >> 1) & 1u), static_cast(index) & 1u); + const __m128d replacement = _mm_set_sd(rhs); + if constexpr (index == 0) + return _mm_move_sd(lhs, replacement); + else + return _mm_unpacklo_pd(lhs, replacement); + } + /** + * @brief Replaces one runtime-selected 64-bit floating-point lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane index. + * @return Register with the selected lane replaced. + */ + static __m128d SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m128d lhs, const double rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 2, "64-bit floating-point insertion requires a valid 128-bit lane index"); + switch (index) + { + case 1: + return insert<1>(lhs, rhs); + default: + return insert<0>(lhs, rhs); + } } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm_unpacklo_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm_unpackhi_pd(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(auto lhs, auto rhs, unsigned int imm8) noexcept + /** @brief Emulates an immediate-controlled floating shuffle with a runtime scalar control. + * @param lhs Source for the lower selected lanes in each group. + * @param rhs Source for the upper selected lanes in each group. + * @param imm8 Runtime control byte. + * @return Register containing the shuffled lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) shuffle_slow(auto lhs, auto rhs, unsigned int imm8) noexcept + { + return register_shuffle_double_slow(lhs, rhs, imm8); + } + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_double(lhs, rhs, imm8); + const unsigned int control = static_cast(imm8); + const __m128d mask = _mm_castsi128_pd(_mm_set_epi64x(-static_cast((control >> 1) & 0x1u), -static_cast(control & 0x1u))); + return _mm_blendv_pd(lhs, rhs, mask); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects 64-bit floating-point lanes from two registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m128d lhs, const __m128d rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm_blend_pd(lhs, rhs, imm8 & 0x03); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL movemask(auto lhs) noexcept + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) movemask(auto lhs) noexcept { return _mm_movemask_pd(lhs); } @@ -1955,6 +3327,9 @@ template struct SimdMappings<128, element_t> : public SimdImpl using impl = SimdImpl128; public: + using impl::extract; + using impl::shuffle; + template using Mappings = SimdMappings<128, ty>; template using mapped_vector_t = typename Mappings::vector_t; template @@ -1973,21 +3348,8 @@ template struct SimdMappings<128, element_t> : public SimdImpl constexpr static inline std::size_t element_count = register_width / (sizeof(element_t) * 8); constexpr static inline int_vector_t vector0 = register_from_values(0, 0); - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(const vector_t lhs) noexcept - { - static_assert(index >= 0 && static_cast(index) < element_count, "SimdMappings<128>::extract index out of range."); - if constexpr (requires(vector_t value) { impl::template extract(value); }) - { - return impl::template extract(lhs); - } - else - { - return get_element(lhs, index); - } - } - #pragma region Set - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL setzero() noexcept + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) setzero() noexcept { if (std::is_constant_evaluated()) { @@ -2006,11 +3368,11 @@ template struct SimdMappings<128, element_t> : public SimdImpl template ... Args> requires(sizeof...(Args) == element_count) - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL setr(Args &&...args) noexcept + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) setr(Args &&...args) noexcept { if (std::is_constant_evaluated()) { - return register_from_values(static_cast(args)...); + return setr_constexpr(std::forward(args)...); } else { @@ -2018,7 +3380,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl } } - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL construct(const std::array data) noexcept + constexpr static vector_t SIMD_FLAGS(Out, ForceInline, Flatten) construct(const std::array &data) noexcept { if (std::is_constant_evaluated()) { @@ -2030,21 +3392,31 @@ template struct SimdMappings<128, element_t> : public SimdImpl } } - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL set1(const element_t value) noexcept + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) set1(const element_t value) noexcept { if (std::is_constant_evaluated()) - { - std::array lanes{}; - lanes.fill(value); - return register_from_array(lanes); - } + return set1_constexpr(value); else { return impl::set1(value); } } - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL multiply_add(const vector_t lhs, const vector_t rhs, const vector_t addend) noexcept + /** @brief Broadcasts one value through the portable compile-time register representation. */ + constexpr static vector_t set1_constexpr(const element_t value) noexcept + { + return register_from_repeated_value(value); + } + + /** @brief Constructs a register from forward-order lanes during constant evaluation. */ + template ... Args> + requires(sizeof...(Args) == element_count) + constexpr static vector_t setr_constexpr(Args &&...args) noexcept + { + return register_from_values(static_cast(args)...); + } + + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add(const vector_t lhs, const vector_t rhs, const vector_t addend) noexcept { if constexpr (requires(vector_t left, vector_t right, vector_t sum) { impl::multiply_add(left, right, sum); }) return impl::multiply_add(lhs, rhs, addend); @@ -2053,44 +3425,49 @@ template struct SimdMappings<128, element_t> : public SimdImpl } /// Broadcasts a 128-bit integer vector into both 128-bit lanes of a 256-bit integer vector. - SIMDLIB_FORCE_INLINE static __m256i VECTORCALL broadcast_128(const typename SimdMappings<128, element_t>::int_vector_t v) noexcept + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) broadcast_128(const typename SimdMappings<128, element_t>::int_vector_t v) noexcept requires std::is_integral_v { return _mm256_broadcastsi128_si256(v); } - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL set_element(vector_t vec, int index, element_t value) noexcept - { - register_set(vec, static_cast(index), value); - return vec; - } - - SIMDLIB_FORCE_INLINE constexpr static element_t VECTORCALL get_element(vector_t vec, int index) noexcept - { - return register_get(vec, static_cast(index)); - } - - SIMDLIB_FORCE_INLINE static std::span VECTORCALL view_data(vector_t &vec) noexcept + static std::span SIMD_FLAGS(Neither, ForceInline, Flatten) view_data(vector_t &vec) noexcept { return std::span{register_data(vec), element_count}; } - SIMDLIB_FORCE_INLINE static std::span VECTORCALL view_data(const vector_t &vec) noexcept + static std::span SIMD_FLAGS(Neither, ForceInline, Flatten) view_data(const vector_t &vec) noexcept { return std::span{register_data(vec), element_count}; } #pragma endregion #pragma region Load + /** + * @brief Loads a complete 128-bit object representation without an alignment requirement. + * @param ptr Source containing at least 16 accessible bytes. + * @return Native register preserving every source bit. + */ + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_bytes(const void *ptr) noexcept + { + const int_vector_t bits = _mm_loadu_si128(reinterpret_cast(ptr)); + if constexpr (std::is_integral_v) + return bits; + else if constexpr (std::is_same_v) + return _mm_castsi128_ps(bits); + else + return _mm_castsi128_pd(bits); + } + /// Loads a full register from memory. Pointer must be appropriately aligned for the register width. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL load(const element_t *ptr) noexcept + static int_vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load(const element_t *ptr) noexcept requires std::is_integral_v { return _mm_load_si128(reinterpret_cast(ptr)); } /// Loads a full register from memory without requiring alignment. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL load_unaligned(const element_t *ptr) noexcept + static int_vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_unaligned(const element_t *ptr) noexcept requires std::is_integral_v { return _mm_loadu_si128(reinterpret_cast(ptr)); @@ -2100,14 +3477,14 @@ template struct SimdMappings<128, element_t> : public SimdImpl /// Loads the lower half of the register from memory (in bytes), zeroing the upper half. /// Intended for safe tail handling without over-reading past the end of a buffer. /// - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL load_half(const element_t *ptr) noexcept + static int_vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_half(const element_t *ptr) noexcept requires std::is_integral_v { return _mm_loadl_epi64(reinterpret_cast(ptr)); } /// Loads a full register from memory. Pointer must be appropriately aligned for the register width. - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL load(const element_t *ptr) noexcept + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load(const element_t *ptr) noexcept requires std::is_floating_point_v { if constexpr (std::is_same_v) @@ -2117,7 +3494,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl } /// Loads a full register from memory without requiring alignment. - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL load_unaligned(const element_t *ptr) noexcept + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_unaligned(const element_t *ptr) noexcept requires std::is_floating_point_v { if constexpr (std::is_same_v) @@ -2129,14 +3506,14 @@ template struct SimdMappings<128, element_t> : public SimdImpl #pragma region Store /// Stores a full register to memory. Pointer must be appropriately aligned for the register width. - SIMDLIB_FORCE_INLINE static void VECTORCALL store(int_vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store(int_vector_t lhs, void *ptr) noexcept requires std::is_integral_v { _mm_store_si128(reinterpret_cast(ptr), lhs); } /// Stores a full register to memory without requiring alignment. - SIMDLIB_FORCE_INLINE static void VECTORCALL store_unaligned(int_vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store_unaligned(int_vector_t lhs, void *ptr) noexcept requires std::is_integral_v { _mm_storeu_si128(reinterpret_cast(ptr), lhs); @@ -2146,14 +3523,14 @@ template struct SimdMappings<128, element_t> : public SimdImpl /// Stores the lower half of the register to memory (in bytes). /// Intended for safe tail handling without over-writing past the end of a buffer. /// - SIMDLIB_FORCE_INLINE static void VECTORCALL store_half(int_vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store_half(int_vector_t lhs, void *ptr) noexcept requires std::is_integral_v { _mm_storel_epi64(reinterpret_cast(ptr), lhs); } /// Stores a full register to memory. Pointer must be appropriately aligned for the register width. - SIMDLIB_FORCE_INLINE static void VECTORCALL store(vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store(vector_t lhs, void *ptr) noexcept requires std::is_floating_point_v { if constexpr (std::is_same_v) @@ -2163,7 +3540,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl } /// Stores a full register to memory without requiring alignment. - SIMDLIB_FORCE_INLINE static void VECTORCALL store_unaligned(vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store_unaligned(vector_t lhs, void *ptr) noexcept requires std::is_floating_point_v { if constexpr (std::is_same_v) @@ -2181,7 +3558,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl * @param rhs The second register. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_and(vector_t lhs, vector_t rhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_and(vector_t lhs, vector_t rhs) noexcept { if constexpr (std::is_integral_v) return _mm_and_si128(lhs, rhs); @@ -2197,7 +3574,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl * @param rhs The second register. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_or(vector_t lhs, vector_t rhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_or(vector_t lhs, vector_t rhs) noexcept { if constexpr (std::is_integral_v) return _mm_or_si128(lhs, rhs); @@ -2213,7 +3590,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl * @param rhs The second register. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_xor(vector_t lhs, vector_t rhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_xor(vector_t lhs, vector_t rhs) noexcept { if constexpr (std::is_integral_v) return _mm_xor_si128(lhs, rhs); @@ -2228,7 +3605,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl * @param lhs The source register. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_not(vector_t lhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_not(vector_t lhs) noexcept { if constexpr (std::is_integral_v) return _mm_xor_si128(lhs, _mm_cmpeq_epi32(_mm_setzero_si128(), _mm_setzero_si128())); @@ -2244,7 +3621,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl * @param rhs The register to combine with the complement. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_andnot(vector_t lhs, vector_t rhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_andnot(vector_t lhs, vector_t rhs) noexcept { if constexpr (std::is_integral_v) return _mm_andnot_si128(lhs, rhs); @@ -2256,7 +3633,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl #pragma endregion #pragma region Arithmetic Operations - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL negate(int_vector_t lhs) noexcept + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) negate(int_vector_t lhs) noexcept requires std::is_integral_v { if constexpr (sizeof(element_t) == 8) @@ -2269,7 +3646,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl return _mm_sub_epi8(_mm_setzero_si128(), lhs); } - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL negate(vector_t lhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) negate(vector_t lhs) noexcept requires std::is_floating_point_v { if constexpr (std::same_as) @@ -2281,82 +3658,141 @@ template struct SimdMappings<128, element_t> : public SimdImpl #pragma region 128-bit Shifting - /// Shifts all bytes in the vector to the left by the specified number of bytes. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL byte_shift_left(int_vector_t lhs, int shift) noexcept + /** + * @brief Shifts a complete register toward higher byte indices. + * @param lhs Source register. + * @param shift Runtime byte count. + * @return Shifted register with zero-filled low bytes. + */ + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_left_slow(int_vector_t lhs, int shift) noexcept { - return register_byte_shift_left(lhs, shift); + return _ext128_shift_bytes_left_slow(lhs, shift); } - /// Shifts all bytes in the vector to the right by the specified number of bytes. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL byte_shift_right(int_vector_t lhs, int shift) noexcept + /** + * @brief Shifts a complete register toward lower byte indices. + * @param lhs Source register. + * @param shift Runtime byte count. + * @return Shifted register with zero-filled high bytes. + */ + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_right_slow(int_vector_t lhs, int shift) noexcept + { + return _ext128_shift_bytes_right_slow(lhs, shift); + } + + /** + * @brief Shifts a complete register toward higher byte indices by a compile-time count. + * @tparam count Nonnegative byte count; counts of at least 16 produce zero. + * @param lhs Source register. + * @return Shifted register with zero-filled low bytes. + */ + template static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_left(const int_vector_t lhs) noexcept { - return register_byte_shift_right(lhs, shift); + static_assert(count >= 0, "Complete-register byte shifts require a nonnegative count."); + if constexpr (count == 0) + return lhs; + else if constexpr (count >= 16) + return _mm_setzero_si128(); + else + return _mm_slli_si128(lhs, count); } - /// Shifts all bits of the vector to the left by the specified number of bits. - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL bit_shift_left(int_vector_t lhs, int shift) noexcept + /** + * @brief Shifts a complete register toward lower byte indices by a compile-time count. + * @tparam count Nonnegative byte count; counts of at least 16 produce zero. + * @param lhs Source register. + * @return Shifted register with zero-filled high bytes. + */ + template static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_right(const int_vector_t lhs) noexcept + { + static_assert(count >= 0, "Complete-register byte shifts require a nonnegative count."); + if constexpr (count == 0) + return lhs; + else if constexpr (count >= 16) + return _mm_setzero_si128(); + else + return _mm_srli_si128(lhs, count); + } + + /** + * @brief Shifts a complete 128-bit register left by a runtime bit count. + * @param lhs Source register interpreted as one unsigned 128-bit bit string. + * @param shift Runtime count; nonpositive counts are identity and counts of at least 128 produce zero. + * @return Shifted register with zero-filled low bits. + */ + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_left_slow(const int_vector_t lhs, const int shift) noexcept { - return _ext128_shift_left_bits_dynamic(lhs, shift); + return _ext128_shift_bits_left_slow(lhs, shift); } - /// Shifts all bits of the vector to the right by the specified number of bits. - SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL bit_shift_right(int_vector_t lhs, int shift) noexcept + /** + * @brief Shifts a complete 128-bit register right by a runtime bit count. + * @param lhs Source register interpreted as one unsigned 128-bit bit string. + * @param shift Runtime count; nonpositive counts are identity and counts of at least 128 produce zero. + * @return Shifted register with zero-filled high bits. + */ + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_right_slow(const int_vector_t lhs, const int shift) noexcept { - return _ext128_shift_right_bits_dynamic(lhs, shift); + return _ext128_shift_bits_right_slow(lhs, shift); } - /// Shifts all bits of the vector to the left by the specified number of bits. - template SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL bit_shift_left(int_vector_t lhs) noexcept + /** + * @brief Shifts a complete 128-bit register left by a compile-time bit count. + * @tparam shift Nonnegative bit count; counts of at least 128 produce zero. + * @param lhs Source register interpreted as one unsigned 128-bit bit string. + * @return Shifted register with zero-filled low bits. + */ + template static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_left(const int_vector_t lhs) noexcept { - return _ext128_shift_left_bits_static(lhs); + return _ext128_shift_bits_left_static(lhs); } - /// Shifts all bits of the vector to the right by the specified number of bits. - template SIMDLIB_FORCE_INLINE constexpr static int_vector_t VECTORCALL bit_shift_right(int_vector_t lhs) noexcept + /** + * @brief Shifts a complete 128-bit register right by a compile-time bit count. + * @tparam shift Nonnegative bit count; counts of at least 128 produce zero. + * @param lhs Source register interpreted as one unsigned 128-bit bit string. + * @return Shifted register with zero-filled high bits. + */ + template static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_right(const int_vector_t lhs) noexcept { - return _ext128_shift_right_bits_static(lhs); + return _ext128_shift_bits_right_static(lhs); } #pragma endregion #pragma region Shuffling - /// Shuffles the 32-bit integers in the vector using the specified control mask. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL shuffle_32(int_vector_t lhs, std::uint32_t imm8) noexcept + /** @brief Emulates an immediate-controlled 32-bit shuffle with a runtime scalar control. + * @param lhs Source register. + * @param imm8 Runtime control byte. + * @return Register with each four-lane group shuffled. + */ + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_32_slow(int_vector_t lhs, std::uint32_t imm8) noexcept requires std::is_integral_v { - return register_shuffle_32(lhs, imm8); + return register_shuffle_32_slow(lhs, imm8); } /// Shuffles the 32-bit integers in the vector using a compile-time control mask. template - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL shuffle_32(int_vector_t lhs) noexcept + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_32(int_vector_t lhs) noexcept requires std::is_integral_v { return _mm_shuffle_epi32(lhs, imm8); } /// Shuffles the bytes in the vector using the indexes in the second vector. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL shuffle(int_vector_t lhs, int_vector_t indices) noexcept + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(int_vector_t lhs, int_vector_t indices) noexcept requires std::is_integral_v { return _mm_shuffle_epi8(lhs, indices); } - /// Shuffles the bytes in the vector using the templated index sequence. - template - requires(sizeof...(indices) == 16) - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL shuffle(int_vector_t lhs) noexcept - { - // A constexpr register initializer was intentionally replaced by the portable runtime intrinsic. - // The active compiler-independent constexpr register construction lives in Detail::register_from_values. - return _mm_shuffle_epi8(lhs, _mm_setr_epi8(indices...)); - } #pragma endregion #pragma region Miscellaneous Operations /// Returns a mask of the most significant BIT of each BYTE in each element. - SIMDLIB_FORCE_INLINE static mask_t VECTORCALL movemask(const vector_t lhs) noexcept + static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) movemask(const vector_t lhs) noexcept { if constexpr (std::is_integral_v) return _mm_movemask_epi8(lhs); @@ -2367,7 +3803,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl } /// Returns a mask of the most significant BIT of each element. - SIMDLIB_FORCE_INLINE static mask_t VECTORCALL movemask_slim(const vector_t lhs) noexcept + static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) movemask_slim(const vector_t lhs) noexcept { if constexpr (std::is_integral_v) return movemask(swizzle_msb(lhs)); @@ -2379,14 +3815,14 @@ template struct SimdMappings<128, element_t> : public SimdImpl /// Compute the bitwise AND of 128 bits (representing integer data) in a and b, and set ZF to 1 if the result is zero, otherwise set ZF to 0. /// Compute the bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, otherwise set CF to 0. Return the CF value. - SIMDLIB_FORCE_INLINE static int VECTORCALL test(int_vector_t lhs, int_vector_t rhs) noexcept + static int SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) test(int_vector_t lhs, int_vector_t rhs) noexcept { return _mm_testc_si128(lhs, rhs); } /// Compute the bitwise AND of 128 bits (representing integer data) in a and b, and set ZF to 1 if the result is zero, otherwise set ZF to 0. /// Compute the bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, otherwise set CF to 0. Return the ZF value. - SIMDLIB_FORCE_INLINE static int VECTORCALL testz(int_vector_t lhs, int_vector_t rhs) noexcept + static int SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) testz(int_vector_t lhs, int_vector_t rhs) noexcept { return _mm_testz_si128(lhs, rhs); } @@ -2394,7 +3830,7 @@ template struct SimdMappings<128, element_t> : public SimdImpl /// Compute the bitwise AND of 128 bits (representing integer data) in a and b, and set ZF to 1 if the result is zero, otherwise set ZF to 0. /// Compute the bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, otherwise set CF to 0. Return 1 if both the ZF and CF values /// are zero, otherwise return 0. - SIMDLIB_FORCE_INLINE static int VECTORCALL testnzc(int_vector_t lhs, int_vector_t rhs) noexcept + static int SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) testnzc(int_vector_t lhs, int_vector_t rhs) noexcept { return _mm_testnzc_si128(lhs, rhs); } @@ -2411,19 +3847,19 @@ template struct SimdMappings<128, element_t> : public SimdImpl if (i < elem_count) { // Select the MSB byte of each element, packing them into the low bytes. - register_set(seq, i, static_cast((i * elem_size) + (elem_size - 1))); + register_set_constexpr(seq, i, static_cast((i * elem_size) + (elem_size - 1))); } else { // Zero out the rest (PSHUFB: high bit set => 0). - register_set(seq, i, 0x80); + register_set_constexpr(seq, i, 0x80); } } return seq; } /// Swizzle the vector to only contain the most significant bit of each byte. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL swizzle_msb(int_vector_t lhs) noexcept + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) swizzle_msb(int_vector_t lhs) noexcept { return shuffle(lhs, get_msb_swizzle_order()); } @@ -2443,52 +3879,141 @@ struct SimdImpl256 { }; +/** + * @brief Reports whether a 256-bit logical shuffle selects any lane from the opposite 128-bit half. + * @tparam lanes_per_half Logical lanes in one 128-bit half. + * @tparam indices Complete logical selector array. + * @return True when at least one output lane crosses the 128-bit boundary. + */ +template [[nodiscard]] consteval bool logical_shuffle_256_has_cross_half_selector() noexcept +{ + for (std::size_t output = 0; output < indices.size(); ++output) + if (output / lanes_per_half != indices[output] / lanes_per_half) + return true; + return false; +} + +/** + * @brief Reports whether a 256-bit logical shuffle selects any lane from its original 128-bit half. + * @tparam lanes_per_half Logical lanes in one 128-bit half. + * @tparam indices Complete logical selector array. + * @return True when at least one output lane remains within its original 128-bit half. + */ +template [[nodiscard]] consteval bool logical_shuffle_256_has_local_half_selector() noexcept +{ + for (std::size_t output = 0; output < indices.size(); ++output) + if (output / lanes_per_half == indices[output] / lanes_per_half) + return true; + return false; +} + +/** + * @brief Encodes one byte of a full-width 256-bit byte or word shuffle control. + * @param element_bytes Bytes in each logical lane. + * @param select_cross_half Whether this control selects cross-half or local-half lanes. + * @param output_lane Logical output lane containing the byte. + * @param source_lane Logical source lane selected for the output lane. + * @param byte_in_lane Byte position within the logical output lane. + * @return Lane-relative VPSHUFB selector or the zeroing sentinel when handled by the other control. + */ +[[nodiscard]] consteval int encode_logical_shuffle_256_byte(const std::size_t element_bytes, const bool select_cross_half, const std::size_t output_lane, + const std::size_t source_lane, const std::size_t byte_in_lane) noexcept +{ + const std::size_t lanes_per_half = 16 / element_bytes; + const bool crosses_half = output_lane / lanes_per_half != source_lane / lanes_per_half; + if (crosses_half != select_cross_half) + return 0x80; + return static_cast((source_lane % lanes_per_half) * element_bytes + byte_in_lane); +} + +/** + * @brief Builds one constant VPSHUFB control for a full-width 256-bit byte or word shuffle. + * @tparam element_bytes Bytes in each logical lane. + * @tparam select_cross_half Whether this control selects cross-half or local-half lanes. + * @tparam indices Complete logical selector array. + * @tparam byte_positions Output byte positions. + * @return Native AVX2 byte-control register. + */ +template +static __m256i SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) make_logical_shuffle_256_byte_control(std::index_sequence) noexcept +{ + return _mm256_setr_epi8(static_cast(encode_logical_shuffle_256_byte(element_bytes, select_cross_half, byte_positions / element_bytes, + indices[byte_positions / element_bytes], byte_positions % element_bytes))...); +} + template <> struct SimdImpl256 { + /** @brief Selects bytes from two registers using a canonical predicate register. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256i condition, __m256i when_true, __m256i when_false) noexcept + { + return _mm256_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical signed-byte lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 32 && ((indices < 32) && ...)) + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256i lhs) noexcept + { + constexpr auto selectors = std::array{indices...}; + if constexpr (!logical_shuffle_256_has_cross_half_selector<16, selectors>()) + { + return _mm256_shuffle_epi8(lhs, make_logical_shuffle_256_byte_control<1, false, selectors>(std::make_index_sequence<32>{})); + } + else + { + const __m256i swapped = _mm256_permute2x128_si256(lhs, lhs, 0x01); + if constexpr (!logical_shuffle_256_has_local_half_selector<16, selectors>()) + return _mm256_shuffle_epi8(swapped, make_logical_shuffle_256_byte_control<1, true, selectors>(std::make_index_sequence<32>{})); + else + return _mm256_or_si256(_mm256_shuffle_epi8(lhs, make_logical_shuffle_256_byte_control<1, false, selectors>(std::make_index_sequence<32>{})), + _mm256_shuffle_epi8(swapped, make_logical_shuffle_256_byte_control<1, true, selectors>(std::make_index_sequence<32>{}))); + } + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies signed byte lanes and adds adjacent products into signed 16-bit lanes. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { - const __m128i low = _mm256_castsi256_si128(lhs); - const __m128i lowRhs = _mm256_castsi256_si128(rhs); - const __m128i high = _mm256_extracti128_si256(lhs, 1); - const __m128i highRhs = _mm256_extracti128_si256(rhs, 1); - const __m128i lhsWideLo = _mm_cvtepi8_epi16(low); - const __m128i rhsWideLo = _mm_cvtepi8_epi16(lowRhs); - const __m128i lhsWideHi = _mm_cvtepi8_epi16(_mm_srli_si128(low, 8)); - const __m128i rhsWideHi = _mm_cvtepi8_epi16(_mm_srli_si128(lowRhs, 8)); - const __m128i lowResult = _mm_hadd_epi16(_mm_mullo_epi16(lhsWideLo, rhsWideLo), _mm_mullo_epi16(lhsWideHi, rhsWideHi)); - const __m128i highWideLo = _mm_cvtepi8_epi16(high); - const __m128i highRhsWideLo = _mm_cvtepi8_epi16(highRhs); - const __m128i highWideHi = _mm_cvtepi8_epi16(_mm_srli_si128(high, 8)); - const __m128i highRhsWideHi = _mm_cvtepi8_epi16(_mm_srli_si128(highRhs, 8)); - const __m128i highResult = _mm_hadd_epi16(_mm_mullo_epi16(highWideLo, highRhsWideLo), _mm_mullo_epi16(highWideHi, highRhsWideHi)); - return _mm256_inserti128_si256(_mm256_castsi128_si256(lowResult), highResult, 1); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + const __m256i lowProducts = _mm256_mullo_epi16(_mm256_cvtepi8_epi16(_mm256_castsi256_si128(lhs)), _mm256_cvtepi8_epi16(_mm256_castsi256_si128(rhs))); + const __m256i highProducts = + _mm256_mullo_epi16(_mm256_cvtepi8_epi16(_mm256_extracti128_si256(lhs, 1)), _mm256_cvtepi8_epi16(_mm256_extracti128_si256(rhs, 1))); + const __m256i interleavedSums = _mm256_hadd_epi16(lowProducts, highProducts); + return _mm256_permute4x64_epi64(interleavedSums, _MM_SHUFFLE(3, 1, 2, 0)); + } + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm256_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _ext256_mul_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding signed 8-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext256_div_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes scalar-equivalent signed 8-bit remainders with register-only extraction and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext256_rem_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { auto sqrt16x16 = [](__m256i values) noexcept { @@ -2513,136 +4038,194 @@ template <> struct SimdImpl256 const __m128i packedHigh = _mm_packs_epi16(_mm256_castsi256_si128(rootsHigh16), _mm256_extracti128_si256(rootsHigh16, 1)); return _mm256_inserti128_si256(_mm256_castsi128_si256(packedLow), packedHigh, 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes one unchecked magnitude in lane zero of each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept { - const __m128i low = _mm256_castsi256_si128(lhs); - const __m128i high = _mm256_extracti128_si256(lhs, 1); - const __m128i lowMeta = SimdImpl128::min_position(low); - const __m128i highMeta = SimdImpl128::min_position(high); - alignas(16) std::array lowData{}; - alignas(16) std::array highData{}; - alignas(32) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(lowData.data()), lowMeta); - _mm_store_si128(reinterpret_cast<__m128i *>(highData.data()), highMeta); - highData[1] = static_cast(highData[1] + 16); - output[0] = highData[0] < lowData[0] ? highData[0] : lowData[0]; - output[1] = highData[0] < lowData[0] ? highData[1] : lowData[1]; - return _mm256_load_si256(reinterpret_cast(output.data())); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + const __m128i lowMagnitude = SimdImpl128::magnitude(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Computes saturated magnitudes and adjacent overflow masks for both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude_checked(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude_checked(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Returns the minimum value and its first lane position without materializing register data in memory. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept + { + const __m128i lowMeta = SimdImpl128::min_position(_mm256_castsi256_si128(lhs)); + const __m128i highMeta = SimdImpl128::min_position(_mm256_extracti128_si256(lhs, 1)); + const auto lowValue = static_cast(_mm_extract_epi8(lowMeta, 0)); + const auto highValue = static_cast(_mm_extract_epi8(highMeta, 0)); + const bool chooseHigh = highValue < lowValue; + const __m128i selectedMeta = chooseHigh ? highMeta : lowMeta; + const int position = static_cast(_mm_extract_epi8(selectedMeta, 1)) + (chooseHigh ? 16 : 0); + __m128i output = _mm_setzero_si128(); + output = _mm_insert_epi8(output, _mm_extract_epi8(selectedMeta, 0), 0); + output = _mm_insert_epi8(output, position, 1); + return _mm256_zextsi128_si256(output); + } + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm256_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m256i lhs, __m256i rhs) noexcept { return _mm256_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm256_abs_epi8(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm256_min_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm256_max_epi8(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _ext256_slli_epx8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _ext256_srli_epx8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _ext256_srai_epx8(lhs, rhs); } // arithmetic (saturated) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_saturated(auto lhs, auto rhs) noexcept + /** @brief Adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(auto lhs, auto rhs) noexcept { return _mm256_adds_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(auto lhs, auto rhs) noexcept { return _mm256_subs_epi8(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm256_set1_epi8(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args &&...args) noexcept { return _mm256_set_epi8(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args &&...args) noexcept { return _mm256_setr_epi8(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm256_cmpeq_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm256_cmpgt_epi8(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm256_cvtepi8_epi16(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm256_extract_epi8(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected signed 8-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 32)`. + * @return Selected scalar lane. + */ + static int8_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 32, "Signed 8-bit extraction requires a valid 256-bit lane index"); + const __m256i selected = _mm256_permutevar8x32_epi32(lhs, _mm256_set1_epi32(index >> 2)); + const uint32_t selected_dword = static_cast(_mm_cvtsi128_si32(_mm256_castsi256_si128(selected))); + return static_cast(selected_dword >> ((index & 3) * 8)); + } + /** @brief Replaces the compile-time-selected signed 8-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const int8_t rhs) noexcept + { + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected signed 8-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const int8_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return _mm256_insert_epi8(lhs, static_cast(rhs), index); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int imm8) noexcept + /** + * @brief Replaces one runtime-selected signed 8-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 32)`. + * @return Register with the selected lane replaced. + */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256i lhs, const int8_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(imm8)); + SIMDLIB_PRECONDITION(index >= 0 && index < 32, "Signed 8-bit insertion requires a valid 256-bit lane index"); + if (index < 16) + { + const __m128i lower = SimdImpl128::insert_slow(_mm256_castsi256_si128(lhs), rhs, index); + return _mm256_inserti128_si256(lhs, lower, 0); + } + const __m128i upper = SimdImpl128::insert_slow(_mm256_extracti128_si256(lhs, 1), rhs, index - 16); + return _mm256_inserti128_si256(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_epi8(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(auto lhs, auto rhs) noexcept + /** @brief Shuffles bytes through the native runtime selector-register instruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle(auto lhs, auto rhs) noexcept + requires(std::same_as && std::same_as) { return _mm256_shuffle_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, auto mask) noexcept + /** @brief Selects bytes through the native runtime mask-register operation. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) blend(const __m256i lhs, const __m256i rhs, const __m256i mask) noexcept { return register_blend_bytes(lhs, rhs, mask); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL movemask(auto lhs) noexcept + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) movemask(auto lhs) noexcept { return _mm256_movemask_epi8(lhs); } @@ -2650,50 +4233,77 @@ template <> struct SimdImpl256 template <> struct SimdImpl256 { + /** @brief Selects bytes from two registers using a canonical predicate register. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256i condition, __m256i when_true, __m256i when_false) noexcept + { + return _mm256_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical unsigned-byte lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 32 && ((indices < 32) && ...)) + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256i lhs) noexcept + { + constexpr auto selectors = std::array{indices...}; + if constexpr (!logical_shuffle_256_has_cross_half_selector<16, selectors>()) + { + return _mm256_shuffle_epi8(lhs, make_logical_shuffle_256_byte_control<1, false, selectors>(std::make_index_sequence<32>{})); + } + else + { + const __m256i swapped = _mm256_permute2x128_si256(lhs, lhs, 0x01); + if constexpr (!logical_shuffle_256_has_local_half_selector<16, selectors>()) + return _mm256_shuffle_epi8(swapped, make_logical_shuffle_256_byte_control<1, true, selectors>(std::make_index_sequence<32>{})); + else + return _mm256_or_si256(_mm256_shuffle_epi8(lhs, make_logical_shuffle_256_byte_control<1, false, selectors>(std::make_index_sequence<32>{})), + _mm256_shuffle_epi8(swapped, make_logical_shuffle_256_byte_control<1, true, selectors>(std::make_index_sequence<32>{}))); + } + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned byte lanes and adds adjacent products into unsigned 16-bit lanes. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { - const __m128i low = _mm256_castsi256_si128(lhs); - const __m128i lowRhs = _mm256_castsi256_si128(rhs); - const __m128i high = _mm256_extracti128_si256(lhs, 1); - const __m128i highRhs = _mm256_extracti128_si256(rhs, 1); - const __m128i lhsWideLo = _mm_cvtepu8_epi16(low); - const __m128i rhsWideLo = _mm_cvtepu8_epi16(lowRhs); - const __m128i lhsWideHi = _mm_cvtepu8_epi16(_mm_srli_si128(low, 8)); - const __m128i rhsWideHi = _mm_cvtepu8_epi16(_mm_srli_si128(lowRhs, 8)); - const __m128i lowResult = _mm_hadd_epi16(_mm_mullo_epi16(lhsWideLo, rhsWideLo), _mm_mullo_epi16(lhsWideHi, rhsWideHi)); - const __m128i highWideLo = _mm_cvtepu8_epi16(high); - const __m128i highRhsWideLo = _mm_cvtepu8_epi16(highRhs); - const __m128i highWideHi = _mm_cvtepu8_epi16(_mm_srli_si128(high, 8)); - const __m128i highRhsWideHi = _mm_cvtepu8_epi16(_mm_srli_si128(highRhs, 8)); - const __m128i highResult = _mm_hadd_epi16(_mm_mullo_epi16(highWideLo, highRhsWideLo), _mm_mullo_epi16(highWideHi, highRhsWideHi)); - return _mm256_inserti128_si256(_mm256_castsi128_si256(lowResult), highResult, 1); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + const __m256i lowProducts = _mm256_mullo_epi16(_mm256_cvtepu8_epi16(_mm256_castsi256_si128(lhs)), _mm256_cvtepu8_epi16(_mm256_castsi256_si128(rhs))); + const __m256i highProducts = + _mm256_mullo_epi16(_mm256_cvtepu8_epi16(_mm256_extracti128_si256(lhs, 1)), _mm256_cvtepu8_epi16(_mm256_extracti128_si256(rhs, 1))); + const __m256i interleavedSums = _mm256_hadd_epi16(lowProducts, highProducts); + return _mm256_permute4x64_epi64(interleavedSums, _MM_SHUFFLE(3, 1, 2, 0)); + } + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm256_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _ext256_mul_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding unsigned 8-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext256_div_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes scalar-equivalent unsigned 8-bit remainders with register-only extraction and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext256_rem_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { auto sqrt16x16 = [](__m256i values) noexcept { @@ -2718,138 +4328,199 @@ template <> struct SimdImpl256 const __m128i packedHigh = _mm_packus_epi16(_mm256_castsi256_si128(rootsHigh16), _mm256_extracti128_si256(rootsHigh16, 1)); return _mm256_inserti128_si256(_mm256_castsi128_si256(packedLow), packedHigh, 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes one unchecked magnitude in lane zero of each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Computes saturated magnitudes and adjacent overflow masks for both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude_checked(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude_checked(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Returns the minimum value and its first lane position without materializing register data in memory. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { const __m128i lowMeta = SimdImpl128::min_position(_mm256_castsi256_si128(lhs)); const __m128i highMeta = SimdImpl128::min_position(_mm256_extracti128_si256(lhs, 1)); - alignas(16) std::array lowData{}; - alignas(16) std::array highData{}; - alignas(32) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(lowData.data()), lowMeta); - _mm_store_si128(reinterpret_cast<__m128i *>(highData.data()), highMeta); - highData[1] = static_cast(highData[1] + 16); - output[0] = highData[0] < lowData[0] ? highData[0] : lowData[0]; - output[1] = highData[0] < lowData[0] ? highData[1] : lowData[1]; - return _mm256_load_si256(reinterpret_cast(output.data())); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + const auto lowValue = static_cast(_mm_extract_epi8(lowMeta, 0)); + const auto highValue = static_cast(_mm_extract_epi8(highMeta, 0)); + const bool chooseHigh = highValue < lowValue; + const __m128i selectedMeta = chooseHigh ? highMeta : lowMeta; + const int position = static_cast(_mm_extract_epi8(selectedMeta, 1)) + (chooseHigh ? 16 : 0); + __m128i output = _mm_setzero_si128(); + output = _mm_insert_epi8(output, _mm_extract_epi8(selectedMeta, 0), 0); + output = _mm_insert_epi8(output, position, 1); + return _mm256_zextsi128_si256(output); + } + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm256_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m256i lhs, __m256i rhs) noexcept { return _mm256_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm256_abs_epi8(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm256_min_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm256_max_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL avg(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise averages for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) avg(auto lhs, auto rhs) noexcept { return _mm256_avg_epu8(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _ext256_slli_epx8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _ext256_srli_epx8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _ext256_srai_epx8(lhs, rhs); } // arithmetic (saturated) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_saturated(auto lhs, auto rhs) noexcept + /** @brief Adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(auto lhs, auto rhs) noexcept { return _mm256_adds_epu8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(auto lhs, auto rhs) noexcept { return _mm256_subs_epu8(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _ext256_set1_epu8(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args &&...args) noexcept { return _mm256_set_epi8(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args &&...args) noexcept { return _mm256_setr_epi8(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm256_cmpeq_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext256_cmpgt_epu8(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm256_cvtepu8_epi16(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm256_extract_epi8(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected unsigned 8-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 32)`. + * @return Selected scalar lane. + */ + static uint8_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256i lhs, const int index) noexcept { - return register_get(lhs, static_cast(rhs)); + SIMDLIB_PRECONDITION(index >= 0 && index < 32, "Unsigned 8-bit extraction requires a valid 256-bit lane index"); + const __m256i selected = _mm256_permutevar8x32_epi32(lhs, _mm256_set1_epi32(index >> 2)); + const uint32_t selected_dword = static_cast(_mm_cvtsi128_si32(_mm256_castsi256_si128(selected))); + return static_cast(selected_dword >> ((index & 3) * 8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Replaces the compile-time-selected unsigned 8-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const uint8_t rhs) noexcept { - return register_insert(lhs, rhs, static_cast(imm8)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected unsigned 8-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const uint8_t rhs) noexcept + { + return _mm256_insert_epi8(lhs, static_cast(rhs), index); + } + /** + * @brief Replaces one runtime-selected unsigned 8-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 32)`. + * @return Register with the selected lane replaced. + */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256i lhs, const uint8_t rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 32, "Unsigned 8-bit insertion requires a valid 256-bit lane index"); + if (index < 16) + { + const __m128i lower = SimdImpl128::insert_slow(_mm256_castsi256_si128(lhs), rhs, index); + return _mm256_inserti128_si256(lhs, lower, 0); + } + const __m128i upper = SimdImpl128::insert_slow(_mm256_extracti128_si256(lhs, 1), rhs, index - 16); + return _mm256_inserti128_si256(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_epi8(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(auto lhs, auto rhs) noexcept + /** @brief Shuffles bytes through the native runtime selector-register instruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle(auto lhs, auto rhs) noexcept + requires(std::same_as && std::same_as) { return _mm256_shuffle_epi8(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, auto mask) noexcept + /** @brief Selects bytes through the native runtime mask-register operation. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) blend(const __m256i lhs, const __m256i rhs, const __m256i mask) noexcept { return register_blend_bytes(lhs, rhs, mask); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL movemask(auto lhs) noexcept + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) movemask(auto lhs) noexcept { return _mm256_movemask_epi8(lhs); } @@ -2857,36 +4528,73 @@ template <> struct SimdImpl256 template <> struct SimdImpl256 { + /** @brief Selects 16-bit lanes from two registers using a canonical predicate register. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256i condition, __m256i when_true, __m256i when_false) noexcept + { + return _mm256_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical signed 16-bit lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 16 && ((indices < 16) && ...)) + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256i lhs) noexcept + { + constexpr auto selectors = std::array{indices...}; + if constexpr (!logical_shuffle_256_has_cross_half_selector<8, selectors>()) + { + return _mm256_shuffle_epi8(lhs, make_logical_shuffle_256_byte_control<2, false, selectors>(std::make_index_sequence<32>{})); + } + else + { + const __m256i swapped = _mm256_permute2x128_si256(lhs, lhs, 0x01); + if constexpr (!logical_shuffle_256_has_local_half_selector<8, selectors>()) + return _mm256_shuffle_epi8(swapped, make_logical_shuffle_256_byte_control<2, true, selectors>(std::make_index_sequence<32>{})); + else + return _mm256_or_si256(_mm256_shuffle_epi8(lhs, make_logical_shuffle_256_byte_control<2, false, selectors>(std::make_index_sequence<32>{})), + _mm256_shuffle_epi8(swapped, make_logical_shuffle_256_byte_control<2, true, selectors>(std::make_index_sequence<32>{}))); + } + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { return _mm256_madd_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm256_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm256_mullo_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding signed 16-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext256_div_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes scalar-equivalent signed 16-bit remainders with register-only extraction and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext256_rem_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { auto sqrt16x8 = [](__m128i values) noexcept { @@ -2902,88 +4610,116 @@ template <> struct SimdImpl256 const __m128i rootsHigh = sqrt16x8(high16); return _mm256_inserti128_si256(_mm256_castsi128_si256(rootsLow), rootsHigh, 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes one unchecked magnitude in lane zero of each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Computes saturated magnitudes and adjacent overflow masks for both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude_checked(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude_checked(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Returns the minimum value and its first lane position without materializing register data in memory. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { const __m128i lowMeta = SimdImpl128::min_position(_mm256_castsi256_si128(lhs)); const __m128i highMeta = SimdImpl128::min_position(_mm256_extracti128_si256(lhs, 1)); - alignas(16) std::array lowData{}; - alignas(16) std::array highData{}; - alignas(32) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(lowData.data()), lowMeta); - _mm_store_si128(reinterpret_cast<__m128i *>(highData.data()), highMeta); - highData[1] = static_cast(highData[1] + 8); - output[0] = highData[0] < lowData[0] ? highData[0] : lowData[0]; - output[1] = highData[0] < lowData[0] ? highData[1] : lowData[1]; - return _mm256_load_si256(reinterpret_cast(output.data())); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + const auto lowValue = static_cast(_mm_extract_epi16(lowMeta, 0)); + const auto highValue = static_cast(_mm_extract_epi16(highMeta, 0)); + const bool chooseHigh = highValue < lowValue; + const __m128i selectedMeta = chooseHigh ? highMeta : lowMeta; + const int position = static_cast(_mm_extract_epi16(selectedMeta, 1)) + (chooseHigh ? 8 : 0); + __m128i output = _mm_setzero_si128(); + output = _mm_insert_epi16(output, _mm_extract_epi16(selectedMeta, 0), 0); + output = _mm_insert_epi16(output, position, 1); + return _mm256_zextsi128_si256(output); + } + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm256_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m256i lhs, __m256i rhs) noexcept { return _mm256_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm256_abs_epi16(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm256_min_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm256_max_epi16(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm256_slli_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm256_srli_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _mm256_srai_epi16(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hadd_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hsub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL hadd_saturated(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hadd_saturated(auto lhs, auto rhs) noexcept { return _mm256_hadds_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL hsubtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hsubtract_saturated(auto lhs, auto rhs) noexcept { return _mm256_hsubs_epi16(lhs, rhs); } // arithmetic (saturated) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_saturated(auto lhs, auto rhs) noexcept + /** @brief Adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(auto lhs, auto rhs) noexcept { return _mm256_adds_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(auto lhs, auto rhs) noexcept { return _mm256_subs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_saturated(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) multiply_saturated(auto lhs, auto rhs) noexcept { const __m128i lhsLo128 = _mm256_castsi256_si128(lhs); const __m128i rhsLo128 = _mm256_castsi256_si128(rhs); @@ -2999,110 +4735,217 @@ template <> struct SimdImpl256 } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm256_set1_epi16(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args &&...args) noexcept { return _mm256_set_epi16(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args &&...args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args &&...args) noexcept { return _mm256_setr_epi16(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm256_cmpeq_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm256_cmpgt_epi16(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm256_cvtepi16_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL compress(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) compress(auto lhs, auto rhs) noexcept { return _mm256_packs_epi16(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm256_extract_epi16(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected signed 16-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 16)`. + * @return Selected scalar lane. + */ + static int16_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256i lhs, const int index) noexcept { - return register_get(lhs, static_cast(rhs)); + SIMDLIB_PRECONDITION(index >= 0 && index < 16, "Signed 16-bit extraction requires a valid 256-bit lane index"); + const __m256i selected = _mm256_permutevar8x32_epi32(lhs, _mm256_set1_epi32(index >> 1)); + const uint32_t selected_dword = static_cast(_mm_cvtsi128_si32(_mm256_castsi256_si128(selected))); + return static_cast(selected_dword >> ((index & 1) * 16)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Replaces the compile-time-selected signed 16-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const int16_t rhs) noexcept { - return register_insert(lhs, rhs, static_cast(imm8)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected signed 16-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const int16_t rhs) noexcept + { + return _mm256_insert_epi16(lhs, static_cast(rhs), index); + } + /** + * @brief Replaces one runtime-selected signed 16-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 16)`. + * @return Register with the selected lane replaced. + */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256i lhs, const int16_t rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 16, "Signed 16-bit insertion requires a valid 256-bit lane index"); + if (index < 8) + { + const __m128i lower = SimdImpl128::insert_slow(_mm256_castsi256_si128(lhs), rhs, index); + return _mm256_inserti128_si256(lhs, lower, 0); + } + const __m128i upper = SimdImpl128::insert_slow(_mm256_extracti128_si256(lhs, 1), rhs, index - 8); + return _mm256_inserti128_si256(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_epi16(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_lo(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled low-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each low four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_lo_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_half_16_slow(lhs, static_cast(rhs), false); + } + /** @brief Shuffles the low four signed 16-bit lanes in each 128-bit group with an immediate control. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_lo(__m256i lhs) noexcept + { + return _mm256_shufflelo_epi16(lhs, imm8); + } + /** @brief Emulates an immediate-controlled high-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each high four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_hi_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_half_16_slow(lhs, static_cast(rhs), true); + } + /** @brief Shuffles the high four signed 16-bit lanes in each 128-bit group with an immediate control. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_hi(__m256i lhs) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), false); + return _mm256_shufflehi_epi16(lhs, imm8); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_hi(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), true); + return register_blend_slow(lhs, rhs, static_cast(imm8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects signed 16-bit lanes from two 256-bit registers with a repeated immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m256i lhs, const __m256i rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm256_blend_epi16(lhs, rhs, imm8); } }; template <> struct SimdImpl256 { + /** @brief Selects 16-bit lanes from two registers using a canonical predicate register. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256i condition, __m256i when_true, __m256i when_false) noexcept + { + return _mm256_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical unsigned 16-bit lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 16 && ((indices < 16) && ...)) + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256i lhs) noexcept + { + constexpr auto selectors = std::array{indices...}; + if constexpr (!logical_shuffle_256_has_cross_half_selector<8, selectors>()) + { + return _mm256_shuffle_epi8(lhs, make_logical_shuffle_256_byte_control<2, false, selectors>(std::make_index_sequence<32>{})); + } + else + { + const __m256i swapped = _mm256_permute2x128_si256(lhs, lhs, 0x01); + if constexpr (!logical_shuffle_256_has_local_half_selector<8, selectors>()) + return _mm256_shuffle_epi8(swapped, make_logical_shuffle_256_byte_control<2, true, selectors>(std::make_index_sequence<32>{})); + else + return _mm256_or_si256(_mm256_shuffle_epi8(lhs, make_logical_shuffle_256_byte_control<2, false, selectors>(std::make_index_sequence<32>{})), + _mm256_shuffle_epi8(swapped, make_logical_shuffle_256_byte_control<2, true, selectors>(std::make_index_sequence<32>{}))); + } + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm256_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent unsigned 16-bit lanes and adds their products into unsigned 32-bit lanes. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { - return _mm256_madd_epi16(lhs, rhs); + const __m256i lowProducts = _mm256_mullo_epi32(_mm256_cvtepu16_epi32(_mm256_castsi256_si128(lhs)), _mm256_cvtepu16_epi32(_mm256_castsi256_si128(rhs))); + const __m256i highProducts = + _mm256_mullo_epi32(_mm256_cvtepu16_epi32(_mm256_extracti128_si256(lhs, 1)), _mm256_cvtepu16_epi32(_mm256_extracti128_si256(rhs, 1))); + const __m256i interleavedSums = _mm256_hadd_epi32(lowProducts, highProducts); + return _mm256_permute4x64_epi64(interleavedSums, _MM_SHUFFLE(3, 1, 2, 0)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm256_mullo_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding unsigned 16-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext256_div_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes scalar-equivalent unsigned 16-bit remainders with register-only extraction and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext256_rem_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { auto sqrt16x8 = [](__m128i values) noexcept { @@ -3118,92 +4961,127 @@ template <> struct SimdImpl256 const __m128i rootsHigh = sqrt16x8(high16); return _mm256_inserti128_si256(_mm256_castsi128_si256(rootsLow), rootsHigh, 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes one unchecked magnitude in lane zero of each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Computes saturated magnitudes and adjacent overflow masks for both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude_checked(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude_checked(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Returns the minimum value and its first lane position without materializing register data in memory. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { const __m128i lowMeta = SimdImpl128::min_position(_mm256_castsi256_si128(lhs)); const __m128i highMeta = SimdImpl128::min_position(_mm256_extracti128_si256(lhs, 1)); - alignas(16) std::array lowData{}; - alignas(16) std::array highData{}; - alignas(32) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(lowData.data()), lowMeta); - _mm_store_si128(reinterpret_cast<__m128i *>(highData.data()), highMeta); - highData[1] = static_cast(highData[1] + 8); - output[0] = highData[0] < lowData[0] ? highData[0] : lowData[0]; - output[1] = highData[0] < lowData[0] ? highData[1] : lowData[1]; - return _mm256_load_si256(reinterpret_cast(output.data())); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + const auto lowValue = static_cast(_mm_extract_epi16(lowMeta, 0)); + const auto highValue = static_cast(_mm_extract_epi16(highMeta, 0)); + const bool chooseHigh = highValue < lowValue; + const __m128i selectedMeta = chooseHigh ? highMeta : lowMeta; + const int position = static_cast(_mm_extract_epi16(selectedMeta, 1)) + (chooseHigh ? 8 : 0); + __m128i output = _mm_setzero_si128(); + output = _mm_insert_epi16(output, _mm_extract_epi16(selectedMeta, 0), 0); + output = _mm_insert_epi16(output, position, 1); + return _mm256_zextsi128_si256(output); + } + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm256_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m256i lhs, __m256i rhs) noexcept { return _mm256_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm256_abs_epi16(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm256_min_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm256_max_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL avg(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise averages for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) avg(auto lhs, auto rhs) noexcept { return _mm256_avg_epu16(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm256_slli_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm256_srli_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _mm256_srai_epi16(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hadd_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hsub_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL hadd_saturated(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds unsigned 16-bit lanes with unsigned saturation in each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hadd_saturated(auto lhs, auto rhs) noexcept { - return _mm256_hadds_epi16(lhs, rhs); + const __m256i zero = _mm256_setzero_si256(); + const __m256i lhsPairs = _mm256_adds_epu16(lhs, _mm256_srli_epi32(lhs, 16)); + const __m256i rhsPairs = _mm256_adds_epu16(rhs, _mm256_srli_epi32(rhs, 16)); + return _mm256_packus_epi32(_mm256_blend_epi16(lhsPairs, zero, 0xAA), _mm256_blend_epi16(rhsPairs, zero, 0xAA)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL hsubtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts unsigned 16-bit lanes with unsigned saturation in each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) hsubtract_saturated(auto lhs, auto rhs) noexcept { - return _mm256_hsubs_epi16(lhs, rhs); + const __m256i zero = _mm256_setzero_si256(); + const __m256i lhsPairs = _mm256_subs_epu16(lhs, _mm256_srli_epi32(lhs, 16)); + const __m256i rhsPairs = _mm256_subs_epu16(rhs, _mm256_srli_epi32(rhs, 16)); + return _mm256_packus_epi32(_mm256_blend_epi16(lhsPairs, zero, 0xAA), _mm256_blend_epi16(rhsPairs, zero, 0xAA)); } // arithmetic (saturated) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_saturated(auto lhs, auto rhs) noexcept + /** @brief Adds lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(auto lhs, auto rhs) noexcept { return _mm256_adds_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_saturated(auto lhs, auto rhs) noexcept + /** @brief Subtracts lanes with saturation for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(auto lhs, auto rhs) noexcept { return _mm256_subs_epu16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_saturated(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) multiply_saturated(auto lhs, auto rhs) noexcept { const __m128i lhsLo128 = _mm256_castsi256_si128(lhs); const __m128i rhsLo128 = _mm256_castsi256_si128(rhs); @@ -3219,257 +5097,445 @@ template <> struct SimdImpl256 } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm256_set1_epi16(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm256_set_epi16(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm256_setr_epi16(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm256_cmpeq_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext256_cmpgt_epu16(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm256_cvtepu16_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL compress(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) compress(auto lhs, auto rhs) noexcept { return _mm256_packus_epi16(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm256_extract_epi16(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected unsigned 16-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 16)`. + * @return Selected scalar lane. + */ + static uint16_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 16, "Unsigned 16-bit extraction requires a valid 256-bit lane index"); + const __m256i selected = _mm256_permutevar8x32_epi32(lhs, _mm256_set1_epi32(index >> 1)); + const uint32_t selected_dword = static_cast(_mm_cvtsi128_si32(_mm256_castsi256_si128(selected))); + return static_cast(selected_dword >> ((index & 1) * 16)); + } + /** @brief Replaces the compile-time-selected unsigned 16-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const uint16_t rhs) noexcept + { + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected unsigned 16-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const uint16_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return _mm256_insert_epi16(lhs, static_cast(rhs), index); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int imm8) noexcept + /** + * @brief Replaces one runtime-selected unsigned 16-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 16)`. + * @return Register with the selected lane replaced. + */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256i lhs, const uint16_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(imm8)); + SIMDLIB_PRECONDITION(index >= 0 && index < 16, "Unsigned 16-bit insertion requires a valid 256-bit lane index"); + if (index < 8) + { + const __m128i lower = SimdImpl128::insert_slow(_mm256_castsi256_si128(lhs), rhs, index); + return _mm256_inserti128_si256(lhs, lower, 0); + } + const __m128i upper = SimdImpl128::insert_slow(_mm256_extracti128_si256(lhs, 1), rhs, index - 8); + return _mm256_inserti128_si256(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_epi16(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_lo(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled low-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each low four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_lo_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_half_16_slow(lhs, static_cast(rhs), false); + } + /** @brief Shuffles the low four unsigned 16-bit lanes in each 128-bit group with an immediate control. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_lo(__m256i lhs) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), false); + return _mm256_shufflelo_epi16(lhs, imm8); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_hi(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled high-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each high four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_hi_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_half_16_slow(lhs, static_cast(rhs), true); + } + /** @brief Shuffles the high four unsigned 16-bit lanes in each 128-bit group with an immediate control. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_hi(__m256i lhs) noexcept + { + return _mm256_shufflehi_epi16(lhs, imm8); + } + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_half_16(lhs, static_cast(rhs), true); + return register_blend_slow(lhs, rhs, static_cast(imm8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects unsigned 16-bit lanes from two 256-bit registers with a repeated immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m256i lhs, const __m256i rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm256_blend_epi16(lhs, rhs, imm8); } }; template <> struct SimdImpl256 { + /** @brief Selects 32-bit lanes from two registers using a canonical predicate register. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256i condition, __m256i when_true, __m256i when_false) noexcept + { + return _mm256_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical signed 32-bit lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 8 && ((indices < 8) && ...)) + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256i lhs) noexcept + { + return _mm256_permutevar8x32_epi32(lhs, _mm256_setr_epi32(static_cast(indices)...)); + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m256i evenProducts = _mm256_mul_epi32(lhs, rhs); const __m256i oddProducts = _mm256_mul_epi32(_mm256_srli_si256(lhs, 4), _mm256_srli_si256(rhs, 4)); return _mm256_add_epi64(evenProducts, oddProducts); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm256_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm256_mullo_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding signed 32-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext256_div_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes scalar-equivalent signed 32-bit remainders with register-only extraction and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext256_rem_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { const __m256 roots = _mm256_sqrt_ps(_mm256_cvtepi32_ps(lhs)); return _mm256_cvtps_epi32(roots); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes one unchecked magnitude in lane zero of each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Computes saturated magnitudes and adjacent overflow masks for both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude_checked(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude_checked(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Returns the minimum value and its first lane position without materializing register data in memory. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { const __m128i lowMeta = SimdImpl128::min_position(_mm256_castsi256_si128(lhs)); const __m128i highMeta = SimdImpl128::min_position(_mm256_extracti128_si256(lhs, 1)); - alignas(16) std::array lowData{}; - alignas(16) std::array highData{}; - alignas(32) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(lowData.data()), lowMeta); - _mm_store_si128(reinterpret_cast<__m128i *>(highData.data()), highMeta); - highData[1] += 4; - output[0] = highData[0] < lowData[0] ? highData[0] : lowData[0]; - output[1] = highData[0] < lowData[0] ? highData[1] : lowData[1]; - return _mm256_load_si256(reinterpret_cast(output.data())); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + const auto lowValue = static_cast(_mm_extract_epi32(lowMeta, 0)); + const auto highValue = static_cast(_mm_extract_epi32(highMeta, 0)); + const bool chooseHigh = highValue < lowValue; + const __m128i selectedMeta = chooseHigh ? highMeta : lowMeta; + const int position = static_cast(_mm_extract_epi32(selectedMeta, 1)) + (chooseHigh ? 4 : 0); + __m128i output = _mm_setzero_si128(); + output = _mm_insert_epi32(output, _mm_extract_epi32(selectedMeta, 0), 0); + output = _mm_insert_epi32(output, position, 1); + return _mm256_zextsi128_si256(output); + } + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm256_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m256i lhs, __m256i rhs) noexcept { return _mm256_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm256_abs_epi32(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm256_min_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm256_max_epi32(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm256_slli_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm256_srli_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _mm256_srai_epi32(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hadd_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hsub_epi32(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm256_set1_epi32(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm256_set_epi32(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm256_setr_epi32(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm256_cmpeq_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm256_cmpgt_epi32(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm256_cvtepi32_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL compress(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) compress(auto lhs, auto rhs) noexcept { return _mm256_packs_epi32(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm256_extract_epi32(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected signed 32-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 8)`. + * @return Selected scalar lane. + */ + static int32_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256i lhs, const int index) noexcept { - return register_get(lhs, static_cast(rhs)); + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "Signed 32-bit extraction requires a valid 256-bit lane index"); + const __m256i selected = _mm256_permutevar8x32_epi32(lhs, _mm256_set1_epi32(index)); + return _mm_cvtsi128_si32(_mm256_castsi256_si128(selected)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Replaces the compile-time-selected signed 32-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const int32_t rhs) noexcept { - return register_insert(lhs, rhs, static_cast(imm8)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected signed 32-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const int32_t rhs) noexcept + { + return _mm256_insert_epi32(lhs, rhs, index); + } + /** + * @brief Replaces one runtime-selected signed 32-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 8)`. + * @return Register with the selected lane replaced. + */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256i lhs, const int32_t rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "Signed 32-bit insertion requires a valid 256-bit lane index"); + if (index < 4) + { + const __m128i lower = SimdImpl128::insert_slow(_mm256_castsi256_si128(lhs), rhs, index); + return _mm256_inserti128_si256(lhs, lower, 0); + } + const __m128i upper = SimdImpl128::insert_slow(_mm256_extracti128_si256(lhs, 1), rhs, index - 4); + return _mm256_inserti128_si256(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_epi32(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_lo(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled low-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each low four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_lo_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_32_slow(lhs, static_cast(rhs)); + } + /** @brief Emulates an immediate-controlled high-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each high four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_hi_slow(auto lhs, auto rhs) noexcept { - return register_shuffle_32(lhs, static_cast(rhs)); + return register_shuffle_32_slow(lhs, static_cast(rhs)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_hi(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_32(lhs, static_cast(rhs)); + return register_blend_slow(lhs, rhs, static_cast(imm8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects signed 32-bit lanes from two 256-bit registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m256i lhs, const __m256i rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm256_blend_epi32(lhs, rhs, imm8); } }; template <> struct SimdImpl256 { + /** @brief Selects 32-bit lanes from two registers using a canonical predicate register. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256i condition, __m256i when_true, __m256i when_false) noexcept + { + return _mm256_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical unsigned 32-bit lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 8 && ((indices < 8) && ...)) + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256i lhs) noexcept + { + return _mm256_permutevar8x32_epi32(lhs, _mm256_setr_epi32(static_cast(indices)...)); + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_epi32(lhs, rhs); } @@ -3479,37 +5545,42 @@ template <> struct SimdImpl256 * @param lhs The unsigned integer lanes. * @return The converted floating-point lanes. */ - SIMDLIB_FORCE_INLINE static auto VECTORCALL convert_to_float(const __m256i lhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) convert_to_float(const __m256i lhs) noexcept { return _ext256_cvtepu32_ps(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m256i evenProducts = _mm256_mul_epu32(lhs, rhs); const __m256i oddProducts = _mm256_mul_epu32(_mm256_srli_si256(lhs, 4), _mm256_srli_si256(rhs, 4)); return _mm256_add_epi64(evenProducts, oddProducts); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm256_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm256_mullo_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding unsigned 32-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left / right; }); + return _ext256_div_epu32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes scalar-equivalent unsigned 32-bit remainders with register-only extraction and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { - return register_transform_binary(lhs, rhs, [](auto left, auto right) noexcept { return left % right; }); + return _ext256_rem_epu32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { const __m128i low = _mm256_castsi256_si128(lhs); const __m128i high = _mm256_extracti128_si256(lhs, 1); @@ -3519,295 +5590,442 @@ template <> struct SimdImpl256 const __m128i highInts = _mm_cvtps_epi32(highRoots); return _mm256_inserti128_si256(_mm256_castsi128_si256(lowInts), highInts, 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes one unchecked magnitude in lane zero of each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Computes saturated magnitudes and adjacent overflow masks for both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude_checked(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude_checked(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Returns the minimum value and its first lane position without materializing register data in memory. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { const __m128i lowMeta = SimdImpl128::min_position(_mm256_castsi256_si128(lhs)); const __m128i highMeta = SimdImpl128::min_position(_mm256_extracti128_si256(lhs, 1)); - alignas(16) std::array lowData{}; - alignas(16) std::array highData{}; - alignas(32) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(lowData.data()), lowMeta); - _mm_store_si128(reinterpret_cast<__m128i *>(highData.data()), highMeta); - highData[1] += 4; - output[0] = highData[0] < lowData[0] ? highData[0] : lowData[0]; - output[1] = highData[0] < lowData[0] ? highData[1] : lowData[1]; - return _mm256_load_si256(reinterpret_cast(output.data())); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + const auto lowValue = static_cast(_mm_extract_epi32(lowMeta, 0)); + const auto highValue = static_cast(_mm_extract_epi32(highMeta, 0)); + const bool chooseHigh = highValue < lowValue; + const __m128i selectedMeta = chooseHigh ? highMeta : lowMeta; + const int position = static_cast(_mm_extract_epi32(selectedMeta, 1)) + (chooseHigh ? 4 : 0); + __m128i output = _mm_setzero_si128(); + output = _mm_insert_epi32(output, _mm_extract_epi32(selectedMeta, 0), 0); + output = _mm_insert_epi32(output, position, 1); + return _mm256_zextsi128_si256(output); + } + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm256_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m256i lhs, __m256i rhs) noexcept { return _mm256_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _mm256_abs_epi32(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm256_min_epu32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm256_max_epu32(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm256_slli_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm256_srli_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _mm256_srai_epi32(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hadd_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hsub_epi32(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm256_set1_epi32(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm256_set_epi32(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm256_setr_epi32(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm256_cmpeq_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext256_cmpgt_epu32(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm256_cvtepu32_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL compress(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) compress(auto lhs, auto rhs) noexcept { return _mm256_packus_epi32(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm256_extract_epi32(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected unsigned 32-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 8)`. + * @return Selected scalar lane. + */ + static uint32_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256i lhs, const int index) noexcept { - return register_get(lhs, static_cast(rhs)); + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "Unsigned 32-bit extraction requires a valid 256-bit lane index"); + const __m256i selected = _mm256_permutevar8x32_epi32(lhs, _mm256_set1_epi32(index)); + return static_cast(_mm_cvtsi128_si32(_mm256_castsi256_si128(selected))); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** @brief Replaces the compile-time-selected unsigned 32-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const uint32_t rhs) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected unsigned 32-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const uint32_t rhs) noexcept + { + return _mm256_insert_epi32(lhs, std::bit_cast(rhs), index); + } + /** + * @brief Replaces one runtime-selected unsigned 32-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 8)`. + * @return Register with the selected lane replaced. + */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256i lhs, const uint32_t rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "Unsigned 32-bit insertion requires a valid 256-bit lane index"); + if (index < 4) + { + const __m128i lower = SimdImpl128::insert_slow(_mm256_castsi256_si128(lhs), rhs, index); + return _mm256_inserti128_si256(lhs, lower, 0); + } + const __m128i upper = SimdImpl128::insert_slow(_mm256_extracti128_si256(lhs, 1), rhs, index - 4); + return _mm256_inserti128_si256(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_epi32(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_epi32(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_lo(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled low-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each low four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_lo_slow(auto lhs, auto rhs) noexcept + { + return register_shuffle_32_slow(lhs, static_cast(rhs)); + } + /** @brief Emulates an immediate-controlled high-half shuffle with a runtime scalar control. + * @param lhs Source register. + * @param rhs Runtime control byte. + * @return Register with each high four-lane group shuffled. + */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) shuffle_hi_slow(auto lhs, auto rhs) noexcept { - return register_shuffle_32(lhs, static_cast(rhs)); + return register_shuffle_32_slow(lhs, static_cast(rhs)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle_hi(auto lhs, auto rhs) noexcept + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_32(lhs, static_cast(rhs)); + return register_blend_slow(lhs, rhs, static_cast(imm8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects unsigned 32-bit lanes from two 256-bit registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m256i lhs, const __m256i rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm256_blend_epi32(lhs, rhs, imm8); } }; template <> struct SimdImpl256 { + /** @brief Selects 64-bit lanes from two registers using a canonical predicate register. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256i condition, __m256i when_true, __m256i when_false) noexcept + { + return _mm256_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical signed 64-bit lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 4 && ((indices < 4) && ...)) + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256i lhs) noexcept + { + return _mm256_permute4x64_epi64(lhs, encode_logical_shuffle_32_immediate()); + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m128i low = SimdImpl128::multiply_add_adjacent(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); const __m128i high = SimdImpl128::multiply_add_adjacent(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); return _mm256_inserti128_si256(_mm256_castsi128_si256(low), high, 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm256_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _ext256_mullo_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding signed 64-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { return _ext256_div_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes scalar-equivalent signed 64-bit remainders with register-only extraction and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { return _ext256_rem_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes integer square roots lane-wise using register extracts and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { - auto sqrt64x2 = [](__m128i values) noexcept - { - alignas(16) std::int64_t input[2]; - _mm_storeu_si128(reinterpret_cast<__m128i *>(input), values); - const __m128d roots = _mm_sqrt_pd(_mm_setr_pd(static_cast(input[0]), static_cast(input[1]))); - alignas(16) double result[2]; - _mm_storeu_pd(result, roots); - return register_from_values<__m128i, std::int64_t>(static_cast(result[0]), static_cast(result[1])); - }; - - const __m128i low = _mm256_castsi256_si128(lhs); - const __m128i high = _mm256_extracti128_si256(lhs, 1); - const __m128i lowRoots = sqrt64x2(low); - const __m128i highRoots = sqrt64x2(high); + const __m128i lowRoots = SimdImpl128::sqrt(_mm256_castsi256_si128(lhs)); + const __m128i highRoots = SimdImpl128::sqrt(_mm256_extracti128_si256(lhs, 1)); return _mm256_inserti128_si256(_mm256_castsi128_si256(lowRoots), highRoots, 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes one unchecked magnitude in lane zero of each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Computes saturated magnitudes and adjacent overflow masks for both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude_checked(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude_checked(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Returns the minimum value and its first lane position without materializing register data in memory. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { const __m128i lowMeta = SimdImpl128::min_position(_mm256_castsi256_si128(lhs)); const __m128i highMeta = SimdImpl128::min_position(_mm256_extracti128_si256(lhs, 1)); - alignas(16) std::array lowData{}; - alignas(16) std::array highData{}; - alignas(32) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(lowData.data()), lowMeta); - _mm_store_si128(reinterpret_cast<__m128i *>(highData.data()), highMeta); - highData[1] += 2; - output[0] = highData[0] < lowData[0] ? highData[0] : lowData[0]; - output[1] = highData[0] < lowData[0] ? highData[1] : lowData[1]; - return _mm256_load_si256(reinterpret_cast(output.data())); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + const auto lowValue = static_cast(_mm_extract_epi64(lowMeta, 0)); + const auto highValue = static_cast(_mm_extract_epi64(highMeta, 0)); + const bool chooseHigh = highValue < lowValue; + const __m128i selectedMeta = chooseHigh ? highMeta : lowMeta; + const int position = static_cast(_mm_extract_epi64(selectedMeta, 1)) + (chooseHigh ? 2 : 0); + __m128i output = _mm_setzero_si128(); + output = _mm_insert_epi64(output, _mm_extract_epi64(selectedMeta, 0), 0); + output = _mm_insert_epi64(output, position, 1); + return _mm256_zextsi128_si256(output); + } + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm256_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m256i lhs, __m256i rhs) noexcept { return _mm256_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _ext256_abs_epi64(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _ext256_min_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _ext256_max_epi64(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm256_slli_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm256_srli_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _ext256_srai_epi64(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm256_set1_epi64x(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm256_set_epi64x(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm256_setr_epi64x(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm256_cmpeq_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _mm256_cmpgt_epi64(lhs, rhs); } // conversion - // static SIMDLIB_FORCE_INLINE auto VECTORCALL expand (auto lhs, auto rhs) noexcept { return _mm256_cvtepi64_epi128(lhs, rhs); } + // static auto SIMD_FLAGS(InOut, ForceInline) expand (auto lhs, auto rhs) noexcept { return _mm256_cvtepi64_epi128(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm256_extract_epi64(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected signed 64-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 4)`. + * @return Selected scalar lane. + */ + static int64_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "Signed 64-bit extraction requires a valid 256-bit lane index"); + const __m256i first_word = _mm256_set1_epi32(index * 2); + const __m256i word_offsets = _mm256_setr_epi32(0, 1, 0, 1, 0, 1, 0, 1); + const __m256i selected = _mm256_permutevar8x32_epi32(lhs, _mm256_add_epi32(first_word, word_offsets)); + return SimdImpl128::template extract<0>(_mm256_castsi256_si128(selected)); + } + /** @brief Replaces the compile-time-selected signed 64-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const int64_t rhs) noexcept + { + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected signed 64-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const int64_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return _mm256_insert_epi64(lhs, rhs, index); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** + * @brief Replaces one runtime-selected signed 64-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 4)`. + * @return Register with the selected lane replaced. + */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256i lhs, const int64_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "Signed 64-bit insertion requires a valid 256-bit lane index"); + if (index < 2) + { + const __m128i lower = SimdImpl128::insert_slow(_mm256_castsi256_si128(lhs), rhs, index); + return _mm256_inserti128_si256(lhs, lower, 0); + } + const __m128i upper = SimdImpl128::insert_slow(_mm256_extracti128_si256(lhs, 1), rhs, index - 2); + return _mm256_inserti128_si256(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_epi64(lhs, rhs); } @@ -3815,156 +6033,224 @@ template <> struct SimdImpl256 template <> struct SimdImpl256 { + /** @brief Selects 64-bit lanes from two registers using a canonical predicate register. */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256i condition, __m256i when_true, __m256i when_false) noexcept + { + return _mm256_blendv_epi8(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical unsigned 64-bit lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 4 && ((indices < 4) && ...)) + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256i lhs) noexcept + { + return _mm256_permute4x64_epi64(lhs, encode_logical_shuffle_32_immediate()); + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_adjacent(auto lhs, auto rhs) noexcept + /** @brief Multiplies adjacent lanes and adds their products for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_adjacent(auto lhs, auto rhs) noexcept { const __m128i low = SimdImpl128::multiply_add_adjacent(_mm256_castsi256_si128(lhs), _mm256_castsi256_si128(rhs)); const __m128i high = SimdImpl128::multiply_add_adjacent(_mm256_extracti128_si256(lhs, 1), _mm256_extracti128_si256(rhs, 1)); return _mm256_inserti128_si256(_mm256_castsi128_si256(low), high, 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept + /** @brief Multiplies unsigned and signed byte pairs for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(auto lhs, auto rhs) noexcept { return _mm256_maddubs_epi16(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _ext256_mullo_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + /** @brief Divides corresponding unsigned 64-bit lanes with scalar instructions and intrinsic reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { return _ext256_div_epu64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL modulus(auto lhs, auto rhs) noexcept + /** @brief Computes scalar-equivalent unsigned 64-bit remainders with register-only extraction and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) modulus(auto lhs, auto rhs) noexcept { return _ext256_rem_epu64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes integer square roots lane-wise using register extracts and reconstruction. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { - auto sqrt64x2 = [](__m128i values) noexcept - { - alignas(16) std::uint64_t input[2]; - _mm_storeu_si128(reinterpret_cast<__m128i *>(input), values); - const __m128d roots = _mm_sqrt_pd(_mm_setr_pd(static_cast(input[0]), static_cast(input[1]))); - alignas(16) double result[2]; - _mm_storeu_pd(result, roots); - return register_from_values<__m128i, std::int64_t>(static_cast(result[0]), static_cast(result[1])); - }; - - const __m128i low = _mm256_castsi256_si128(lhs); - const __m128i high = _mm256_extracti128_si256(lhs, 1); - const __m128i lowRoots = sqrt64x2(low); - const __m128i highRoots = sqrt64x2(high); + const __m128i lowRoots = SimdImpl128::sqrt(_mm256_castsi256_si128(lhs)); + const __m128i highRoots = SimdImpl128::sqrt(_mm256_extracti128_si256(lhs, 1)); return _mm256_inserti128_si256(_mm256_castsi128_si256(lowRoots), highRoots, 1); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min_position(auto lhs) noexcept + /** @brief Computes one unchecked magnitude in lane zero of each 128-bit group. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Computes saturated magnitudes and adjacent overflow masks for both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(auto lhs) noexcept + { + const __m128i lowMagnitude = SimdImpl128::magnitude_checked(_mm256_castsi256_si128(lhs)); + const __m128i highMagnitude = SimdImpl128::magnitude_checked(_mm256_extracti128_si256(lhs, 1)); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lowMagnitude), highMagnitude, 1); + } + + /** @brief Returns the minimum value and its first lane position without materializing register data in memory. */ + static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(auto lhs) noexcept { const __m128i lowMeta = SimdImpl128::min_position(_mm256_castsi256_si128(lhs)); const __m128i highMeta = SimdImpl128::min_position(_mm256_extracti128_si256(lhs, 1)); - alignas(16) std::array lowData{}; - alignas(16) std::array highData{}; - alignas(32) std::array output{}; - _mm_store_si128(reinterpret_cast<__m128i *>(lowData.data()), lowMeta); - _mm_store_si128(reinterpret_cast<__m128i *>(highData.data()), highMeta); - highData[1] += 2; - output[0] = highData[0] < lowData[0] ? highData[0] : lowData[0]; - output[1] = highData[0] < lowData[0] ? highData[1] : lowData[1]; - return _mm256_load_si256(reinterpret_cast(output.data())); - } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + const auto lowValue = static_cast(_mm_extract_epi64(lowMeta, 0)); + const auto highValue = static_cast(_mm_extract_epi64(highMeta, 0)); + const bool chooseHigh = highValue < lowValue; + const __m128i selectedMeta = chooseHigh ? highMeta : lowMeta; + const int position = static_cast(_mm_extract_epi64(selectedMeta, 1)) + (chooseHigh ? 2 : 0); + __m128i output = _mm_setzero_si128(); + output = _mm_insert_epi64(output, _mm_extract_epi64(selectedMeta, 0), 0); + output = _mm_insert_epi64(output, position, 1); + return _mm256_zextsi128_si256(output); + } + /** @brief Computes byte-wise absolute-difference sums for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sum_absolute_byte_differences(auto lhs, auto rhs) noexcept { return _mm256_sad_epu8(lhs, rhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL multi_sum_absolute_byte_differences(auto lhs, auto rhs) noexcept + /** @brief Computes immediate-selected byte-window absolute-difference sums for this native register specialization. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multi_sum_absolute_byte_differences(__m256i lhs, __m256i rhs) noexcept { return _mm256_mpsadbw_epu8(lhs, rhs, imm8); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return lhs; } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _ext256_min_epu64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _ext256_max_epu64(lhs, rhs); } // shifting - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_left(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_left(auto lhs, auto rhs) noexcept { return _mm256_slli_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right(auto lhs, auto rhs) noexcept { return _mm256_srli_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL shift_right_arithmetic(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) shift_right_arithmetic(auto lhs, auto rhs) noexcept { return _ext256_srai_epi64(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm256_set1_epi64x(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm256_set_epi64x(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm256_setr_epi64x(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _mm256_cmpeq_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext256_cmpgt_epu64(lhs, rhs); } // conversion - // static SIMDLIB_FORCE_INLINE auto VECTORCALL expand (auto lhs, auto rhs) noexcept { return _mm256_cvtepu64_epi128(lhs, rhs); } + // static auto SIMD_FLAGS(InOut, ForceInline) expand (auto lhs, auto rhs) noexcept { return _mm256_cvtepu64_epi128(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { return static_cast(_mm256_extract_epi64(lhs, index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected unsigned 64-bit lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 4)`. + * @return Selected scalar lane. + */ + static uint64_t SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256i lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "Unsigned 64-bit extraction requires a valid 256-bit lane index"); + const __m256i first_word = _mm256_set1_epi32(index * 2); + const __m256i word_offsets = _mm256_setr_epi32(0, 1, 0, 1, 0, 1, 0, 1); + const __m256i selected = _mm256_permutevar8x32_epi32(lhs, _mm256_add_epi32(first_word, word_offsets)); + return SimdImpl128::template extract<0>(_mm256_castsi256_si128(selected)); + } + /** @brief Replaces the compile-time-selected unsigned 64-bit lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const uint64_t rhs) noexcept + { + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected unsigned 64-bit lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const uint64_t rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return _mm256_insert_epi64(lhs, std::bit_cast(rhs), index); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** + * @brief Replaces one runtime-selected unsigned 64-bit lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane in the range `[0, 4)`. + * @return Register with the selected lane replaced. + */ + static __m256i SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256i lhs, const uint64_t rhs, const int index) noexcept { - return register_insert(lhs, rhs, static_cast(index)); + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "Unsigned 64-bit insertion requires a valid 256-bit lane index"); + if (index < 2) + { + const __m128i lower = SimdImpl128::insert_slow(_mm256_castsi256_si128(lhs), rhs, index); + return _mm256_inserti128_si256(lhs, lower, 0); + } + const __m128i upper = SimdImpl128::insert_slow(_mm256_extracti128_si256(lhs, 1), rhs, index - 2); + return _mm256_inserti128_si256(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_epi64(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_epi64(lhs, rhs); } @@ -3972,32 +6258,59 @@ template <> struct SimdImpl256 template <> struct SimdImpl256 { + /** @brief Selects float lanes from two registers using a canonical predicate register. */ + static __m256 SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256 condition, __m256 when_true, __m256 when_false) noexcept + { + return _mm256_blendv_ps(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical floating-point lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 8 && ((indices < 8) && ...)) + static __m256 SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256 lhs) noexcept + { + return _mm256_permutevar8x32_ps(lhs, _mm256_setr_epi32(static_cast(indices)...)); + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_subtract(auto lhs, auto rhs) noexcept + /** @brief Alternates lane subtraction and addition for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_subtract(auto lhs, auto rhs) noexcept { return _mm256_addsub_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm256_mul_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { return _mm256_div_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { return _mm256_sqrt_ps(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add(auto lhs, auto rhs, auto addend) noexcept + /** @brief Computes and broadcasts floating-point magnitudes independently in both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + return _mm256_sqrt_ps(_mm256_dp_ps(lhs, lhs, 0xFF)); + } + /** @brief Multiplies lanes and adds a third register for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add(auto lhs, auto rhs, auto addend) noexcept { #if SIMDLIB_HAS_FMA return _mm256_fmadd_ps(lhs, rhs, addend); @@ -4005,7 +6318,8 @@ template <> struct SimdImpl256 return _mm256_add_ps(_mm256_mul_ps(lhs, rhs), addend); #endif } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL dot_product(auto lhs, auto rhs) noexcept + /** @brief Computes an immediate-controlled dot product for this native register specialization. */ + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) dot_product(__m256 lhs, __m256 rhs) noexcept { const __m128 lhsLow = _mm256_castps256_ps128(lhs); const __m128 lhsHigh = _mm256_extractf128_ps(lhs, 1); @@ -4016,127 +6330,228 @@ template <> struct SimdImpl256 return _mm256_insertf128_ps(_mm256_castps128_ps256(dotLow), dotHigh, 1); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _ext256_abs_ps(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm256_min_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm256_max_ps(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hadd_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hsub_ps(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm256_set1_ps(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm256_set_ps(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm256_setr_ps(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _ext256_cmpeq_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext256_cmpgt_ps(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm256_cvtps_epi32(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { - return static_cast(_ext256_extract_ps(lhs, index)); + constexpr int half_index = index / 4; + constexpr int lane_index = index % 4; + const __m128 half = [&]() + { + if constexpr (half_index == 0) + return _mm256_castps256_ps128(lhs); + else + return _mm256_extractf128_ps(lhs, half_index); + }(); + return _mm_cvtss_f32(_mm_shuffle_ps(half, half, lane_index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected 32-bit floating-point lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 8)`. + * @return Selected scalar lane. + */ + static float SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256 lhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "32-bit floating-point extraction requires a valid 256-bit lane index"); + const __m256 selected = _mm256_permutevar8x32_ps(lhs, _mm256_set1_epi32(index)); + return SimdImpl128::template extract<0>(_mm256_castps256_ps128(selected)); + } + /** @brief Replaces the compile-time-selected 32-bit floating-point lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const float rhs) noexcept { - return register_get(lhs, static_cast(rhs)); + return register_insert_constexpr(lhs, rhs, static_cast(index)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** @brief Replaces the compile-time-selected 32-bit floating-point lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const float rhs) noexcept { - return _ext256_insert_ps(lhs, rhs, index); + constexpr int half_index = index / 4; + constexpr int lane_index = index % 4; + __m128 half; + if constexpr (half_index == 0) + half = _mm256_castps256_ps128(lhs); + else + half = _mm256_extractf128_ps(lhs, half_index); + half = _mm_insert_ps(half, _mm_set_ss(rhs), lane_index << 4); + return _mm256_insertf128_ps(lhs, half, half_index); + } + /** + * @brief Replaces one runtime-selected 32-bit floating-point lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane index. + * @return Register with the selected lane replaced. + */ + static __m256 SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256 lhs, const float rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 8, "32-bit floating-point insertion requires a valid 256-bit lane index"); + if (index < 4) + { + const __m128 lower = SimdImpl128::insert_slow(_mm256_castps256_ps128(lhs), rhs, index); + return _mm256_insertf128_ps(lhs, lower, 0); + } + const __m128 upper = SimdImpl128::insert_slow(_mm256_extractf128_ps(lhs, 1), rhs, index - 4); + return _mm256_insertf128_ps(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_ps(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_ps(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Emulates an immediate-controlled floating shuffle with a runtime scalar control. + * @param lhs Source for the lower selected lanes in each group. + * @param rhs Source for the upper selected lanes in each group. + * @param imm8 Runtime control byte. + * @return Register containing the shuffled lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) shuffle_slow(auto lhs, auto rhs, const int imm8) noexcept + { + return register_shuffle_float_slow(lhs, rhs, imm8); + } + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_float(lhs, rhs, imm8); + return register_blend_slow(lhs, rhs, static_cast(imm8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects 32-bit floating-point lanes from two 256-bit registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m256 lhs, const __m256 rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm256_blend_ps(lhs, rhs, imm8); } }; template <> struct SimdImpl256 { + /** @brief Selects double lanes from two registers using a canonical predicate register. */ + static __m256d SIMD_FLAGS(InOut, RegisterOnly, ForceInline) select(__m256d condition, __m256d when_true, __m256d when_false) noexcept + { + return _mm256_blendv_pd(when_false, when_true, condition); + } + + /** + * @brief Shuffles logical double-precision floating-point lanes across the complete 256-bit register. + * @tparam indices Source lane for each result lane in low-to-high order. + * @param lhs Source register. + * @return Register containing the selected logical lanes. + */ + template + requires(sizeof...(indices) == 4 && ((indices < 4) && ...)) + static __m256d SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(__m256d lhs) noexcept + { + return _mm256_permute4x64_pd(lhs, encode_logical_shuffle_32_immediate()); + } + // arithmetic - SIMDLIB_FORCE_INLINE static auto VECTORCALL add(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add(auto lhs, auto rhs) noexcept { return _mm256_add_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_subtract(auto lhs, auto rhs) noexcept + /** @brief Alternates lane subtraction and addition for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_subtract(auto lhs, auto rhs) noexcept { return _mm256_addsub_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract(auto lhs, auto rhs) noexcept { return _mm256_sub_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply(auto lhs, auto rhs) noexcept { return _mm256_mul_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL divide(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) divide(auto lhs, auto rhs) noexcept { return _mm256_div_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL sqrt(auto lhs) noexcept + /** @brief Computes lane-wise square roots for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(auto lhs) noexcept { return _mm256_sqrt_pd(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL multiply_add(auto lhs, auto rhs, auto addend) noexcept + /** @brief Computes and broadcasts floating-point magnitudes independently in both 128-bit groups. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(auto lhs) noexcept + { + const __m256d squares = _mm256_mul_pd(lhs, lhs); + return _mm256_sqrt_pd(_mm256_hadd_pd(squares, squares)); + } + /** @brief Multiplies lanes and adds a third register for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add(auto lhs, auto rhs, auto addend) noexcept { #if SIMDLIB_HAS_FMA return _mm256_fmadd_pd(lhs, rhs, addend); @@ -4144,7 +6559,8 @@ template <> struct SimdImpl256 return _mm256_add_pd(_mm256_mul_pd(lhs, rhs), addend); #endif } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL dot_product(auto lhs, auto rhs) noexcept + /** @brief Computes an immediate-controlled dot product for this native register specialization. */ + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) dot_product(__m256d lhs, __m256d rhs) noexcept { const __m128d lhsLow = _mm256_castpd256_pd128(lhs); const __m128d lhsHigh = _mm256_extractf128_pd(lhs, 1); @@ -4155,96 +6571,178 @@ template <> struct SimdImpl256 return _mm256_insertf128_pd(_mm256_castpd128_pd256(dotLow), dotHigh, 1); } // - SIMDLIB_FORCE_INLINE static auto VECTORCALL absolute(auto lhs) noexcept + /** @brief Computes lane-wise absolute values for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(auto lhs) noexcept { return _ext256_abs_pd(lhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL negate(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) negate(auto lhs, auto rhs) noexcept { return _mm256_sub_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL min(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise minima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(auto lhs, auto rhs) noexcept { return _mm256_min_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL max(auto lhs, auto rhs) noexcept + /** @brief Computes lane-wise maxima for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(auto lhs, auto rhs) noexcept { return _mm256_max_pd(lhs, rhs); } // arithmetic (horizontal) - SIMDLIB_FORCE_INLINE static auto VECTORCALL add_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally adds adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hadd_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL subtract_horizontal(auto lhs, auto rhs) noexcept + /** @brief Horizontally subtracts adjacent lanes for this native register specialization. */ + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_horizontal(auto lhs, auto rhs) noexcept { return _mm256_hsub_pd(lhs, rhs); } // loading - SIMDLIB_FORCE_INLINE static auto VECTORCALL set1(auto lhs) noexcept + static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set1(auto lhs) noexcept { return _mm256_set1_pd(lhs); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL set(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) set(Args... args) noexcept { return _mm256_set_pd(args...); } - template SIMDLIB_FORCE_INLINE static auto VECTORCALL setr(Args... args) noexcept + template static auto SIMD_FLAGS(Out, RegisterOnly, ForceInline) setr(Args... args) noexcept { return _mm256_setr_pd(args...); } // comparison - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpeq(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpeq(auto lhs, auto rhs) noexcept { return _ext256_cmpeq_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL cmpgt(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) cmpgt(auto lhs, auto rhs) noexcept { return _ext256_cmpgt_pd(lhs, rhs); } // conversion - SIMDLIB_FORCE_INLINE static auto VECTORCALL expand(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) expand(auto lhs, auto rhs) noexcept { return _mm256_cvtps_epi32(lhs, rhs); } // extract / insert - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs) noexcept + template static auto SIMD_FLAGS(In, RegisterOnly, ForceInline) extract(auto lhs) noexcept { - return register_get(lhs, static_cast(index)); + constexpr int half_index = index / 2; + constexpr int lane_index = index % 2; + const __m128d half = [&]() + { + if constexpr (half_index == 0) + return _mm256_castpd256_pd128(lhs); + else + return _mm256_extractf128_pd(lhs, half_index); + }(); + if constexpr (lane_index == 0) + return _mm_cvtsd_f64(half); + else + return _mm_cvtsd_f64(_mm_unpackhi_pd(half, half)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(auto lhs, auto rhs) noexcept + /** + * @brief Extracts one runtime-selected 64-bit floating-point lane. + * @param lhs Source register. + * @param index Selected lane in the range `[0, 4)`. + * @return Selected scalar lane. + */ + static double SIMD_FLAGS(In, RegisterOnly, ForceInline) extract_slow(const __m256d lhs, const int index) noexcept { - return register_get(lhs, static_cast(rhs)); + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "64-bit floating-point extraction requires a valid 256-bit lane index"); + const __m256i first_word = _mm256_set1_epi32(index * 2); + const __m256i word_offsets = _mm256_setr_epi32(0, 1, 0, 1, 0, 1, 0, 1); + const __m256i selected = _mm256_permutevar8x32_epi32(_mm256_castpd_si256(lhs), _mm256_add_epi32(first_word, word_offsets)); + return SimdImpl128::template extract<0>(_mm_castsi128_pd(_mm256_castsi256_si128(selected))); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL insert(auto lhs, auto rhs, const int index) noexcept + /** @brief Replaces the compile-time-selected 64-bit floating-point lane during constant evaluation. */ + template [[nodiscard]] constexpr static auto insert_constexpr(auto lhs, const double rhs) noexcept { - return _ext256_insert_pd(lhs, rhs, index); + return register_insert_constexpr(lhs, rhs, static_cast(index)); + } + /** @brief Replaces the compile-time-selected 64-bit floating-point lane. */ + template static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert(auto lhs, const double rhs) noexcept + { + constexpr int half_index = index / 2; + constexpr int lane_index = index % 2; + __m128d half; + if constexpr (half_index == 0) + half = _mm256_castpd256_pd128(lhs); + else + half = _mm256_extractf128_pd(lhs, half_index); + const __m128d replacement = _mm_set_sd(rhs); + if constexpr (lane_index == 0) + half = _mm_move_sd(half, replacement); + else + half = _mm_unpacklo_pd(half, replacement); + return _mm256_insertf128_pd(lhs, half, half_index); + } + /** + * @brief Replaces one runtime-selected 64-bit floating-point lane. + * @param lhs Source register. + * @param rhs Replacement scalar lane. + * @param index Selected lane index. + * @return Register with the selected lane replaced. + */ + static __m256d SIMD_FLAGS(InOut, RegisterOnly, ForceInline) insert_slow(const __m256d lhs, const double rhs, const int index) noexcept + { + SIMDLIB_PRECONDITION(index >= 0 && index < 4, "64-bit floating-point insertion requires a valid 256-bit lane index"); + if (index < 2) + { + const __m128d lower = SimdImpl128::insert_slow(_mm256_castpd256_pd128(lhs), rhs, index); + return _mm256_insertf128_pd(lhs, lower, 0); + } + const __m128d upper = SimdImpl128::insert_slow(_mm256_extractf128_pd(lhs, 1), rhs, index - 2); + return _mm256_insertf128_pd(lhs, upper, 1); } // unpack / pack - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_lo(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_lo(auto lhs, auto rhs) noexcept { return _mm256_unpacklo_pd(lhs, rhs); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL unpack_hi(auto lhs, auto rhs) noexcept + static auto SIMD_FLAGS(InOut, ForceInline) unpack_hi(auto lhs, auto rhs) noexcept { return _mm256_unpackhi_pd(lhs, rhs); } // misc - SIMDLIB_FORCE_INLINE static auto VECTORCALL shuffle(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Emulates an immediate-controlled floating shuffle with a runtime scalar control. + * @param lhs Source for the lower selected lanes in each group. + * @param rhs Source for the upper selected lanes in each group. + * @param imm8 Runtime control byte. + * @return Register containing the shuffled lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) shuffle_slow(auto lhs, auto rhs, const int imm8) noexcept + { + return register_shuffle_double_slow(lhs, rhs, imm8); + } + /** @brief Emulates an immediate-controlled blend with a runtime scalar mask. + * @param lhs Source for lanes whose control bits are clear. + * @param rhs Source for lanes whose control bits are set. + * @param imm8 Runtime control byte. + * @return Register containing the selected lanes. + */ + static auto SIMD_FLAGS(InOut, ForceInline) blend_slow(auto lhs, auto rhs, const int imm8) noexcept { - return register_shuffle_double(lhs, rhs, imm8); + return register_blend_slow(lhs, rhs, static_cast(imm8)); } - SIMDLIB_FORCE_INLINE static auto VECTORCALL blend(auto lhs, auto rhs, const int imm8) noexcept + /** @brief Selects 64-bit floating-point lanes from two 256-bit registers with an immediate control. */ + template constexpr static auto SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(const __m256d lhs, const __m256d rhs) noexcept { - return register_blend(lhs, rhs, static_cast(imm8)); + if (std::is_constant_evaluated()) + return register_blend_slow(lhs, rhs, static_cast(imm8)); + return _mm256_blend_pd(lhs, rhs, imm8 & 0x0F); } }; @@ -4258,6 +6756,9 @@ template struct SimdMappings<256, element_t> : public SimdImpl using impl = SimdImpl256; public: + using impl::extract; + using impl::shuffle; + template using Mappings = SimdMappings<256, ty>; template using mapped_vector_t = typename Mappings::vector_t; template @@ -4277,32 +6778,74 @@ template struct SimdMappings<256, element_t> : public SimdImpl constexpr static inline std::size_t element_size = sizeof(element_t); constexpr static inline std::size_t element_width = 8 * element_size; - template SIMDLIB_FORCE_INLINE static auto VECTORCALL extract(const vector_t lhs) noexcept + static typename SimdMappings<128, element_t>::vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) lower_half(const vector_t lhs) noexcept { - static_assert(index >= 0 && static_cast(index) < element_count, "SimdMappings<256>::extract index out of range."); - if constexpr (requires(vector_t value) { impl::template extract(value); }) - { - return impl::template extract(lhs); - } + if constexpr (std::is_integral_v) + return _mm256_castsi256_si128(lhs); + else if constexpr (std::same_as) + return _mm256_castps256_ps128(lhs); + else + return _mm256_castpd256_pd128(lhs); + } + +#pragma region 256-bit Shifting + + /** + * @brief Shifts a complete 256-bit register toward higher byte indices by a compile-time count. + * @tparam count Nonnegative byte count; counts of at least 32 produce zero. + * @param lhs Source register. + * @return Shifted register with zero fill across the 128-bit boundary. + */ + template static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_left(const int_vector_t lhs) noexcept + { + static_assert(count >= 0, "Complete-register byte shifts require a nonnegative count."); + if constexpr (count == 0) + return lhs; + else if constexpr (count >= 32) + return _mm256_setzero_si256(); else { - return get_element(lhs, index); + const __m256i previous_half = _mm256_permute2x128_si256(lhs, lhs, 0x08); + if constexpr (count < 16) + return _mm256_alignr_epi8(lhs, previous_half, 16 - count); + else if constexpr (count == 16) + return previous_half; + else + return _mm256_slli_si256(previous_half, count - 16); } } - SIMDLIB_FORCE_INLINE static typename SimdMappings<128, element_t>::vector_t VECTORCALL lower_half(const vector_t lhs) noexcept + /** + * @brief Shifts a complete 256-bit register toward lower byte indices by a compile-time count. + * @tparam count Nonnegative byte count; counts of at least 32 produce zero. + * @param lhs Source register. + * @return Shifted register with zero fill across the 128-bit boundary. + */ + template static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_right(const int_vector_t lhs) noexcept { - if constexpr (std::is_integral_v) - return _mm256_castsi256_si128(lhs); - else if constexpr (std::same_as) - return _mm256_castps256_ps128(lhs); + static_assert(count >= 0, "Complete-register byte shifts require a nonnegative count."); + if constexpr (count == 0) + return lhs; + else if constexpr (count >= 32) + return _mm256_setzero_si256(); else - return _mm256_castpd256_pd128(lhs); + { + const __m128i high_half = _mm256_extracti128_si256(lhs, 1); + const __m256i next_half = _mm256_zextsi128_si256(high_half); + if constexpr (count < 16) + return _mm256_alignr_epi8(next_half, lhs, count); + else if constexpr (count == 16) + return next_half; + else + return _mm256_srli_si256(next_half, count - 16); + } } +#pragma endregion + #pragma region Set /// Set all elements of the register to 0 (often a noop). - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL setzero() noexcept + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) setzero() noexcept { if (std::is_constant_evaluated()) { @@ -4321,11 +6864,11 @@ template struct SimdMappings<256, element_t> : public SimdImpl template ... Args> requires(sizeof...(Args) == element_count) - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL setr(Args &&...args) noexcept + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) setr(Args &&...args) noexcept { if (std::is_constant_evaluated()) { - return register_from_values(static_cast(args)...); + return setr_constexpr(std::forward(args)...); } else { @@ -4333,7 +6876,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl } } - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL construct(std::array data) noexcept + constexpr static vector_t SIMD_FLAGS(Out, ForceInline, Flatten) construct(const std::array &data) noexcept { if (std::is_constant_evaluated()) { @@ -4345,45 +6888,44 @@ template struct SimdMappings<256, element_t> : public SimdImpl } } - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL set1(const element_t value) noexcept + constexpr static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) set1(const element_t value) noexcept { if (std::is_constant_evaluated()) - { - std::array lanes{}; - lanes.fill(value); - return register_from_array(lanes); - } + return set1_constexpr(value); else { return impl::set1(value); } } - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL multiply_add(const vector_t lhs, const vector_t rhs, const vector_t addend) noexcept + /** @brief Broadcasts one value through the portable compile-time register representation. */ + constexpr static vector_t set1_constexpr(const element_t value) noexcept { - if constexpr (requires(vector_t left, vector_t right, vector_t sum) { impl::multiply_add(left, right, sum); }) - return impl::multiply_add(lhs, rhs, addend); - else - return impl::add(impl::multiply(lhs, rhs), addend); + return register_from_repeated_value(value); } - SIMDLIB_FORCE_INLINE constexpr static vector_t VECTORCALL set_element(vector_t vec, int index, element_t value) noexcept + /** @brief Constructs a register from forward-order lanes during constant evaluation. */ + template ... Args> + requires(sizeof...(Args) == element_count) + constexpr static vector_t setr_constexpr(Args &&...args) noexcept { - register_set(vec, static_cast(index), value); - return vec; + return register_from_values(static_cast(args)...); } - SIMDLIB_FORCE_INLINE constexpr static element_t VECTORCALL get_element(vector_t vec, int index) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add(const vector_t lhs, const vector_t rhs, const vector_t addend) noexcept { - return register_get(vec, static_cast(index)); + if constexpr (requires(vector_t left, vector_t right, vector_t sum) { impl::multiply_add(left, right, sum); }) + return impl::multiply_add(lhs, rhs, addend); + else + return impl::add(impl::multiply(lhs, rhs), addend); } - SIMDLIB_FORCE_INLINE static std::span VECTORCALL view_data(vector_t &vec) noexcept + static std::span SIMD_FLAGS(Neither, ForceInline, Flatten) view_data(vector_t &vec) noexcept { return std::span{register_data(vec), element_count}; } - SIMDLIB_FORCE_INLINE static std::span VECTORCALL view_data(const vector_t &vec) noexcept + static std::span SIMD_FLAGS(Neither, ForceInline, Flatten) view_data(const vector_t &vec) noexcept { return std::span{register_data(vec), element_count}; } @@ -4391,15 +6933,31 @@ template struct SimdMappings<256, element_t> : public SimdImpl #pragma endregion #pragma region Load + /** + * @brief Loads a complete 256-bit object representation without an alignment requirement. + * @param ptr Source containing at least 32 accessible bytes. + * @return Native register preserving every source bit. + */ + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_bytes(const void *ptr) noexcept + { + const int_vector_t bits = _mm256_loadu_si256(reinterpret_cast(ptr)); + if constexpr (std::is_integral_v) + return bits; + else if constexpr (std::is_same_v) + return _mm256_castsi256_ps(bits); + else + return _mm256_castsi256_pd(bits); + } + /// Loads a full register from memory. Pointer must be appropriately aligned for the register width. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL load(const element_t *ptr) noexcept + static int_vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load(const element_t *ptr) noexcept requires std::is_integral_v { return _mm256_load_si256(reinterpret_cast(ptr)); } /// Loads a full register from memory without requiring alignment. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL load_unaligned(const element_t *ptr) noexcept + static int_vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_unaligned(const element_t *ptr) noexcept requires std::is_integral_v { return _mm256_loadu_si256(reinterpret_cast(ptr)); @@ -4410,7 +6968,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl /// For 256-bit registers this loads the low 128-bit lane and clears the high lane. /// Intended for safe tail handling without over-reading past the end of a buffer. /// - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL load_half(const element_t *ptr) noexcept + static int_vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_half(const element_t *ptr) noexcept requires std::is_integral_v { const __m128i lo = _mm_loadu_si128(reinterpret_cast(ptr)); @@ -4418,7 +6976,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl } /// Loads a full register from memory. Pointer must be appropriately aligned for the register width. - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL load(const element_t *ptr) noexcept + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load(const element_t *ptr) noexcept requires std::is_floating_point_v { if constexpr (std::is_same_v) @@ -4428,7 +6986,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl } /// Loads a full register from memory without requiring alignment. - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL load_unaligned(const element_t *ptr) noexcept + static vector_t SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_unaligned(const element_t *ptr) noexcept requires std::is_floating_point_v { if constexpr (std::is_same_v) @@ -4440,14 +6998,14 @@ template struct SimdMappings<256, element_t> : public SimdImpl #pragma region Store /// Stores a full register to memory. Pointer must be appropriately aligned for the register width. - SIMDLIB_FORCE_INLINE static void VECTORCALL store(int_vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store(int_vector_t lhs, void *ptr) noexcept requires std::is_integral_v { _mm256_store_si256(reinterpret_cast(ptr), lhs); } /// Stores a full register to memory without requiring alignment. - SIMDLIB_FORCE_INLINE static void VECTORCALL store_unaligned(int_vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store_unaligned(int_vector_t lhs, void *ptr) noexcept requires std::is_integral_v { _mm256_storeu_si256(reinterpret_cast(ptr), lhs); @@ -4458,7 +7016,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl /// For 256-bit registers this stores only the low 128-bit lane. /// Intended for safe tail handling without over-writing past the end of a buffer. /// - SIMDLIB_FORCE_INLINE static void VECTORCALL store_half(int_vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store_half(int_vector_t lhs, void *ptr) noexcept requires std::is_integral_v { const __m128i lo = _mm256_castsi256_si128(lhs); @@ -4466,7 +7024,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl } /// Stores a full register to memory. Pointer must be appropriately aligned for the register width. - SIMDLIB_FORCE_INLINE static void VECTORCALL store(vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store(vector_t lhs, void *ptr) noexcept requires std::is_floating_point_v { if constexpr (std::is_same_v) @@ -4476,7 +7034,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl } /// Stores a full register to memory without requiring alignment. - SIMDLIB_FORCE_INLINE static void VECTORCALL store_unaligned(vector_t lhs, void *ptr) noexcept + static void SIMD_FLAGS(In, ForceInline, Flatten) store_unaligned(vector_t lhs, void *ptr) noexcept requires std::is_floating_point_v { if constexpr (std::is_same_v) @@ -4494,7 +7052,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl * @param rhs The second register. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_and(vector_t lhs, vector_t rhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_and(vector_t lhs, vector_t rhs) noexcept { if constexpr (std::is_integral_v) return _mm256_and_si256(lhs, rhs); @@ -4510,7 +7068,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl * @param rhs The second register. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_or(vector_t lhs, vector_t rhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_or(vector_t lhs, vector_t rhs) noexcept { if constexpr (std::is_integral_v) return _mm256_or_si256(lhs, rhs); @@ -4526,7 +7084,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl * @param rhs The second register. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_xor(vector_t lhs, vector_t rhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_xor(vector_t lhs, vector_t rhs) noexcept { if constexpr (std::is_integral_v) return _mm256_xor_si256(lhs, rhs); @@ -4542,7 +7100,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl * @param rhs The register to combine with the complement. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_andnot(vector_t lhs, vector_t rhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_andnot(vector_t lhs, vector_t rhs) noexcept { if constexpr (std::is_integral_v) return _mm256_andnot_si256(lhs, rhs); @@ -4557,7 +7115,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl * @param lhs The source register. * @return The resulting mapped register. */ - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL bitwise_not(vector_t lhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_not(vector_t lhs) noexcept { if constexpr (std::is_integral_v) return _mm256_xor_si256(lhs, _mm256_cmpeq_epi32(_mm256_setzero_si256(), _mm256_setzero_si256())); @@ -4569,7 +7127,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl #pragma endregion #pragma region Arithmetic Operations - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL negate(int_vector_t lhs) noexcept + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) negate(int_vector_t lhs) noexcept requires std::is_integral_v { if constexpr (sizeof(element_t) == 8) @@ -4582,7 +7140,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl return _mm256_sub_epi8(_mm256_setzero_si256(), lhs); } - SIMDLIB_FORCE_INLINE static vector_t VECTORCALL negate(vector_t lhs) noexcept + static vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) negate(vector_t lhs) noexcept requires std::is_floating_point_v { if constexpr (std::same_as) @@ -4593,42 +7151,37 @@ template struct SimdMappings<256, element_t> : public SimdImpl #pragma endregion #pragma region Shuffling - /// Shuffles the 32-bit integers in the vector using the specified control mask. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL shuffle_32(int_vector_t lhs, std::uint32_t imm8) noexcept + /** @brief Emulates an immediate-controlled 32-bit shuffle with a runtime scalar control. + * @param lhs Source register. + * @param imm8 Runtime control byte. + * @return Register with each four-lane group shuffled. + */ + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_32_slow(int_vector_t lhs, std::uint32_t imm8) noexcept requires std::is_integral_v { - return register_shuffle_32(lhs, imm8); + return register_shuffle_32_slow(lhs, imm8); } /// Shuffles the 32-bit integers in the vector using a compile-time control mask. template - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL shuffle_32(int_vector_t lhs) noexcept + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_32(int_vector_t lhs) noexcept requires std::is_integral_v { return _mm256_shuffle_epi32(lhs, imm8); } /// Shuffles the bytes in the vector using the indexes in the second vector. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL shuffle(int_vector_t lhs, int_vector_t rhs) noexcept + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(int_vector_t lhs, int_vector_t rhs) noexcept requires std::is_integral_v { return _mm256_shuffle_epi8(lhs, rhs); } - /// Shuffles the bytes in the vector using the templated index sequence. - template - requires(sizeof...(indices) == 32) - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL shuffle(int_vector_t lhs) noexcept - { - // A constexpr register initializer was intentionally replaced by the portable runtime intrinsic. - // The active compiler-independent constexpr register construction lives in Detail::register_from_values. - return _mm256_shuffle_epi8(lhs, _mm256_setr_epi8(indices...)); - } #pragma endregion #pragma region Miscellaneous Operations /// Returns a mask of the most significant BIT of each BYTE in each element. - SIMDLIB_FORCE_INLINE static mask_t VECTORCALL movemask(const vector_t lhs) noexcept + static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) movemask(const vector_t lhs) noexcept { if constexpr (std::is_integral_v) return _mm256_movemask_epi8(lhs); @@ -4639,7 +7192,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl } /// Returns a mask of the most significant BIT of each element. - SIMDLIB_FORCE_INLINE static mask_t VECTORCALL movemask_slim(const vector_t lhs) noexcept + static mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) movemask_slim(const vector_t lhs) noexcept { if constexpr (std::is_integral_v) { @@ -4675,14 +7228,14 @@ template struct SimdMappings<256, element_t> : public SimdImpl /// Compute the bitwise AND of 256 bits (representing integer data) in a and b, and set ZF to 1 if the result is zero, otherwise set ZF to 0. /// Compute the bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, otherwise set CF to 0. Return the CF value. - SIMDLIB_FORCE_INLINE static int VECTORCALL test(int_vector_t lhs, int_vector_t rhs) noexcept + static int SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) test(int_vector_t lhs, int_vector_t rhs) noexcept { return _mm256_testc_si256(lhs, rhs); } /// Compute the bitwise AND of 256 bits (representing integer data) in a and b, and set ZF to 1 if the result is zero, otherwise set ZF to 0. /// Compute the bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, otherwise set CF to 0. Return the ZF value. - SIMDLIB_FORCE_INLINE static int VECTORCALL testz(int_vector_t lhs, int_vector_t rhs) noexcept + static int SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) testz(int_vector_t lhs, int_vector_t rhs) noexcept { return _mm256_testz_si256(lhs, rhs); } @@ -4690,7 +7243,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl /// Compute the bitwise AND of 256 bits (representing integer data) in a and b, and set ZF to 1 if the result is zero, otherwise set ZF to 0. /// Compute the bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, otherwise set CF to 0. Return 1 if both the ZF and CF values /// are zero, otherwise return 0. - SIMDLIB_FORCE_INLINE static int VECTORCALL testnzc(int_vector_t lhs, int_vector_t rhs) noexcept + static int SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) testnzc(int_vector_t lhs, int_vector_t rhs) noexcept { return _mm256_testnzc_si256(lhs, rhs); } @@ -4698,7 +7251,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl /// Returns the shuffle order to move the most significant byte of each element into the least significant bytes. constexpr static int_vector_t get_msb_swizzle_order() noexcept { - int_vector_t seq; + int_vector_t seq{}; constexpr const auto elem_size = sizeof(element_t); constexpr const auto elems_per_lane = 16 / elem_size; @@ -4710,12 +7263,12 @@ template struct SimdMappings<256, element_t> : public SimdImpl if (i < elems_per_lane) { // Select the MSB byte of each element within the 128-bit lane. - register_set(seq, lane_base + i, static_cast((i * elem_size) + (elem_size - 1))); + register_set_constexpr(seq, lane_base + i, static_cast((i * elem_size) + (elem_size - 1))); } else { // Zero out the rest (PSHUFB: high bit set => 0). - register_set(seq, lane_base + i, 0x80); + register_set_constexpr(seq, lane_base + i, 0x80); } } } @@ -4723,7 +7276,7 @@ template struct SimdMappings<256, element_t> : public SimdImpl } /// Swizzle the vector to only contain the most significant bit of each byte. - SIMDLIB_FORCE_INLINE static int_vector_t VECTORCALL swizzle_msb(int_vector_t lhs) noexcept + static int_vector_t SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) swizzle_msb(int_vector_t lhs) noexcept { return shuffle(lhs, get_msb_swizzle_order()); } diff --git a/include/SimdLib/IApi.h b/include/SimdLib/IApi.h new file mode 100644 index 0000000..bc1606f --- /dev/null +++ b/include/SimdLib/IApi.h @@ -0,0 +1,288 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace SimdLib +{ + +/** @brief Reports whether one scalar type and register width have a configured SIMD backend. */ +template +inline constexpr bool is_api_available_v = + (std::same_as || std::same_as || std::same_as || + std::same_as || std::same_as || std::same_as || + std::same_as || std::same_as || std::same_as || std::same_as) && + Config::target_x86 && ((register_width == 128 && Config::has_sse42) || (register_width == 256 && Config::has_sse42 && Config::has_avx2)); + +/** @brief Constrains one scalar type and register width to a configured SIMD backend. */ +template +concept ApiAvailable = is_api_available_v; + +/** @brief Reports whether the native-width API alias is available for an element type. */ +template +concept NativeApiAvailable = ApiAvailable<128, element_t>; + +/** @brief Structural interface requirements exposed by an API layer. */ +namespace IApi +{ + +/** @brief Identifies an API-shaped type with native vector and scalar metadata. */ +template +concept Type = requires { + typename api_t::element_type; + typename api_t::vector_t; +}; + +/** @brief Identifies a valid widening destination API shape. */ +template +concept WidenTarget = Type && requires { + { api_t::register_width } -> std::convertible_to; +}; + +/** @brief Reports whether an API exposes lane-wise addition. */ +template +concept Add = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::add(lhs, rhs); }; + +/** @brief Reports whether an API exposes lane-wise subtraction. */ +template +concept Subtract = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::subtract(lhs, rhs); }; + +/** @brief Reports whether an API exposes lane-wise multiplication. */ +template +concept Multiply = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::multiply(lhs, rhs); }; + +/** @brief Reports whether an API exposes lane-wise division. */ +template +concept Divide = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::divide(lhs, rhs); }; + +/** @brief Reports whether an API exposes lane-wise remainder. */ +template +concept Modulus = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::modulus(lhs, rhs); }; + +/** @brief Reports whether an API exposes arithmetic negation. */ +template +concept Negate = Type && requires(typename api_t::vector_t value) { api_t::negate(value); }; + +/** @brief Reports whether an API exposes lane-wise minimum. */ +template +concept Min = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::min(lhs, rhs); }; + +/** @brief Reports whether an API exposes lane-wise maximum. */ +template +concept Max = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::max(lhs, rhs); }; + +/** @brief Reports whether an API exposes lane-wise absolute value. */ +template +concept Absolute = Type && requires(typename api_t::vector_t value) { api_t::absolute(value); }; + +/** @brief Reports whether an API exposes lane-wise square root. */ +template +concept Sqrt = Type && requires(typename api_t::vector_t value) { api_t::sqrt(value); }; + +/** @brief Reports whether an API exposes lane-wise average. */ +template +concept Average = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::avg(lhs, rhs); }; + +/** @brief Reports whether an API exposes fused or emulated multiply-add. */ +template +concept MultiplyAdd = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs, typename api_t::vector_t addend) { + api_t::multiply_add(lhs, rhs, addend); +}; + +/** @brief Reports whether an API exposes register magnitude. */ +template +concept Magnitude = Type && requires(typename api_t::vector_t value) { api_t::magnitude(value); }; + +/** @brief Reports whether an API exposes checked integer magnitude. */ +template +concept MagnitudeChecked = Type && requires(typename api_t::vector_t value) { api_t::magnitude_checked(value); }; + +/** @brief Reports whether an API exposes floating-point normalization. */ +template +concept Normalize = Type && requires(typename api_t::vector_t value) { api_t::normalize(value); }; + +/** @brief Reports whether an API exposes adjacent horizontal addition. */ +template +concept HorizontalAdd = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::add_horizontal(lhs, rhs); }; + +/** @brief Reports whether an API exposes adjacent horizontal subtraction. */ +template +concept HorizontalSubtract = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::subtract_horizontal(lhs, rhs); }; + +/** @brief Reports whether an API exposes adjacent multiply-add. */ +template +concept MultiplyAddAdjacent = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::multiply_add_adjacent(lhs, rhs); }; + +/** @brief Reports whether an API exposes unsigned-byte by signed-byte multiply-add. */ +template +concept ByteMultiplyAdd = + Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::multiply_add_unsigned_signed_bytes(lhs, rhs); }; + +/** @brief Reports whether an API exposes byte sum-of-absolute-differences. */ +template +concept Sad = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::sum_absolute_byte_differences(lhs, rhs); }; + +/** @brief Reports whether an API exposes immediate-controlled multi-SAD. */ +template +concept MultiSad = immediate >= 0 && immediate <= 255 && Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { + api_t::template multi_sum_absolute_byte_differences(lhs, rhs); +}; + +/** @brief Reports whether an API exposes minimum-position lookup. */ +template +concept MinPosition = Type && requires(typename api_t::vector_t value) { api_t::min_position(value); }; + +/** @brief Reports whether an API exposes maximum-position lookup. */ +template +concept MaxPosition = Type && requires(typename api_t::vector_t value) { api_t::max_position(value); }; + +/** @brief Reports whether an API exposes saturating addition. */ +template +concept AddSaturated = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::add_saturated(lhs, rhs); }; + +/** @brief Reports whether an API exposes saturating subtraction. */ +template +concept SubtractSaturated = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::subtract_saturated(lhs, rhs); }; + +/** @brief Reports whether an API exposes saturating horizontal addition. */ +template +concept HorizontalAddSaturated = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::hadd_saturated(lhs, rhs); }; + +/** @brief Reports whether an API exposes saturating horizontal subtraction. */ +template +concept HorizontalSubtractSaturated = + Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::hsubtract_saturated(lhs, rhs); }; + +/** @brief Reports whether an API exposes alternating add-subtract. */ +template +concept AddSubtract = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::add_subtract(lhs, rhs); }; + +/** @brief Reports whether an API exposes an immediate-controlled dot product. */ +template +concept DotProduct = immediate >= 0 && immediate <= 255 && Type && + requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::template dot_product(lhs, rhs); }; + +/** @brief Reports whether an API exposes per-lane left shift. */ +template +concept ShiftLeft = Type && requires(typename api_t::vector_t value) { api_t::shift_left(value, 1); }; + +/** @brief Reports whether an API exposes per-lane logical right shift. */ +template +concept ShiftRight = Type && requires(typename api_t::vector_t value) { api_t::shift_right(value, 1); }; + +/** @brief Reports whether an API exposes per-lane arithmetic right shift. */ +template +concept ArithmeticShiftRight = Type && requires(typename api_t::vector_t value) { api_t::shift_right_arithmetic(value, 1); }; + +/** @brief Reports whether an API exposes explicit slow-path complete-register byte shifts. */ +template +concept ShiftBytesSlow = Type && requires(typename api_t::vector_t value) { + api_t::shift_bytes_left_slow(value, 1); + api_t::shift_bytes_right_slow(value, 1); +}; + +/** @brief Reports whether an API exposes an immediate complete-register byte left shift. */ +template +concept ShiftBytesLeft = count >= 0 && Type && requires(typename api_t::int_vector_t value) { api_t::template shift_bytes_left(value); }; + +/** @brief Reports whether an API exposes an immediate complete-register byte right shift. */ +template +concept ShiftBytesRight = count >= 0 && Type && requires(typename api_t::int_vector_t value) { api_t::template shift_bytes_right(value); }; + +/** @brief Reports whether an API exposes explicit slow-path complete-register bit shifts. */ +template +concept ShiftBitsSlow = Type && requires(typename api_t::vector_t value) { + api_t::shift_bits_left_slow(value, 1); + api_t::shift_bits_right_slow(value, 1); +}; + +/** @brief Reports whether an API exposes compile-time complete-register bit shifts. */ +template +concept ShiftBits = Type && requires(typename api_t::int_vector_t value) { + api_t::template shift_bits_left(value); + api_t::template shift_bits_right(value); +}; + +/** @brief Reports whether an API exposes explicit slow-path runtime-selected lane extraction. */ +template +concept ExtractSlow = Type && requires(typename api_t::vector_t value, selector_t selector) { api_t::extract_slow(value, selector); }; + +/** @brief Reports whether an API exposes explicit slow-path runtime-selected lane insertion. */ +template +concept InsertSlow = Type && requires(typename api_t::vector_t value, typename api_t::element_type lane) { api_t::insert_slow(value, lane, 0); }; +/** @brief Reports whether an API exposes extraction of a 256-bit register's lower 128-bit half. */ +template +concept LowerHalf = Type && requires(typename api_t::vector_t value) { api_t::lower_half(value); }; + +/** @brief Reports whether an API exposes low-lane unpacking. */ +template +concept UnpackLow = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::unpack_lo(lhs, rhs); }; + +/** @brief Reports whether an API exposes high-lane unpacking. */ +template +concept UnpackHigh = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::unpack_hi(lhs, rhs); }; + +/** @brief Reports whether an API accepts one compile-time logical shuffle selector sequence. */ +template +concept Shuffle = Type && requires(typename api_t::vector_t value) { api_t::template shuffle(value); }; + +/** @brief Reports whether an API exposes an immediate-controlled low-half shuffle. */ +template +concept ShuffleLow = Type && requires(typename api_t::vector_t value) { api_t::template shuffle_lo(value); }; + +/** @brief Reports whether an API exposes an immediate-controlled high-half shuffle. */ +template +concept ShuffleHigh = Type && requires(typename api_t::vector_t value) { api_t::template shuffle_hi(value); }; + +/** @brief Reports whether an API exposes an immediate-controlled two-register blend. */ +template +concept Blend = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::template blend(lhs, rhs); }; + +/** @brief Reports whether an API exposes a native register-selector shuffle. */ +template +concept RegisterShuffle = Type && requires(typename api_t::vector_t value) { api_t::shuffle(value, value); }; + +/** @brief Reports whether an API exposes an explicit slow-path scalar-controlled shuffle. */ +template +concept ShuffleSlow = Type && requires(typename api_t::vector_t value) { api_t::shuffle_slow(value, value, 0); }; + +/** @brief Reports whether an API exposes an explicit slow-path low-half shuffle. */ +template +concept ShuffleLowSlow = Type && requires(typename api_t::vector_t value) { api_t::shuffle_lo_slow(value, 0); }; + +/** @brief Reports whether an API exposes an explicit slow-path high-half shuffle. */ +template +concept ShuffleHighSlow = Type && requires(typename api_t::vector_t value) { api_t::shuffle_hi_slow(value, 0); }; + +/** @brief Reports whether an API exposes a native register-mask blend. */ +template +concept RegisterBlend = + Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs, typename api_t::vector_t mask) { api_t::blend(lhs, rhs, mask); }; + +/** @brief Reports whether an API exposes an explicit slow-path scalar-controlled blend. */ +template +concept BlendSlow = Type && requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs) { api_t::blend_slow(lhs, rhs, 0); }; + +/** @brief Reports whether an API exposes explicit slow-path 32-bit immediate-mask shuffling. */ +template +concept Shuffle32Slow = Type && requires(typename api_t::int_vector_t value) { api_t::shuffle_32_slow(value, std::uint32_t{}); }; +/** @brief Reports whether an API can reinterpret a complete register as the requested target element type. */ +template +concept BitCast = Type && requires(typename api_t::vector_t value) { api_t::template bit_cast(value); }; + +/** @brief Reports whether an API can numerically convert a complete register to the requested target element type. */ +template +concept Convert = Type && requires(typename api_t::vector_t value) { api_t::template convert(value); }; + +/** @brief Reports whether an API can widen its lowest source lanes into one complete target API register. */ +template +concept Widen = Type && WidenTarget && requires(typename api_t::vector_t value) { api_t::template widen(value); }; + +} // namespace IApi + +} // namespace SimdLib diff --git a/include/SimdLib/IImpl.h b/include/SimdLib/IImpl.h new file mode 100644 index 0000000..acda5c7 --- /dev/null +++ b/include/SimdLib/IImpl.h @@ -0,0 +1,337 @@ +#pragma once + +#include +#include +#include +#include + +namespace SimdLib::IImpl +{ + +/** @brief Identifies a backend mapping that exposes a native vector type. */ +template +concept Mapping = requires { typename implementation_t::vector_t; }; + +/** @brief Reports whether a backend can create an all-zero register. */ +template +concept SetZero = Mapping && requires { implementation_t::setzero(); }; + +/** @brief Reports whether a backend can broadcast one scalar value. */ +template +concept SetOne = Mapping && requires(scalar_t value) { implementation_t::set1(value); }; + +/** @brief Reports whether a backend accepts a native-order lane list. */ +template +concept Set = Mapping && requires(argument_t &&...values) { implementation_t::set(std::forward(values)...); }; + +/** @brief Reports whether a backend accepts a logical-order lane list. */ +template +concept SetReverse = Mapping && requires(argument_t &&...values) { implementation_t::setr(std::forward(values)...); }; + +/** @brief Reports whether a backend exposes lane-wise addition. */ +template +concept Add = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::add(lhs, rhs); }; + +/** @brief Reports whether a backend exposes lane-wise subtraction. */ +template +concept Subtract = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::subtract(lhs, rhs); }; + +/** @brief Reports whether a backend exposes lane-wise multiplication. */ +template +concept Multiply = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::multiply(lhs, rhs); }; + +/** @brief Reports whether a backend exposes lane-wise division. */ +template +concept Divide = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::divide(lhs, rhs); }; + +/** @brief Reports whether a backend exposes lane-wise remainder. */ +template +concept Modulus = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::modulus(lhs, rhs); }; + +/** @brief Reports whether a backend exposes arithmetic negation. */ +template +concept Negate = Mapping && requires(typename implementation_t::vector_t value) { implementation_t::negate(value); }; + +/** @brief Reports whether a backend exposes lane-wise minimum. */ +template +concept Min = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::min(lhs, rhs); }; + +/** @brief Reports whether a backend exposes lane-wise maximum. */ +template +concept Max = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::max(lhs, rhs); }; + +/** @brief Reports whether a backend exposes lane-wise absolute value. */ +template +concept Absolute = Mapping && requires(typename implementation_t::vector_t value) { implementation_t::absolute(value); }; + +/** @brief Reports whether a backend exposes lane-wise square root. */ +template +concept Sqrt = Mapping && requires(typename implementation_t::vector_t value) { implementation_t::sqrt(value); }; + +/** @brief Reports whether a backend exposes a register magnitude operation. */ +template +concept Magnitude = Mapping && requires(typename implementation_t::vector_t value) { implementation_t::magnitude(value); }; + +/** @brief Reports whether a backend exposes checked integer magnitude. */ +template +concept MagnitudeChecked = Mapping && requires(typename implementation_t::vector_t value) { implementation_t::magnitude_checked(value); }; + +/** @brief Reports whether a backend exposes lane-wise average. */ +template +concept Average = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::avg(lhs, rhs); }; + +/** @brief Reports whether a backend exposes fused or emulated multiply-add. */ +template +concept MultiplyAdd = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs, + typename implementation_t::vector_t addend) { implementation_t::multiply_add(lhs, rhs, addend); }; + +/** @brief Reports whether backend primitives required by normalization are available. */ +template +concept Normalize = Magnitude && Divide; + +/** @brief Reports whether a backend exposes adjacent horizontal addition. */ +template +concept HorizontalAdd = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::add_horizontal(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes adjacent horizontal subtraction. */ +template +concept HorizontalSubtract = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::subtract_horizontal(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes adjacent multiply-add. */ +template +concept MultiplyAddAdjacent = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::multiply_add_adjacent(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes unsigned-byte by signed-byte multiply-add. */ +template +concept ByteMultiplyAdd = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::multiply_add_unsigned_signed_bytes(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes byte sum-of-absolute-differences. */ +template +concept Sad = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::sum_absolute_byte_differences(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes immediate-controlled multi-SAD. */ +template +concept MultiSad = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::template multi_sum_absolute_byte_differences(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes the primitives used to locate an extremum. */ +template +concept Position = Mapping && requires(typename implementation_t::vector_t value) { + implementation_t::min_position(value); + implementation_t::template extract<1>(value); +}; + +/** @brief Reports whether a backend exposes saturating addition. */ +template +concept AddSaturated = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::add_saturated(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes saturating subtraction. */ +template +concept SubtractSaturated = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::subtract_saturated(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes saturating horizontal addition. */ +template +concept HorizontalAddSaturated = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::hadd_saturated(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes saturating horizontal subtraction. */ +template +concept HorizontalSubtractSaturated = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::hsubtract_saturated(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes alternating add-subtract. */ +template +concept AddSubtract = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::add_subtract(lhs, rhs); }; + +/** @brief Reports whether a backend exposes an immediate-controlled dot product. */ +template +concept DotProduct = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::template dot_product(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes bitwise AND. */ +template +concept BitwiseAnd = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::bitwise_and(lhs, rhs); }; + +/** @brief Reports whether a backend exposes bitwise OR. */ +template +concept BitwiseOr = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::bitwise_or(lhs, rhs); }; + +/** @brief Reports whether a backend exposes bitwise XOR. */ +template +concept BitwiseXor = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::bitwise_xor(lhs, rhs); }; + +/** @brief Reports whether a backend exposes bitwise AND-NOT. */ +template +concept BitwiseAndNot = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::bitwise_andnot(lhs, rhs); +}; + +/** @brief Reports whether a backend exposes bitwise complement. */ +template +concept BitwiseNot = Mapping && requires(typename implementation_t::vector_t value) { implementation_t::bitwise_not(value); }; + +/** @brief Reports whether a backend exposes predicate-based selection. */ +template +concept Select = + Mapping && requires(typename implementation_t::vector_t condition, typename implementation_t::vector_t when_true, + typename implementation_t::vector_t when_false) { implementation_t::select(condition, when_true, when_false); }; + +/** @brief Reports whether a backend exposes its legacy expand operation. */ +template +concept Expand = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::expand(lhs, rhs); }; + +/** @brief Reports whether a backend exposes its legacy compress operation. */ +template +concept Compress = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::compress(lhs, rhs); }; + +/** @brief Reports whether a backend can widen into the requested destination mapping. */ +template +concept Widen = Mapping && requires(typename implementation_t::vector_t value) { implementation_t::template widen(value); }; + +/** @brief Reports whether a backend exposes compile-time lane extraction. */ +template +concept IndexedExtract = Mapping && requires(typename implementation_t::vector_t value) { implementation_t::template extract(value); }; + +/** @brief Reports whether a backend exposes explicit slow-path runtime-selected extraction. */ +template +concept ExtractSlow = + Mapping && requires(typename implementation_t::vector_t value, selector_t selector) { implementation_t::extract_slow(value, selector); }; + +/** @brief Reports whether a backend exposes extraction of its lower 128-bit half. */ +template +concept LowerHalf = Mapping && requires(typename implementation_t::vector_t value) { implementation_t::lower_half(value); }; + +/** @brief Reports whether a backend accepts explicit slow-path runtime insertion arguments. */ +template +concept InsertSlow = Mapping && requires(argument_t &&...values) { implementation_t::insert_slow(std::forward(values)...); }; + +/** @brief Reports whether a backend exposes low-lane unpacking. */ +template +concept UnpackLow = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::unpack_lo(lhs, rhs); }; + +/** @brief Reports whether a backend exposes high-lane unpacking. */ +template +concept UnpackHigh = Mapping && + requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { implementation_t::unpack_hi(lhs, rhs); }; + +/** @brief Reports whether a backend accepts a logical shuffle index sequence for its native vector type. */ +template +concept IndexedShuffle = + Mapping && requires(typename implementation_t::vector_t value) { implementation_t::template shuffle(value); }; + +/** @brief Reports whether a backend accepts the supplied shuffle arguments. */ +template +concept Shuffle = Mapping && requires(argument_t &&...values) { implementation_t::shuffle(std::forward(values)...); }; +/** @brief Reports whether a backend accepts explicit slow-path scalar-controlled shuffle arguments. */ +template +concept ShuffleSlow = Mapping && requires(argument_t &&...values) { implementation_t::shuffle_slow(std::forward(values)...); }; + +/** @brief Reports whether a backend accepts the supplied low-half shuffle arguments. */ +template +concept ShuffleLow = Mapping && requires(argument_t &&...values) { implementation_t::shuffle_lo(std::forward(values)...); }; +/** @brief Reports whether a backend accepts explicit slow-path low-half shuffle arguments. */ +template +concept ShuffleLowSlow = + Mapping && requires(argument_t &&...values) { implementation_t::shuffle_lo_slow(std::forward(values)...); }; + +/** @brief Reports whether a backend accepts the supplied high-half shuffle arguments. */ +template +concept ShuffleHigh = Mapping && requires(argument_t &&...values) { implementation_t::shuffle_hi(std::forward(values)...); }; +/** @brief Reports whether a backend accepts explicit slow-path high-half shuffle arguments. */ +template +concept ShuffleHighSlow = + Mapping && requires(argument_t &&...values) { implementation_t::shuffle_hi_slow(std::forward(values)...); }; + +/** @brief Reports whether a backend accepts the supplied blend arguments. */ +template +concept Blend = Mapping && requires(argument_t &&...values) { implementation_t::blend(std::forward(values)...); }; +/** @brief Reports whether a backend accepts explicit slow-path scalar-controlled blend arguments. */ +template +concept BlendSlow = Mapping && requires(argument_t &&...values) { implementation_t::blend_slow(std::forward(values)...); }; + +/** @brief Reports whether a backend exposes explicit slow-path 32-bit immediate-mask shuffling. */ +template +concept Shuffle32Slow = + Mapping && requires(typename implementation_t::int_vector_t value) { implementation_t::shuffle_32_slow(value, std::uint32_t{}); }; + +/** @brief Reports whether a backend exposes explicit slow-path complete-register byte shifts. */ +template +concept ShiftBytesSlow = Mapping && requires(typename implementation_t::int_vector_t value) { + implementation_t::shift_bytes_left_slow(value, 1); + implementation_t::shift_bytes_right_slow(value, 1); +}; + +/** @brief Reports whether a backend exposes an immediate complete-register byte left shift. */ +template +concept ShiftBytesLeft = + Mapping && requires(typename implementation_t::int_vector_t value) { implementation_t::template shift_bytes_left(value); }; + +/** @brief Reports whether a backend exposes an immediate complete-register byte right shift. */ +template +concept ShiftBytesRight = + Mapping && requires(typename implementation_t::int_vector_t value) { implementation_t::template shift_bytes_right(value); }; + +/** @brief Reports whether a backend exposes explicit slow-path complete-register bit shifts. */ +template +concept ShiftBitsSlow = Mapping && requires(typename implementation_t::int_vector_t value) { + implementation_t::shift_bits_left_slow(value, 1); + implementation_t::shift_bits_right_slow(value, 1); +}; + +/** @brief Reports whether a backend exposes compile-time complete-register bit shifts. */ +template +concept ShiftBits = Mapping && requires(typename implementation_t::int_vector_t value) { + implementation_t::template shift_bits_left(value); + implementation_t::template shift_bits_right(value); +}; + +/** @brief Reports whether a backend exposes an immediate-controlled low-half shuffle. */ +template +concept IndexedShuffleLow = + Mapping && requires(typename implementation_t::vector_t value) { implementation_t::template shuffle_lo(value); }; + +/** @brief Reports whether a backend exposes an immediate-controlled high-half shuffle. */ +template +concept IndexedShuffleHigh = + Mapping && requires(typename implementation_t::vector_t value) { implementation_t::template shuffle_hi(value); }; + +/** @brief Reports whether a backend exposes an immediate-controlled blend. */ +template +concept IndexedBlend = Mapping && requires(typename implementation_t::vector_t lhs, typename implementation_t::vector_t rhs) { + implementation_t::template blend(lhs, rhs); +}; + +} // namespace SimdLib::IImpl diff --git a/include/SimdLib/IRegister.h b/include/SimdLib/IRegister.h new file mode 100644 index 0000000..381622d --- /dev/null +++ b/include/SimdLib/IRegister.h @@ -0,0 +1,493 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace SimdLib::IRegister +{ + +/** @brief Identifies an aggregate Register-shaped type with public SIMD metadata and native storage. */ +template +concept Type = std::is_aggregate_v && requires(register_t value, typename register_t::native_type native) { + typename register_t::element_type; + typename register_t::api_type; + typename register_t::native_type; + typename register_t::mask_type; + { register_t::register_width } -> std::convertible_to; + { register_t::byte_count } -> std::convertible_to; + { register_t::lane_count } -> std::convertible_to; + { value.native } -> std::same_as; + { register_t{native} } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes zero initialization. */ +template +concept Zero = Type && requires { + { register_t::zero() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes scalar broadcast construction. */ +template +concept Broadcast = Type && requires(typename register_t::element_type value) { + { register_t::broadcast(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type accepts the supplied complete logical lane list. */ +template +concept FromLanes = Type && sizeof...(lane_types) == register_t::lane_count && requires(lane_types &&...lanes) { + { register_t::from_lanes(std::forward(lanes)...) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes fixed-size array construction. */ +template +concept FromArray = Type && requires(const std::array &source) { + { register_t::from_array(source) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes exact-width unaligned loading. */ +template +concept Load = Type && requires(std::span source) { + { register_t::load(source) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes exact-width aligned loading. */ +template +concept LoadAligned = Type && requires(std::span source) { + { register_t::load_aligned(source) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes exact-width raw-byte loading. */ +template +concept LoadBytes = Type && requires(std::span source) { + { register_t::load_bytes(source) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes exact-width unaligned storage. */ +template +concept Store = Type && requires(register_t value, std::span destination) { + { value.store(destination) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes exact-width aligned storage. */ +template +concept StoreAligned = Type && requires(register_t value, std::span destination) { + { value.store_aligned(destination) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes exact-width raw-byte storage. */ +template +concept StoreBytes = Type && requires(register_t value, std::span destination) { + { value.store_bytes(destination) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes fixed-size array conversion. */ +template +concept ToArray = Type && requires(register_t value) { + { value.to_array() } -> std::same_as>; +}; + +/** @brief Reports whether a Register type exposes one compile-time-selected lane. */ +template +concept Lane = Type && requires(register_t value) { + { value.template lane() } -> std::same_as; +}; + +/** @brief Reports whether a Register type can replace one compile-time-selected lane. */ +template +concept WithLane = Type && requires(register_t value, typename register_t::element_type replacement) { + { value.template with_lane(replacement) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes register addition. */ +template +concept Add = Type && requires(register_t lhs, register_t rhs) { + { lhs + rhs } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes register subtraction. */ +template +concept Subtract = Type && requires(register_t lhs, register_t rhs) { + { lhs - rhs } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes register multiplication. */ +template +concept Multiply = Type && requires(register_t lhs, register_t rhs) { + { lhs * rhs } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes register division. */ +template +concept Divide = Type && requires(register_t lhs, register_t rhs) { + { lhs / rhs } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes register remainder. */ +template +concept Modulus = Type && requires(register_t lhs, register_t rhs) { + { lhs % rhs } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes arithmetic negation. */ +template +concept Negate = Type && requires(register_t value) { + { -value } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes lane-wise minimum. */ +template +concept Min = Type && requires(register_t value) { + { value.min(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes lane-wise maximum. */ +template +concept Max = Type && requires(register_t value) { + { value.max(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes lane-wise absolute value. */ +template +concept Absolute = Type && requires(register_t value) { + { value.absolute() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes lane-wise square root. */ +template +concept Sqrt = Type && requires(register_t value) { + { value.sqrt() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes lane-wise average. */ +template +concept Average = Type && requires(register_t value) { + { value.average(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes multiply-add. */ +template +concept MultiplyAdd = Type && requires(register_t value) { + { value.multiply_add(value, value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes magnitude. */ +template +concept Magnitude = Type && requires(register_t value) { + { value.magnitude() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes checked magnitude. */ +template +concept MagnitudeChecked = Type && requires(register_t value) { + { value.magnitude_checked() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes normalization. */ +template +concept Normalize = Type && requires(register_t value) { + { value.normalize() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes horizontal addition. */ +template +concept HorizontalAdd = Type && requires(register_t value) { + { value.horizontal_add(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes horizontal subtraction. */ +template +concept HorizontalSubtract = Type && requires(register_t value) { + { value.horizontal_subtract(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes adjacent multiply-add for the requested source type. */ +template +concept MultiplyAddAdjacent = Type && requires(register_t value) { value.template multiply_add_adjacent(value); }; + +/** @brief Reports whether a Register type exposes unsigned/signed byte multiply-add for the requested source type. */ +template +concept MultiplyAddUnsignedSignedBytes = + Type && requires(register_t value) { value.template multiply_add_unsigned_signed_bytes(value); }; + +/** @brief Reports whether a Register type exposes byte sum-of-absolute-differences for the requested source type. */ +template +concept SumAbsoluteByteDifferences = Type && requires(register_t value) { value.template sum_absolute_byte_differences(value); }; + +/** @brief Reports whether a Register type exposes immediate-controlled multi-SAD for the requested source type. */ +template +concept MultiSumAbsoluteByteDifferences = + Type && requires(register_t value) { value.template multi_sum_absolute_byte_differences(value); }; + +/** @brief Reports whether a Register type exposes minimum-position lookup. */ +template +concept MinPosition = Type && requires(register_t value) { + { value.min_position() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes maximum-position lookup. */ +template +concept MaxPosition = Type && requires(register_t value) { + { value.max_position() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes saturating addition. */ +template +concept AddSaturated = Type && requires(register_t value) { + { value.add_saturated(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes saturating subtraction. */ +template +concept SubtractSaturated = Type && requires(register_t value) { + { value.subtract_saturated(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes saturating horizontal addition. */ +template +concept HorizontalAddSaturated = Type && requires(register_t value) { + { value.horizontal_add_saturated(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes saturating horizontal subtraction. */ +template +concept HorizontalSubtractSaturated = Type && requires(register_t value) { + { value.horizontal_subtract_saturated(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes alternating add-subtract. */ +template +concept AddSubtract = Type && requires(register_t value) { + { value.add_subtract(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes an immediate-controlled dot product. */ +template +concept DotProduct = Type && requires(register_t value) { + { value.template dot_product(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes bitwise AND. */ +template +concept BitwiseAnd = Type && requires(register_t lhs, register_t rhs) { + { lhs & rhs } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes bitwise OR. */ +template +concept BitwiseOr = Type && requires(register_t lhs, register_t rhs) { + { lhs | rhs } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes bitwise XOR. */ +template +concept BitwiseXor = Type && requires(register_t lhs, register_t rhs) { + { lhs ^ rhs } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes bitwise complement. */ +template +concept BitwiseNot = Type && requires(register_t value) { + { ~value } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes bitwise AND-NOT. */ +template +concept BitwiseAndNot = Type && requires(register_t lhs, register_t rhs) { + { lhs.andnot(rhs) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes its native-granularity sign mask. */ +template +concept Movemask = Type && requires(register_t value) { + { value.movemask() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes one sign bit per logical lane. */ +template +concept LaneSignBits = Type && requires(register_t value) { + { value.lane_sign_bits() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes per-lane left shift. */ +template +concept ShiftLeft = Type && requires(register_t value) { + { value << 1 } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes logical per-lane right shift. */ +template +concept LogicalShiftRight = Type && requires(register_t value) { + { value.logical_shift_right(1) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes signedness-selected per-lane right shift. */ +template +concept ShiftRight = Type && requires(register_t value) { + { value >> 1 } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes explicit slow-path complete-register dynamic byte left shift. */ +template +concept ShiftBytesLeftSlow = Type && requires(register_t value) { + { value.shift_bytes_left_slow(1) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes explicit slow-path complete-register dynamic byte right shift. */ +template +concept ShiftBytesRightSlow = Type && requires(register_t value) { + { value.shift_bytes_right_slow(1) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes immediate complete-register byte left shift. */ +template +concept ShiftBytesLeft = count >= 0 && Type && requires(register_t value) { + { value.template shift_bytes_left() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes immediate complete-register byte right shift. */ +template +concept ShiftBytesRight = count >= 0 && Type && requires(register_t value) { + { value.template shift_bytes_right() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes explicit slow-path complete-register dynamic bit left shift. */ +template +concept ShiftBitsLeftSlow = Type && requires(register_t value) { + { value.shift_bits_left_slow(1) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes explicit slow-path complete-register dynamic bit right shift. */ +template +concept ShiftBitsRightSlow = Type && requires(register_t value) { + { value.shift_bits_right_slow(1) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes complete-register compile-time bit left shift. */ +template +concept ShiftBitsLeft = Type && requires(register_t value) { + { value.template shift_bits_left() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes complete-register compile-time bit right shift. */ +template +concept ShiftBitsRight = Type && requires(register_t value) { + { value.template shift_bits_right() } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes ordered equality comparison. */ +template +concept CompareEqual = Type && requires(register_t value) { + { value.compare_equal(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes ordered greater-than comparison. */ +template +concept CompareGreater = Type && requires(register_t value) { + { value.compare_greater(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes ordered greater-than-or-equal comparison. */ +template +concept CompareGreaterEqual = Type && requires(register_t value) { + { value.compare_greater_equal(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes ordered less-than comparison. */ +template +concept CompareLess = Type && requires(register_t value) { + { value.compare_less(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes ordered less-than-or-equal comparison. */ +template +concept CompareLessEqual = Type && requires(register_t value) { + { value.compare_less_equal(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes whole-register equality. */ +template +concept Equal = Type && requires(register_t lhs, register_t rhs) { + { lhs == rhs } -> std::same_as; +}; + +/** @brief Reports whether a Register type exposes whole-register inequality. */ +template +concept NotEqual = Type && requires(register_t lhs, register_t rhs) { + { lhs != rhs } -> std::same_as; +}; + +/** @brief Identifies a Register-shaped result with the requested element type and width. */ +template +concept Shape = Type && std::same_as && register_t::register_width == bits; + +/** @brief Reports whether a Register exposes its lower 128-bit half. */ +template +concept LowerHalf = Type && requires(register_t value) { + { value.lower_half() } -> Shape; +}; + +/** @brief Reports whether a Register exposes low-lane unpacking. */ +template +concept UnpackLow = Type && requires(register_t value) { + { value.unpack_low(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register exposes high-lane unpacking. */ +template +concept UnpackHigh = Type && requires(register_t value) { + { value.unpack_high(value) } -> std::same_as; +}; + +/** @brief Reports whether a Register accepts one compile-time logical shuffle selector sequence. */ +template +concept Shuffle = Type && requires(register_t value) { + { value.template shuffle() } -> std::same_as; +}; + +/** @brief Reports whether a Register accepts one compile-time byte selector sequence. */ +template +concept ShuffleBytes = Type && requires(register_t value) { + { value.template shuffle_bytes() } -> std::same_as; +}; + +/** @brief Reports whether a Register exposes an immediate-controlled low-half shuffle. */ +template +concept ShuffleLow = Type && requires(register_t value) { + { value.template shuffle_low() } -> std::same_as; +}; + +/** @brief Reports whether a Register exposes an immediate-controlled high-half shuffle. */ +template +concept ShuffleHigh = Type && requires(register_t value) { + { value.template shuffle_high() } -> std::same_as; +}; + +/** @brief Reports whether a Register exposes an immediate-controlled two-register blend. */ +template +concept Blend = Type && requires(register_t lhs, register_t rhs) { + { lhs.template blend(rhs) } -> std::same_as; +}; + +/** @brief Reports whether a Register can reinterpret its complete bit pattern as the requested element type. */ +template +concept BitCast = Type && requires(register_t value) { + { value.template bit_cast() } -> Shape; +}; + +/** @brief Reports whether a Register can numerically convert every lane to the requested element type. */ +template +concept Convert = Type && requires(register_t value) { + { value.template convert() } -> Shape; +}; + +/** @brief Reports whether a Register can widen its lowest lanes into the requested complete target register. */ +template +concept WidenLow = Type && requires(register_t value) { + { value.template widen_low() } -> Shape; +}; + +} // namespace SimdLib::IRegister diff --git a/include/SimdLib/IRegisterMask.h b/include/SimdLib/IRegisterMask.h new file mode 100644 index 0000000..c088c47 --- /dev/null +++ b/include/SimdLib/IRegisterMask.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include + +namespace SimdLib::IRegisterMask +{ + +/** @brief Identifies an aggregate RegisterMask-shaped type with public predicate metadata and native storage. */ +template +concept Type = std::is_aggregate_v && requires(mask_t value, typename mask_t::native_type native) { + typename mask_t::element_type; + typename mask_t::api_type; + typename mask_t::native_type; + typename mask_t::register_type; + typename mask_t::bits_type; + { mask_t::register_width } -> std::convertible_to; + { mask_t::byte_count } -> std::convertible_to; + { mask_t::lane_count } -> std::convertible_to; + { value.native } -> std::same_as; + { mask_t{native} } -> std::same_as; +}; + +/** @brief Reports whether a RegisterMask type exposes an any-lane reduction. */ +template +concept Any = Type && requires(mask_t value) { + { value.any() } -> std::same_as; +}; + +/** @brief Reports whether a RegisterMask type exposes an all-lanes reduction. */ +template +concept All = Type && requires(mask_t value) { + { value.all() } -> std::same_as; +}; + +/** @brief Reports whether a RegisterMask type exposes a no-lanes reduction. */ +template +concept None = Type && requires(mask_t value) { + { value.none() } -> std::same_as; +}; + +/** @brief Reports whether a RegisterMask type exposes one compact bit per logical lane. */ +template +concept Bits = Type && requires(mask_t value) { + { value.bits() } -> std::same_as; +}; + +/** @brief Reports whether a RegisterMask type can select corresponding lanes from two Registers. */ +template +concept Select = Type && requires(mask_t condition, typename mask_t::register_type when_true, typename mask_t::register_type when_false) { + { condition.select(when_true, when_false) } -> std::same_as; +}; + +/** @brief Reports whether a RegisterMask type exposes predicate intersection. */ +template +concept BitwiseAnd = Type && requires(mask_t lhs, mask_t rhs) { + { lhs & rhs } -> std::same_as; +}; + +/** @brief Reports whether a RegisterMask type exposes predicate union. */ +template +concept BitwiseOr = Type && requires(mask_t lhs, mask_t rhs) { + { lhs | rhs } -> std::same_as; +}; + +/** @brief Reports whether a RegisterMask type exposes predicate exclusive union. */ +template +concept BitwiseXor = Type && requires(mask_t lhs, mask_t rhs) { + { lhs ^ rhs } -> std::same_as; +}; + +/** @brief Reports whether a RegisterMask type exposes predicate complement. */ +template +concept BitwiseNot = Type && requires(mask_t value) { + { ~value } -> std::same_as; +}; + +} // namespace SimdLib::IRegisterMask diff --git a/include/SimdLib/Register.h b/include/SimdLib/Register.h new file mode 100644 index 0000000..ca53930 --- /dev/null +++ b/include/SimdLib/Register.h @@ -0,0 +1,1230 @@ +#pragma once + +#include + +#if !SIMDLIB_REGISTER_INTERFACE_AVAILABLE && !SIMDLIB_REQUIRE_REGISTER_INTERFACE +#error "SIMDLIB_REGISTER_HEADER_REQUIRES_CXX23: requires C++23 explicit object parameter support" +#endif + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace SimdLib +{ + +/** + * @brief Owns one complete SIMD register whose lanes are all active. + * @tparam element_t Scalar interpretation of each register lane. + * @tparam bits Width of the native register in bits. + * @invariant The aggregate contains exactly one complete native register value and no inactive-lane state. + * @remarks Available only when `RegisterAvailable` is satisfied. + */ +template + requires RegisterAvailable +class Register final +{ + public: + using element_type = element_t; + using api_type = Api; + using native_type = typename api_type::vector_t; + using mask_type = RegisterMask; + + constexpr static inline std::size_t register_width = bits; + constexpr static inline std::size_t byte_count = api_type::byte_count; + constexpr static inline std::size_t lane_count = api_type::element_count; + + /** @brief Owns the complete native register value represented by this aggregate. */ + native_type native = api_type::setzero(); + + /** + * @brief Returns a register with every active lane set to zero. + * @return Fully initialized zero register. + */ + [[nodiscard]] constexpr static Register SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) zero() noexcept + { + return Register{api_type::setzero()}; + } + + /** + * @brief Broadcasts one scalar value to every active lane. + * @param value Scalar value to broadcast. + * @return Register containing `value` in every lane. + */ + [[nodiscard]] constexpr static Register SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) broadcast(element_type value) noexcept + { + return Register{api_type::set1(value)}; + } + + /** + * @brief Constructs a register from exactly one complete logical lane list. + * @tparam lane_types Scalar argument types convertible to `element_type`. + * @param lanes Values in low-to-high logical lane order. + * @return Register containing all supplied lane values. + */ + template ... lane_types> + requires(sizeof...(lane_types) == lane_count) + [[nodiscard]] constexpr static Register SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) from_lanes(lane_types &&...lanes) noexcept + { + return Register{api_type::setr(static_cast(std::forward(lanes))...)}; + } + + /** + * @brief Constructs a register from one complete fixed-size lane array. + * @param source Source containing every active lane in logical order. + * @return Register containing all source lane values. + */ + [[nodiscard]] constexpr static Register SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) + from_array(const std::array &source) noexcept + { + return Register{api_type::construct(source)}; + } + + /** + * @brief Loads a complete register from potentially unaligned storage. + * @param source Source containing exactly one register of elements. + * @return Register loaded from `source`. + */ + [[nodiscard]] static Register SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load(std::span source) noexcept + { + return Register{api_type::load(source)}; + } + + /** + * @brief Loads a complete register from register-aligned storage. + * @param source Aligned source containing exactly one register of elements. + * @return Register loaded from `source`. + * @pre `source.data()` is aligned to `byte_count` bytes. + */ + [[nodiscard]] static Register SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_aligned(std::span source) noexcept + { + return Register{api_type::load_aligned(source)}; + } + + /** + * @brief Loads one complete register bit pattern from raw bytes. + * @param source Source containing exactly one register of bytes. + * @return Register containing the source bit pattern. + */ + [[nodiscard]] static Register SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) load_bytes(std::span source) noexcept + { + return Register{api_type::load(source)}; + } + + /** + * @brief Stores every active lane to potentially unaligned storage. + * @param value Register to store. + * @param destination Destination for exactly one register of elements. + */ + void SIMD_FLAGS(In, ForceInline, Flatten) store(this Register value, std::span destination) noexcept + { + api_type::store(value.native, destination); + } + + /** + * @brief Stores every active lane to register-aligned storage. + * @param value Register to store. + * @param destination Aligned destination for one complete register. + * @pre `destination.data()` is aligned to `byte_count` bytes. + */ + void SIMD_FLAGS(In, ForceInline, Flatten) store_aligned(this Register value, std::span destination) noexcept + { + api_type::store_aligned(value.native, destination); + } + + /** + * @brief Stores the complete register bit pattern to raw bytes. + * @param value Register to store. + * @param destination Destination containing exactly one register of bytes. + */ + void SIMD_FLAGS(In, ForceInline, Flatten) store_bytes(this Register value, std::span destination) noexcept + { + api_type::store(value.native, destination); + } + + /** + * @brief Copies every active lane into a fixed-size array. + * @param value Register to copy. + * @return Array containing all lanes in low-to-high logical order. + */ + [[nodiscard]] constexpr std::array SIMD_FLAGS(In, ForceInline, Flatten) to_array(this Register value) noexcept + { + return api_type::to_array(value.native); + } + + /** + * @brief Returns one compile-time-selected lane. + * @tparam index Logical lane index. + * @param value Register containing the selected lane. + * @return Copy of the selected lane. + */ + template + requires(index < lane_count) + [[nodiscard]] constexpr element_type SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) lane(this Register value) noexcept + { + if consteval + { + return lane_constexpr(value); + } + else + { + return api_type::template extract(index)>(value.native); + } + } + + /** + * @brief Returns a copy with one compile-time-selected lane replaced. + * @tparam index Logical lane index. + * @param value Register containing the lanes to copy. + * @param replacement Replacement value for the selected lane. + * @return Register with lane `index` replaced. + */ + template + requires(index < lane_count) + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) with_lane(this Register value, element_type replacement) noexcept + { + value.native = api_type::template insert(value.native, replacement); + return value; + } + +#pragma region Arithmetic Operations + + /** + * @brief Adds corresponding lanes with the selected intrinsic semantics. + * @param lhs Left addend. + * @param rhs Right addend. + * @return Register containing one sum per logical lane. + * @remarks Available exactly when `IApi::Add` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator+(this Register lhs, Register rhs) noexcept + requires IApi::Add + { + return Register{api_type::add(lhs.native, rhs.native)}; + } + + /** + * @brief Subtracts corresponding lanes with the selected intrinsic semantics. + * @param lhs Minuend lanes. + * @param rhs Subtrahend lanes. + * @return Register containing one difference per logical lane. + * @remarks Available exactly when `IApi::Subtract` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator-(this Register lhs, Register rhs) noexcept + requires IApi::Subtract + { + return Register{api_type::subtract(lhs.native, rhs.native)}; + } + + /** + * @brief Multiplies corresponding lanes with the selected intrinsic semantics. + * @param lhs Left factor. + * @param rhs Right factor. + * @return Register containing one product per logical lane. + * @remarks Available exactly when `IApi::Multiply` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator*(this Register lhs, Register rhs) noexcept + requires IApi::Multiply + { + return Register{api_type::multiply(lhs.native, rhs.native)}; + } + + /** + * @brief Divides corresponding lanes. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @return Register containing one quotient per logical lane. + * @pre Every divisor lane is nonzero and signed minimum is not divided by negative one. + * @remarks Available exactly when `IApi::Divide` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator/(this Register lhs, Register rhs) noexcept + requires IApi::Divide + { + return Register{api_type::divide(lhs.native, rhs.native)}; + } + + /** + * @brief Computes corresponding-lane remainders. + * @param lhs Dividend lanes. + * @param rhs Divisor lanes. + * @return Register containing one remainder per logical lane. + * @pre Every divisor lane is nonzero and signed minimum is not divided by negative one. + * @remarks Available exactly when `IApi::Modulus` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator%(this Register lhs, Register rhs) noexcept + requires IApi::Modulus + { + return Register{api_type::modulus(lhs.native, rhs.native)}; + } + + /** + * @brief Negates every lane with the selected intrinsic's overflow behavior. + * @param value Register to negate. + * @return Register containing the negated logical lanes. + * @remarks Available exactly when `IApi::Negate` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator-(this Register value) noexcept + requires IApi::Negate + { + return Register{api_type::negate(value.native)}; + } + + /* + * Disabled compound assignment operators: their convenience does not justify the mutable-reference API surface, + * and MSVC 19.44 emits a redundant 32-byte stack-alignment frame for 256-bit wrapper mutation through references. + * Prefer `lhs = lhs + rhs`, `lhs = lhs - rhs`, `lhs = lhs * rhs`, `lhs = lhs / rhs`, or `lhs = lhs % rhs`. + * + /// @brief Adds another register into this register. + auto SIMD_FLAGS(In, ForceInline, Flatten) operator+=( + this Register &lhs, + Register rhs) noexcept -> Register & + requires IApi::Add + { + lhs.native = api_type::add(lhs.native, rhs.native); + return lhs; + } + + /// @brief Subtracts another register from this register. + auto SIMD_FLAGS(In, ForceInline, Flatten) operator-=( + this Register &lhs, + Register rhs) noexcept -> Register & + requires IApi::Subtract + { + lhs.native = api_type::subtract(lhs.native, rhs.native); + return lhs; + } + + /// @brief Multiplies this register by another register. + auto SIMD_FLAGS(In, ForceInline, Flatten) operator*=( + this Register &lhs, + Register rhs) noexcept -> Register & + requires IApi::Multiply + { + lhs.native = api_type::multiply(lhs.native, rhs.native); + return lhs; + } + + /// + /// @brief Divides this register by another register. + /// @pre Every divisor lane is nonzero and signed minimum is not divided by negative one. + /// + auto SIMD_FLAGS(In, ForceInline, Flatten) operator/=( + this Register &lhs, + Register rhs) noexcept -> Register & + requires IApi::Divide + { + lhs.native = api_type::divide(lhs.native, rhs.native); + return lhs; + } + + /// + /// @brief Replaces this register with corresponding-lane remainders. + /// @pre Every divisor lane is nonzero and signed minimum is not divided by negative one. + /// + auto SIMD_FLAGS(In, ForceInline, Flatten) operator%=( + this Register &lhs, + Register rhs) noexcept -> Register & + requires IApi::Modulus + { + lhs.native = api_type::modulus(lhs.native, rhs.native); + return lhs; + } + */ +#pragma endregion + +#pragma region Specialized Arithmetic and Reductions + + /** + * @brief Selects the minimum value from each corresponding lane. + * @param lhs First candidate register. + * @param rhs Second candidate register. + * @return Register containing the intrinsic-selected minimum in every lane. + * @remarks Floating-point NaN and signed-zero behavior is defined by the selected intrinsic. Available exactly when `IApi::Min` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) min(this Register lhs, Register rhs) noexcept + requires IApi::Min + { + return Register{api_type::min(lhs.native, rhs.native)}; + } + + /** + * @brief Selects the maximum value from each corresponding lane. + * @param lhs First candidate register. + * @param rhs Second candidate register. + * @return Register containing the intrinsic-selected maximum in every lane. + * @remarks Floating-point NaN and signed-zero behavior is defined by the selected intrinsic. Available exactly when `IApi::Max` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) max(this Register lhs, Register rhs) noexcept + requires IApi::Max + { + return Register{api_type::max(lhs.native, rhs.native)}; + } + + /** + * @brief Computes the absolute value of every lane with the selected intrinsic's edge behavior. + * @param value Source register. + * @return Register containing one absolute value per logical lane. + * @remarks Signed minimum follows the backend contract. Available exactly when `IApi::Absolute` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) absolute(this Register value) noexcept + requires IApi::Absolute + { + return Register{api_type::absolute(value.native)}; + } + + /** + * @brief Computes the square root of every supported lane. + * @param value Source register. + * @return Register containing one intrinsic square-root result per logical lane. + * @remarks Available exactly when `IApi::Sqrt` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) sqrt(this Register value) noexcept + requires IApi::Sqrt + { + return Register{api_type::sqrt(value.native)}; + } + + /** + * @brief Computes the intrinsic-defined average of corresponding lanes. + * @param lhs Left input register. + * @param rhs Right input register. + * @return Register containing one average per logical lane, including the backend's rounding rule. + * @remarks Available exactly when `IApi::Average` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) average(this Register lhs, Register rhs) noexcept + requires IApi::Average + { + return Register{api_type::avg(lhs.native, rhs.native)}; + } + + /** + * @brief Multiplies corresponding lanes and adds a third register. + * @param lhs Left multiplicand. + * @param rhs Right multiplicand. + * @param addend Value added to each corresponding product. + * @return Register containing the fused or emulated multiply-add result in every lane. + * @remarks Fusion follows the selected backend configuration. Available exactly when `IApi::MultiplyAdd` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) multiply_add(this Register lhs, Register rhs, Register addend) noexcept + requires IApi::MultiplyAdd + { + return Register{api_type::multiply_add(lhs.native, rhs.native, addend.native)}; + } + + /** + * @brief Computes broadcast floating magnitudes or sparse unchecked integer magnitudes for each 128-bit group. + * @param value Source register whose grouped Euclidean magnitude is requested. + * @return Floating registers broadcast each group result; integer registers place each unchecked result in the leading lane of its group. + * @pre Every integer group magnitude is representable in `element_type`. + * @remarks Available exactly when `IApi::Magnitude` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude(this Register value) noexcept + requires IApi::Magnitude + { + return Register{api_type::magnitude(value.native)}; + } + + /** + * @brief Computes saturated integer magnitudes with each overflow mask stored in the following lane. + * @param value Integral source register. + * @return Each 128-bit group stores its saturated magnitude first, an all-zero or all-one overflow lane second, and unspecified remaining lanes. + * @remarks Available exactly when `IApi::MagnitudeChecked` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) magnitude_checked(this Register value) noexcept + requires IApi::MagnitudeChecked + { + return Register{api_type::magnitude_checked(value.native)}; + } + + /** + * @brief Normalizes each floating-point 128-bit lane group by its magnitude. + * @param value Floating-point source register. + * @return Register containing every logical lane divided by its 128-bit group magnitude. + * @remarks Zero and exceptional inputs follow the selected floating-point intrinsics. Available exactly when `IApi::Normalize` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) normalize(this Register value) noexcept + requires IApi::Normalize + { + return Register{api_type::normalize(value.native)}; + } + + /** + * @brief Adds adjacent lane pairs independently within each 128-bit group of two registers. + * @param lhs Supplies the first half of the intrinsic-defined horizontal results. + * @param rhs Supplies the second half of the intrinsic-defined horizontal results. + * @return Register containing adjacent-pair sums in intrinsic logical lane order. + * @remarks Available exactly when `IApi::HorizontalAdd` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) horizontal_add(this Register lhs, Register rhs) noexcept + requires IApi::HorizontalAdd + { + return Register{api_type::add_horizontal(lhs.native, rhs.native)}; + } + + /** + * @brief Subtracts adjacent lane pairs independently within each 128-bit group of two registers. + * @param lhs Supplies the first half of the intrinsic-defined horizontal results. + * @param rhs Supplies the second half of the intrinsic-defined horizontal results. + * @return Register containing adjacent-pair differences in intrinsic logical lane order. + * @remarks Available exactly when `IApi::HorizontalSubtract` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) horizontal_subtract(this Register lhs, Register rhs) noexcept + requires IApi::HorizontalSubtract + { + return Register{api_type::subtract_horizontal(lhs.native, rhs.native)}; + } + + /** + * @brief Multiplies adjacent integral lane pairs and returns the explicitly promoted Register type. + * @tparam source_element_t Deferred source type used to constrain result-alias availability. + * @param lhs Left factors in logical lane order. + * @param rhs Right factors in logical lane order. + * @return Promoted Register containing one sum of two adjacent products per result lane, grouped independently by the selected intrinsic. + * @remarks Available only for the source type/width cells satisfying `IApi::MultiplyAddAdjacent`. + */ + template + requires std::same_as && std::is_integral_v && IApi::MultiplyAddAdjacent + [[nodiscard]] multiply_add_adjacent_result_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) + multiply_add_adjacent(this Register lhs, Register rhs) noexcept + { + return multiply_add_adjacent_result_t{api_type::multiply_add_adjacent(lhs.native, rhs.native)}; + } + + /** + * @brief Multiplies unsigned and signed byte pairs and returns signed 16-bit sums. + * @tparam source_element_t Deferred source type used to constrain result-alias availability. + * @param lhs Unsigned-byte multiplicands. + * @param rhs Signed-byte multiplicands. + * @return Signed 16-bit Register containing sums of adjacent byte products in intrinsic lane order. + * @remarks Available only for the source type/width cells satisfying `IApi::ByteMultiplyAdd`. + */ + template + requires std::same_as && std::is_integral_v && IApi::ByteMultiplyAdd + [[nodiscard]] byte_multiply_add_result_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) + multiply_add_unsigned_signed_bytes(this Register lhs, Register rhs) noexcept + { + return byte_multiply_add_result_t{api_type::multiply_add_unsigned_signed_bytes(lhs.native, rhs.native)}; + } + + /** + * @brief Sums byte-wise absolute differences into unsigned 64-bit result lanes. + * @tparam source_element_t Deferred source type used to constrain result-alias availability. + * @param lhs Left byte register. + * @param rhs Right byte register. + * @return Unsigned 64-bit Register containing intrinsic-grouped absolute-difference sums. + * @remarks Available only for the source type/width cells satisfying `IApi::Sad`. + */ + template + requires std::same_as && std::is_integral_v && IApi::Sad + [[nodiscard]] sad_result_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) + sum_absolute_byte_differences(this Register lhs, Register rhs) noexcept + { + return sad_result_t{api_type::sum_absolute_byte_differences(lhs.native, rhs.native)}; + } + + /** + * @brief Computes immediate-controlled byte-window absolute-difference sums. + * @tparam imm8 Immediate control value in the intrinsic range `0..255`. + * @tparam source_element_t Deferred source type used to constrain result-alias availability. + * @param lhs Left byte register. + * @param rhs Right byte register. + * @return Unsigned 16-bit Register containing the intrinsic-selected multi-SAD windows in logical result order. + * @remarks Available only for the source type/width cells satisfying `IApi::MultiSad`. + */ + template + requires(imm8 >= 0 && imm8 <= 255 && std::same_as && std::is_integral_v && + IApi::MultiSad) + [[nodiscard]] multi_sad_result_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) + multi_sum_absolute_byte_differences(this Register lhs, Register rhs) noexcept + { + return multi_sad_result_t{api_type::template multi_sum_absolute_byte_differences(lhs.native, rhs.native)}; + } + + /** + * @brief Returns the first logical position containing the minimum integral value. + * @param value Integral source register. + * @return Zero-based logical lane index of the first minimum value. + * @remarks Available exactly when `IApi::MinPosition` is satisfied. + */ + [[nodiscard]] constexpr std::size_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) min_position(this Register value) noexcept + requires IApi::MinPosition + { + return api_type::min_position(value.native); + } + + /** + * @brief Returns the first logical position containing the maximum integral value. + * @param value Integral source register. + * @return Zero-based logical lane index of the first maximum value. + * @remarks Available exactly when `IApi::MaxPosition` is satisfied. + */ + [[nodiscard]] constexpr std::size_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) max_position(this Register value) noexcept + requires IApi::MaxPosition + { + return api_type::max_position(value.native); + } + + /** + * @brief Adds corresponding lanes with intrinsic saturation. + * @param lhs Left addend. + * @param rhs Right addend. + * @return Register containing saturated lane sums. + * @remarks Available exactly when `IApi::AddSaturated` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_saturated(this Register lhs, Register rhs) noexcept + requires IApi::AddSaturated + { + return Register{api_type::add_saturated(lhs.native, rhs.native)}; + } + + /** + * @brief Subtracts corresponding lanes with intrinsic saturation. + * @param lhs Minuend lanes. + * @param rhs Subtrahend lanes. + * @return Register containing saturated lane differences. + * @remarks Available exactly when `IApi::SubtractSaturated` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) subtract_saturated(this Register lhs, Register rhs) noexcept + requires IApi::SubtractSaturated + { + return Register{api_type::subtract_saturated(lhs.native, rhs.native)}; + } + + /** + * @brief Adds adjacent lane pairs with saturation independently within each intrinsic group. + * @param lhs Supplies the first half of the horizontal results. + * @param rhs Supplies the second half of the horizontal results. + * @return Register containing saturated adjacent-pair sums in intrinsic lane order. + * @remarks Available exactly when `IApi::HorizontalAddSaturated` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) horizontal_add_saturated(this Register lhs, Register rhs) noexcept + requires IApi::HorizontalAddSaturated + { + return Register{api_type::hadd_saturated(lhs.native, rhs.native)}; + } + + /** + * @brief Subtracts adjacent lane pairs with saturation independently within each intrinsic group. + * @param lhs Supplies the first half of the horizontal results. + * @param rhs Supplies the second half of the horizontal results. + * @return Register containing saturated adjacent-pair differences in intrinsic lane order. + * @remarks Available exactly when `IApi::HorizontalSubtractSaturated` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) horizontal_subtract_saturated(this Register lhs, Register rhs) noexcept + requires IApi::HorizontalSubtractSaturated + { + return Register{api_type::hsubtract_saturated(lhs.native, rhs.native)}; + } + + /** + * @brief Alternates subtraction and addition across floating-point lanes. + * @param lhs Left input register. + * @param rhs Right input register. + * @return Register containing the intrinsic-defined alternating `lhs - rhs` and `lhs + rhs` lane sequence. + * @remarks Lane polarity repeats independently in each 128-bit group. Available exactly when `IApi::AddSubtract` is satisfied. + */ + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) add_subtract(this Register lhs, Register rhs) noexcept + requires IApi::AddSubtract + { + return Register{api_type::add_subtract(lhs.native, rhs.native)}; + } + + /** + * @brief Computes a masked floating-point dot product with intrinsic-selected output lanes. + * @tparam imm8 Immediate control value in the intrinsic range `0..255`. + * @param lhs Left factors. + * @param rhs Right factors. + * @return Register containing the immediate-selected dot-product outputs and zeroed unselected lanes. + * @remarks Available exactly when `IApi::DotProduct` is satisfied. + */ + template + requires IApi::DotProduct + [[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) dot_product(this Register lhs, Register rhs) noexcept + { + return Register{api_type::template dot_product(lhs.native, rhs.native)}; + } + +#pragma endregion +#pragma region Bitwise Operations + + /** + * @brief Computes the bitwise intersection of two complete registers. + * @param lhs Left bit pattern. + * @param rhs Right bit pattern. + * @return Register whose bits are `lhs & rhs`; logical lane values are not numerically converted. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator&(this Register lhs, Register rhs) noexcept + { + return Register{api_type::bitwise_and(lhs.native, rhs.native)}; + } + + /** + * @brief Computes the bitwise union of two complete registers. + * @param lhs Left bit pattern. + * @param rhs Right bit pattern. + * @return Register whose bits are `lhs | rhs`; logical lane values are not numerically converted. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator|(this Register lhs, Register rhs) noexcept + { + return Register{api_type::bitwise_or(lhs.native, rhs.native)}; + } + + /** + * @brief Computes the bitwise exclusive union of two complete registers. + * @param lhs Left bit pattern. + * @param rhs Right bit pattern. + * @return Register whose bits are `lhs ^ rhs`; logical lane values are not numerically converted. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator^(this Register lhs, Register rhs) noexcept + { + return Register{api_type::bitwise_xor(lhs.native, rhs.native)}; + } + + /** + * @brief Complements every bit in a complete register. + * @param value Source bit pattern. + * @return Register whose complete bit pattern is `~value`. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator~(this Register value) noexcept + { + return Register{api_type::bitwise_not(value.native)}; + } + + /** + * @brief Computes `(~lhs) & rhs` with the selected intrinsic's operand polarity. + * @param lhs Bit pattern complemented before intersection. + * @param rhs Bit pattern intersected with the complemented left operand. + * @return Register containing `(~lhs) & rhs` across every bit. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) andnot(this Register lhs, Register rhs) noexcept + { + return Register{api_type::bitwise_andnot(lhs.native, rhs.native)}; + } + + /* + * Disabled compound assignment operators: their convenience does not justify the mutable-reference API surface, + * and MSVC 19.44 emits a redundant 32-byte stack-alignment frame for 256-bit wrapper mutation through references. + * Prefer `lhs = lhs & rhs`, `lhs = lhs | rhs`, or `lhs = lhs ^ rhs`. + * + /// @brief Intersects this register with another register. + constexpr auto SIMD_FLAGS(In, ForceInline, Flatten) operator&=( + this Register &lhs, + Register rhs) noexcept -> Register & + { + lhs.native = api_type::bitwise_and(lhs.native, rhs.native); + return lhs; + } + + /// @brief Unites this register with another register. + constexpr auto SIMD_FLAGS(In, ForceInline, Flatten) operator|=( + this Register &lhs, + Register rhs) noexcept -> Register & + { + lhs.native = api_type::bitwise_or(lhs.native, rhs.native); + return lhs; + } + + /// @brief Exclusively combines this register with another register. + constexpr auto SIMD_FLAGS(In, ForceInline, Flatten) operator^=( + this Register &lhs, + Register rhs) noexcept -> Register & + { + lhs.native = api_type::bitwise_xor(lhs.native, rhs.native); + return lhs; + } + */ + /** + * @brief Returns the selected intrinsic's native-granularity sign-bit mask. + * @param value Source register. + * @return Scalar mask using the backend operation's native bit granularity and logical lane order. + * @remarks For byte-granular backends this can contain more than one bit per `element_type` lane. + */ + [[nodiscard]] constexpr typename api_type::mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) movemask(this Register value) noexcept + { + return api_type::movemask(value.native); + } + + /** + * @brief Returns one scalar sign bit for every logical lane. + * @param value Source register. + * @return Compact scalar mask whose bit `i` is the sign bit of logical lane `i`; unused high bits are zero. + */ + [[nodiscard]] constexpr typename api_type::mask_t SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) lane_sign_bits(this Register value) noexcept + { + return api_type::movemask_slim(value.native); + } + +#pragma endregion + +#pragma region Shifting Operations + + /** + * @brief Left-shifts every integral lane. + * @param value Integral source register. + * @param count Runtime shift count applied to every logical lane. + * @return Register containing zero-filled left-shifted lanes. + * @pre `count >= 0`; counts at least the lane width produce zero lanes. + * @remarks Available exactly when `IApi::ShiftLeft` is satisfied. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator<<(this Register value, int count) noexcept + requires IApi::ShiftLeft + { + return Register{api_type::shift_left(value.native, count)}; + } + + /** + * @brief Right-shifts every integral lane with zero fill. + * @param value Integral source register. + * @param count Runtime shift count applied to every logical lane. + * @return Register containing zero-filled right-shifted lanes. + * @pre `count >= 0`; counts at least the lane width produce zero lanes. + * @remarks Available exactly when `IApi::ShiftRight` is satisfied. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) logical_shift_right(this Register value, int count) noexcept + requires IApi::ShiftRight + { + return Register{api_type::shift_right(value.native, count)}; + } + + /** + * @brief Right-shifts unsigned lanes logically and signed lanes arithmetically. + * @param value Integral source register. + * @param count Runtime shift count applied to every logical lane. + * @return Register containing signedness-selected right-shift results. + * @pre `count >= 0`; oversized signed counts clamp and unsigned counts produce zero lanes. + * @remarks Availability is selected before the body through `IApi::ArithmeticShiftRight` for signed lanes or `IApi::ShiftRight` for unsigned lanes. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator>>(this Register value, int count) noexcept + requires((std::is_signed_v && IApi::ArithmeticShiftRight) || (std::is_unsigned_v && IApi::ShiftRight)) + { + if constexpr (std::is_signed_v) + return Register{api_type::shift_right_arithmetic(value.native, count)}; + else + return Register{api_type::shift_right(value.native, count)}; + } + + /* + * Disabled compound assignment operators: their convenience does not justify the mutable-reference API surface, + * and MSVC 19.44 emits a redundant 32-byte stack-alignment frame for 256-bit wrapper mutation through references. + * Prefer `value = value << count` or `value = value >> count`. + * + /// @brief Left-shifts every integral lane in this register. + constexpr auto SIMD_FLAGS(In, ForceInline, Flatten) operator<<=( + this Register &value, + int count) noexcept -> Register & + requires std::is_integral_v + { + value.native = api_type::shift_left(value.native, count); + return value; + } + + /// @brief Right-shifts every integral lane in this register using its signedness. + constexpr auto SIMD_FLAGS(In, ForceInline, Flatten) operator>>=( + this Register &value, + int count) noexcept -> Register & + requires std::is_integral_v + { + if constexpr (std::is_signed_v) + value.native = api_type::shift_right_arithmetic(value.native, count); + else + value.native = api_type::shift_right(value.native, count); + return value; + } + */ + /** + * @brief Byte-shifts a complete 128-bit integral register toward higher byte indices. + * @param value Source register interpreted as one 16-byte string. + * @param count Runtime byte count; nonpositive values are identity and values at least 16 produce zero. + * @return Shifted complete register with zero-filled low bytes. + * @remarks Available only at 128 bits when `IApi::ShiftBytesSlow` is satisfied. + * @note `_slow` marks runtime emulation of an immediate complete-register byte shift. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_left_slow(this Register value, int count) noexcept + requires(register_width == 128 && IApi::ShiftBytesSlow) + { + return Register{api_type::shift_bytes_left_slow(value.native, count)}; + } + + /** + * @brief Byte-shifts a complete integral register toward higher byte indices at compile time. + * + * The register is treated as one contiguous byte string, including across the + * 128-bit boundary of a 256-bit register. + * + * @tparam count Nonnegative byte count; values at least as large as the register byte width produce zero. + * @param value Source register interpreted as one contiguous byte string. + * @return Shifted complete register with zero-filled low bytes. + */ + template + requires(count >= 0 && IApi::ShiftBytesLeft) + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_left(this Register value) noexcept + { + return Register{api_type::template shift_bytes_left(value.native)}; + } + + /** + * @brief Byte-shifts a complete 128-bit integral register toward lower byte indices. + * @param value Source register interpreted as one 16-byte string. + * @param count Runtime byte count; nonpositive values are identity and values at least 16 produce zero. + * @return Shifted complete register with zero-filled high bytes. + * @remarks Available only at 128 bits when `IApi::ShiftBytesSlow` is satisfied. + * @note `_slow` marks runtime emulation of an immediate complete-register byte shift. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_right_slow(this Register value, int count) noexcept + requires(register_width == 128 && IApi::ShiftBytesSlow) + { + return Register{api_type::shift_bytes_right_slow(value.native, count)}; + } + + /** + * @brief Byte-shifts a complete integral register toward lower byte indices at compile time. + * + * The register is treated as one contiguous byte string, including across the + * 128-bit boundary of a 256-bit register. + * + * @tparam count Nonnegative byte count; values at least as large as the register byte width produce zero. + * @param value Source register interpreted as one contiguous byte string. + * @return Shifted complete register with zero-filled high bytes. + */ + template + requires(count >= 0 && IApi::ShiftBytesRight) + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bytes_right(this Register value) noexcept + { + return Register{api_type::template shift_bytes_right(value.native)}; + } + + /** + * @brief Shifts a complete 128-bit integral register left as one bit string. + * @param value Source register interpreted as one 128-bit string. + * @param count Runtime bit count; nonpositive values are identity and values at least 128 produce zero. + * @return Complete-register left shift with zero fill. + * @remarks Available only at 128 bits when `IApi::ShiftBitsSlow` is satisfied. + * @note `_slow` marks the synthesized runtime-count substitute for an immediate complete-register shift. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_left_slow(this Register value, int count) noexcept + requires(register_width == 128 && IApi::ShiftBitsSlow) + { + return Register{api_type::shift_bits_left_slow(value.native, count)}; + } + + /** + * @brief Shifts a complete 128-bit integral register right as one bit string. + * @param value Source register interpreted as one 128-bit string. + * @param count Runtime bit count; nonpositive values are identity and values at least 128 produce zero. + * @return Complete-register right shift with zero fill. + * @remarks Available only at 128 bits when `IApi::ShiftBitsSlow` is satisfied. + * @note `_slow` marks the synthesized runtime-count substitute for an immediate complete-register shift. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_right_slow(this Register value, int count) noexcept + requires(register_width == 128 && IApi::ShiftBitsSlow) + { + return Register{api_type::shift_bits_right_slow(value.native, count)}; + } + + /** + * @brief Shifts a complete 128-bit integral register left as one bit string at compile time. + * @tparam count Nonnegative bit count; values at least 128 produce zero. + * @param value Source register interpreted as one 128-bit string. + * @return Complete-register left shift with zero fill. + * @remarks Available only at 128 bits when `IApi::ShiftBits` is satisfied. + */ + template + requires(register_width == 128 && count >= 0 && IApi::ShiftBits) + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_left(this Register value) noexcept + { + return Register{api_type::template shift_bits_left(value.native)}; + } + + /** + * @brief Shifts a complete 128-bit integral register right as one bit string at compile time. + * @tparam count Nonnegative bit count; values at least 128 produce zero. + * @param value Source register interpreted as one 128-bit string. + * @return Complete-register right shift with zero fill. + * @remarks Available only at 128 bits when `IApi::ShiftBits` is satisfied. + */ + template + requires(register_width == 128 && count >= 0 && IApi::ShiftBits) + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shift_bits_right(this Register value) noexcept + { + return Register{api_type::template shift_bits_right(value.native)}; + } + +#pragma endregion + +#pragma region Rearrangement and Conversion Operations + + /** @brief Returns the low 128-bit half of a 256-bit register. + * @param value Source register in logical low-to-high lane order. + * @return `Register` containing the lowest source lanes. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) lower_half(this Register value) noexcept + requires(register_width == 256 && IApi::LowerHalf) + { + return Register{api_type::lower_half(value.native)}; + } + + /** @brief Interleaves the low half of each 128-bit lane group from two registers. + * @param lhs Supplies even-numbered result lanes in every 128-bit group. + * @param rhs Supplies odd-numbered result lanes in every 128-bit group. + * @return Register containing `lhs[0], rhs[0], lhs[1], rhs[1], ...` independently in each 128-bit group. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) unpack_low(this Register lhs, Register rhs) noexcept + requires IApi::UnpackLow + { + return Register{api_type::unpack_lo(lhs.native, rhs.native)}; + } + + /** @brief Interleaves the high half of each 128-bit lane group from two registers. + * @param lhs Supplies even-numbered result lanes in every 128-bit group. + * @param rhs Supplies odd-numbered result lanes in every 128-bit group. + * @return Register containing interleaved lanes from each source group's high half. + */ + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) unpack_high(this Register lhs, Register rhs) noexcept + requires IApi::UnpackHigh + { + return Register{api_type::unpack_hi(lhs.native, rhs.native)}; + } + + /** @brief Rearranges logical lanes with a complete compile-time selector list. + * @tparam indices One source-lane index for every result lane. + * @param value Source register. + * @return Register containing the selected lanes in logical output order. + * @note Every selector may name any logical lane in the complete source register. + */ + template + requires IApi::Shuffle + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle(this Register value) noexcept + { + return Register{api_type::template shuffle(value.native)}; + } + + /** @brief Rearranges the complete register as a sequence of bytes. + * @tparam indices One source-byte index for every result byte. + * @param value Source register. + * @return Register containing the selected bytes while retaining its original element type. + * @note Every selector may name any byte in the complete source register, including across the 128-bit boundary of a 256-bit register. + */ + template + requires IApi::Shuffle, indices...> + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_bytes(this Register value) noexcept + { + using byte_api_type = Api; + const auto bytes = api_type::template bit_cast(value.native); + const auto shuffled = byte_api_type::template shuffle(bytes); + return Register{byte_api_type::template bit_cast(shuffled)}; + } + + /** @brief Shuffles the low four 16-bit lanes in each 128-bit group. + * @tparam imm8 Immediate control in the inclusive range `0..255`; every two-bit field selects one source lane. + * @param value Source register. + * @return Register with low lane groups shuffled and high lane groups preserved. + */ + template + requires IApi::ShuffleLow + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_low(this Register value) noexcept + { + return Register{api_type::template shuffle_lo(value.native)}; + } + + /** @brief Shuffles the high four 16-bit lanes in each 128-bit group. + * @tparam imm8 Immediate control in the inclusive range `0..255`; every two-bit field selects one source lane. + * @param value Source register. + * @return Register with high lane groups shuffled and low lane groups preserved. + */ + template + requires IApi::ShuffleHigh + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) shuffle_high(this Register value) noexcept + { + return Register{api_type::template shuffle_hi(value.native)}; + } + + /** @brief Selects corresponding lanes from two registers with an immediate control mask. + * @tparam imm8 Immediate control in the inclusive range `0..255`; set applicable bits select `rhs`. + * @param lhs Register selected by cleared applicable control bits. + * @param rhs Register selected by set applicable control bits. + * @return Register containing the intrinsic-defined immediate blend. + * @note Unused immediate bits retain intrinsic behavior. A 256-bit 16-bit blend repeats the mask in each 128-bit group. + */ + template + requires IApi::Blend + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) blend(this Register lhs, Register rhs) noexcept + { + return Register{api_type::template blend(lhs.native, rhs.native)}; + } + + /** @brief Reinterprets every bit of this complete register as another supported lane type. + * @tparam target_t Destination lane interpretation at the same register width. + * @param value Source register whose complete bit pattern is preserved. + * @return `Register` containing exactly the source bits. + */ + template + requires RegisterAvailable && IApi::BitCast + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bit_cast(this Register value) noexcept + { + return Register{api_type::template bit_cast(value.native)}; + } + + /** @brief Numerically converts every lane into one complete destination register. + * @tparam target_t Explicit numeric destination lane type. + * @param value Source register. + * @return `Register` containing converted lane values. + * @note The initial surface supports signed or unsigned 32-bit integers to `float`, and `float` to signed 32-bit integers. + */ + template + requires RegisterAvailable && IApi::Convert + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) convert(this Register value) noexcept + { + return Register{api_type::template convert(value.native)}; + } + + /** @brief Widens only the lowest source lanes needed to fill one complete target register. + * @tparam target_t Wider integral destination lane type with the same signedness as `element_type`. + * @tparam target_bits Destination register width, either 128 or 256 bits. + * @param value Source 128-bit integral register. + * @return Complete target register populated from the lowest `target_bits / (sizeof(target_t) * 8)` source lanes. + * @note Source lanes above the returned register's lane count are intentionally not consumed. + */ + template + requires RegisterAvailable && IApi::Widen> + [[nodiscard]] constexpr Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) widen_low(this Register value) noexcept + { + return Register{api_type::template widen>(value.native)}; + } + +#pragma endregion + +#pragma region Comparison Operations + + /** + * @brief Compares corresponding lanes for intrinsic-defined ordered equality. + * @param lhs Left comparison operand. + * @param rhs Right comparison operand. + * @return Canonical RegisterMask with an all-one lane where `lhs[i] == rhs[i]`, otherwise an all-zero lane. + * @remarks Floating NaNs compare false and signed zeros compare equal under the selected ordered intrinsic. + */ + [[nodiscard]] constexpr mask_type SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) compare_equal(this Register lhs, Register rhs) noexcept + { + return mask_type{api_type::compare_equal(lhs.native, rhs.native)}; + } + + /** + * @brief Compares corresponding lanes for intrinsic-defined greater-than ordering. + * @param lhs Left comparison operand. + * @param rhs Right comparison operand. + * @return Canonical RegisterMask with an all-one lane where `lhs[i] > rhs[i]`, otherwise an all-zero lane. + * @remarks Signedness and floating unordered behavior follow the selected intrinsic. + */ + [[nodiscard]] constexpr mask_type SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) compare_greater(this Register lhs, Register rhs) noexcept + { + return mask_type{api_type::compare_greater(lhs.native, rhs.native)}; + } + + /** + * @brief Compares corresponding lanes for intrinsic-defined greater-than-or-equal ordering. + * @param lhs Left comparison operand. + * @param rhs Right comparison operand. + * @return Canonical RegisterMask with an all-one lane where `lhs[i] >= rhs[i]`, otherwise an all-zero lane. + * @remarks Signedness and floating unordered behavior follow the selected intrinsic. + */ + [[nodiscard]] constexpr mask_type SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) compare_greater_equal(this Register lhs, Register rhs) noexcept + { + return mask_type{api_type::compare_greater_equal(lhs.native, rhs.native)}; + } + + /** + * @brief Compares corresponding lanes for intrinsic-defined less-than ordering. + * @param lhs Left comparison operand. + * @param rhs Right comparison operand. + * @return Canonical RegisterMask with an all-one lane where `lhs[i] < rhs[i]`, otherwise an all-zero lane. + * @remarks Signedness and floating unordered behavior follow the selected intrinsic. + */ + [[nodiscard]] constexpr mask_type SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) compare_less(this Register lhs, Register rhs) noexcept + { + return mask_type{api_type::compare_less(lhs.native, rhs.native)}; + } + + /** + * @brief Compares corresponding lanes for intrinsic-defined less-than-or-equal ordering. + * @param lhs Left comparison operand. + * @param rhs Right comparison operand. + * @return Canonical RegisterMask with an all-one lane where `lhs[i] <= rhs[i]`, otherwise an all-zero lane. + * @remarks Signedness and floating unordered behavior follow the selected intrinsic. + */ + [[nodiscard]] constexpr mask_type SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) compare_less_equal(this Register lhs, Register rhs) noexcept + { + return mask_type{api_type::compare_less_equal(lhs.native, rhs.native)}; + } + + /** + * @brief Tests whether every corresponding lane compares equal. + * @param lhs Left comparison operand. + * @param rhs Right comparison operand. + * @return `true` only when `compare_equal(lhs, rhs).all()` is true. + * @remarks This is numeric intrinsic equality, not bit-pattern equality; floating NaNs compare unequal and signed zeros compare equal. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) operator==(this Register lhs, Register rhs) noexcept + { + return lhs.compare_equal(rhs).all(); + } + + /** + * @brief Tests whether at least one corresponding lane compares unequal. + * @param lhs Left comparison operand. + * @param rhs Right comparison operand. + * @return `true` when at least one lane fails ordered equality. + * @remarks This is the logical negation of whole-register equality, not an every-lane-unequal predicate. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) operator!=(this Register lhs, Register rhs) noexcept + { + return !lhs.compare_equal(rhs).all(); + } + +#pragma endregion + + private: + /** + * @brief Implements compile-time lane observation through the portable array representation. + * @tparam index Logical lane index to observe. + * @param value Register containing the selected lane. + * @return Copy of lane `index`. + */ + template [[nodiscard]] constexpr static element_type lane_constexpr(Register value) noexcept + { + return value.to_array()[index]; + } +}; + +/** + * @brief Defines RegisterMask lane selection after the complete Register type is available. + * @tparam element_t Scalar geometry represented by every predicate and value lane. + * @tparam register_bits Width of the predicate and value registers in bits. + * @param condition Canonical predicate lanes; all-one selects `when_true` and all-zero selects `when_false`. + * @param when_true Register supplying lanes selected by true predicates. + * @param when_false Register supplying lanes selected by false predicates. + * @return Register containing the intrinsic-backed per-lane selection in logical lane order. + * @remarks Available only when `RegisterAvailable` is satisfied. + */ +template + requires RegisterAvailable +[[nodiscard]] constexpr Register SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) + RegisterMask::select(this RegisterMask condition, register_type when_true, register_type when_false) noexcept +{ + return register_type{condition.select_native(when_true.native, when_false.native)}; +} + +/** + * @brief Selects the widest complete register available for an element type. + * @tparam element_t Scalar interpretation of each register lane. + * @remarks Resolves to 256 bits when that specialization is available and otherwise to 128 bits; it must not cross incompatible ISA boundaries. + */ +template + requires RegisterAvailable +using NativeRegister = Register ? 256 : 128)>; + +} // namespace SimdLib diff --git a/include/SimdLib/RegisterFwd.h b/include/SimdLib/RegisterFwd.h new file mode 100644 index 0000000..645dcb1 --- /dev/null +++ b/include/SimdLib/RegisterFwd.h @@ -0,0 +1,90 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace SimdLib +{ + +/** + * @brief Reports whether a complete SIMD register is available for an element type and width. + * @tparam element_t Scalar interpretation of the register lanes. + * @tparam bits Width of the native register in bits. + */ +template inline constexpr bool is_register_available_v = is_api_available_v; + +/** + * @brief Constrains a type and width to an available complete SIMD register. + * @tparam element_t Scalar interpretation of the register lanes. + * @tparam bits Width of the native register in bits. + */ +template +concept RegisterAvailable = is_register_available_v; + +/** + * @brief Owns one complete SIMD register whose logical lanes are all active. + * @tparam element_t Scalar interpretation of every logical lane. + * @tparam bits Native register width in bits. + * @remarks Declared only when `RegisterAvailable` is satisfied. + */ +template + requires RegisterAvailable +class Register; + +/** + * @brief Owns one complete canonical SIMD predicate register associated with a Register geometry. + * @tparam element_t Scalar geometry represented by every logical predicate lane. + * @tparam bits Native predicate-register width in bits. + * @remarks Declared only when `RegisterAvailable` is satisfied. + */ +template + requires RegisterAvailable +class RegisterMask; + +/** + * @brief Result Register produced by adjacent integer multiply-add. + * @tparam element_t Source integral lane type. + * @tparam bits Register width in bits. + */ +template + requires RegisterAvailable && std::is_integral_v && IApi::MultiplyAddAdjacent> +using multiply_add_adjacent_result_t = + Register= sizeof(std::int64_t)), element_t, + std::conditional_t< + std::is_signed_v, + std::conditional_t>, + std::conditional_t>>>, + bits>; + +/** + * @brief Signed 16-bit result Register produced by unsigned/signed byte multiply-add. + * @tparam element_t Source lane type whose register bits are interpreted as bytes. + * @tparam bits Register width in bits. + */ +template + requires RegisterAvailable && std::is_integral_v && IApi::ByteMultiplyAdd> +using byte_multiply_add_result_t = Register; + +/** + * @brief Unsigned 64-bit result Register produced by byte absolute-difference sums. + * @tparam element_t Source lane type whose register bits are interpreted as bytes. + * @tparam bits Register width in bits. + */ +template + requires RegisterAvailable && std::is_integral_v && IApi::Sad> +using sad_result_t = Register; + +/** + * @brief Unsigned 16-bit result Register produced by immediate-controlled multi-SAD. + * @tparam element_t Source lane type whose register bits are interpreted as bytes. + * @tparam bits Register width in bits. + */ +template + requires RegisterAvailable && std::is_integral_v && IApi::MultiSad, 0> +using multi_sad_result_t = Register; +} // namespace SimdLib diff --git a/include/SimdLib/RegisterMask.h b/include/SimdLib/RegisterMask.h new file mode 100644 index 0000000..0a32631 --- /dev/null +++ b/include/SimdLib/RegisterMask.h @@ -0,0 +1,260 @@ +#pragma once + +#include + +#if !SIMDLIB_REGISTER_INTERFACE_AVAILABLE && !SIMDLIB_REQUIRE_REGISTER_INTERFACE +#error "SIMDLIB_REGISTER_MASK_HEADER_REQUIRES_CXX23: requires C++23 explicit object parameter support" +#endif + +#include +#include + +#include +#include +#include +#include + +namespace SimdLib +{ + +/** + * @brief Wraps one native Boolean predicate register for a complete register. + * @tparam element_t Scalar geometry associated with each predicate lane. + * + * @tparam register_bits Width of the associated register in bits. + * @invariant Every logical predicate lane is all-zero or all-one for Boolean mask + * operations. + */ +template + requires RegisterAvailable +class RegisterMask final +{ + public: + using element_type = element_t; + using api_type = Api; + using native_type = typename api_type::vector_t; + using register_type = Register; + using bits_type = std::conditional_t<(api_type::element_count <= 32), std::uint32_t, std::uint64_t>; + + constexpr static inline std::size_t register_width = register_bits; + constexpr static inline std::size_t byte_count = api_type::byte_count; + constexpr static inline std::size_t lane_count = api_type::element_count; + + /** + * @brief Owns the complete native predicate value represented by this aggregate. + * @pre Every logical lane is either all-zero or all-one when + * initialized directly. + */ + native_type native = api_type::setzero(); + + /** + * @brief Tests whether any predicate lane is true. + * @param value Canonical predicate register to reduce. + * @return `true` when at least + * one logical predicate lane is all-one. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) any(this RegisterMask value) noexcept + { + return value.bits() != 0; + } + + /** + * @brief Tests whether every predicate lane is true. + * @param value Canonical predicate register to reduce. + * @return `true` when every + * logical predicate lane is all-one. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) all(this RegisterMask value) noexcept + { + return value.bits() == all_bits; + } + + /** + * @brief Tests whether every predicate lane is false. + * @param value Canonical predicate register to reduce. + * @return `true` when every + * logical predicate lane is all-zero. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) none(this RegisterMask value) noexcept + { + return value.bits() == 0; + } + + /** + * @brief Returns one compact bit per logical predicate lane. + * @param value Canonical predicate register to reduce. + * @return Scalar whose + * bit `i` reports logical predicate lane `i`; all unused high bits are zero. + */ + [[nodiscard]] constexpr bits_type SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) bits(this RegisterMask value) noexcept + { + return static_cast(api_type::movemask_slim(value.native)); + } + + /** + * @brief Selects corresponding true or false Register lanes according to this predicate. + * @param condition Canonical predicate lanes; all-one + * selects `when_true` and all-zero selects `when_false`. + * @param when_true Register supplying lanes selected by true predicates. + * @param when_false + * Register supplying lanes selected by false predicates. + * @return Register containing the intrinsic-backed per-lane selection in logical lane order. + + */ + [[nodiscard]] constexpr register_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) + select(this RegisterMask condition, register_type when_true, register_type when_false) noexcept; + + /** + * @brief Computes the intersection of two predicate registers. + * @param lhs Left canonical predicate register. + * @param rhs Right + * canonical predicate register. + * @return Canonical predicate register whose lane is true only where both input lanes are true. + */ + [[nodiscard]] constexpr RegisterMask SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator&(this RegisterMask lhs, RegisterMask rhs) noexcept + { + return RegisterMask{bitwise_and(lhs.native, rhs.native)}; + } + + /** + * @brief Computes the union of two predicate registers. + * @param lhs Left canonical predicate register. + * @param rhs Right canonical + * predicate register. + * @return Canonical predicate register whose lane is true where either input lane is true. + */ + [[nodiscard]] constexpr RegisterMask SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator|(this RegisterMask lhs, RegisterMask rhs) noexcept + { + return RegisterMask{bitwise_or(lhs.native, rhs.native)}; + } + + /** + * @brief Computes the exclusive union of two predicate registers. + * @param lhs Left canonical predicate register. + * @param rhs Right + * canonical predicate register. + * @return Canonical predicate register whose lane is true where exactly one input lane is true. + */ + [[nodiscard]] constexpr RegisterMask SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator^(this RegisterMask lhs, RegisterMask rhs) noexcept + { + return RegisterMask{bitwise_xor(lhs.native, rhs.native)}; + } + + /** + * @brief Inverts every predicate lane. + * @param value Canonical predicate register. + * @return Canonical predicate register with true and + * false lanes exchanged. + */ + [[nodiscard]] constexpr RegisterMask SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator~(this RegisterMask value) noexcept + { + return RegisterMask{bitwise_not(value.native)}; + } + + /* + * Disabled compound assignment operators: their convenience does not justify the mutable-reference API surface, + * and MSVC 19.44 emits a redundant 32-byte stack-alignment frame for 256-bit wrapper mutation through references. + * Prefer `lhs = lhs & rhs`, `lhs = lhs | rhs`, or `lhs = lhs ^ rhs`. + * + /// @brief Intersects this predicate with another predicate. + constexpr auto SIMD_FLAGS(In, ForceInline, Flatten) operator&=( + this RegisterMask &lhs, + RegisterMask rhs) noexcept -> RegisterMask & + { + return lhs = lhs & rhs; + } + + /// @brief Unites this predicate with another predicate. + constexpr auto SIMD_FLAGS(In, ForceInline, Flatten) operator|=( + this RegisterMask &lhs, + RegisterMask rhs) noexcept -> RegisterMask & + { + return lhs = lhs | rhs; + } + + /// @brief Exclusively combines this predicate with another predicate. + constexpr auto SIMD_FLAGS(In, ForceInline, Flatten) operator^=( + this RegisterMask &lhs, + RegisterMask rhs) noexcept -> RegisterMask & + { + return lhs = lhs ^ rhs; + } + */ + private: + constexpr static inline bits_type all_bits = []() constexpr noexcept + { + if constexpr (lane_count == std::numeric_limits::digits) + return std::numeric_limits::max(); + else + return (bits_type{1} << lane_count) - 1; + }(); + + /** + * @brief Computes the bitwise intersection of two native predicate registers. + * @param lhs Left canonical native predicate. + * @param rhs Right + * canonical native predicate. + * @return Canonical native predicate containing `lhs & rhs`. + */ + [[nodiscard]] constexpr static native_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) + bitwise_and(const native_type lhs, const native_type rhs) noexcept + { + return api_type::bitwise_and(lhs, rhs); + } + + /** + * @brief Computes the bitwise union of two native predicate registers. + * @param lhs Left canonical native predicate. + * @param rhs Right + * canonical native predicate. + * @return Canonical native predicate containing `lhs | rhs`. + */ + [[nodiscard]] constexpr static native_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) + bitwise_or(const native_type lhs, const native_type rhs) noexcept + { + return api_type::bitwise_or(lhs, rhs); + } + + /** + * @brief Computes the bitwise exclusive union of two native predicate registers. + * @param lhs Left canonical native predicate. + * @param rhs + * Right canonical native predicate. + * @return Canonical native predicate containing `lhs ^ rhs`. + */ + [[nodiscard]] constexpr static native_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) + bitwise_xor(const native_type lhs, const native_type rhs) noexcept + { + return api_type::bitwise_xor(lhs, rhs); + } + + /** + * @brief Inverts every bit in a native predicate register. + * @param value Canonical native predicate. + * @return Canonical native + * predicate containing the complemented lanes. + */ + [[nodiscard]] constexpr static native_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) bitwise_not(const native_type value) noexcept + { + return api_type::bitwise_not(value); + } + + /** + * @brief Selects native true or false lanes according to a canonical predicate register. + * @param condition Canonical predicate selecting the + * source of every logical lane. + * @param when_true Native register selected by all-one predicate lanes. + * @param when_false Native register + * selected by all-zero predicate lanes. + * @return Intrinsic-backed native register containing the selected lane values. + */ + [[nodiscard]] constexpr native_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) + select_native(this RegisterMask condition, const native_type when_true, const native_type when_false) noexcept + { + return api_type::select(condition.native, when_true, when_false); + } +}; + +} // namespace SimdLib + +#include diff --git a/include/SimdLib/SimdAlgo.h b/include/SimdLib/SimdAlgo.h index eb1e3a9..c5eff76 100644 --- a/include/SimdLib/SimdAlgo.h +++ b/include/SimdLib/SimdAlgo.h @@ -38,10 +38,14 @@ template struct SimdAlgo final template using SimdImpl = Api= 256 ? 256 : 128, read_t>; - /// - /// Returns true if any element in equals . - /// Intended for fast membership checks. - /// + /** + * @brief Reports whether any input element equals a scalar predicate. + * @tparam count Fixed input element count. + * @param read Input + * elements to inspect. + * @param predicate Scalar value to find. + * @return `true` when at least one element equals `predicate`. + */ template [[nodiscard]] constexpr static inline bool AnyEqual(std::span read, const read_t predicate) noexcept { using simd = SimdImpl; @@ -79,14 +83,21 @@ template struct SimdAlgo final const auto mask = simd::movemask_slim(simd::cmpeq(v, predicateVector)); return (mask & ((typename simd::mask_t{1} << (count - i)) - 1)) != 0; } - return false; + else + { + return false; + } } } - /// - /// Returns true if all elements in equal . - /// Intended for fast "uniform" checks. - /// + /** + * @brief Reports whether every input element equals a scalar predicate. + * @tparam count Fixed input element count. + * @param read Input + * elements to inspect. + * @param predicate Scalar value required in every element. + * @return `true` when every element equals `predicate`. + */ template [[nodiscard]] constexpr static inline bool AllEqual(std::span read, const read_t predicate) noexcept { using simd = SimdImpl; @@ -131,8 +142,10 @@ template struct SimdAlgo final const auto needed = (typename simd::mask_t{1} << (count - i)) - 1; return (mask & needed) == needed; } - - return true; + else + { + return true; + } } } @@ -142,13 +155,11 @@ template struct SimdAlgo final { using simd = SimdImpl; constexpr bool legacy_eight_byte_comparison = ReadWidth == 8 && WriteWidth == 8 && count == 8; - static_assert(WriteWidth == 1 || legacy_eight_byte_comparison, - "SimdAlgo comparison output is a packed one-bit mask"); + static_assert(WriteWidth == 1 || legacy_eight_byte_comparison, "SimdAlgo comparison output is a packed one-bit mask"); static_assert(count % write_data_size == 0, "Packed comparison output requires a whole number of destination elements"); const auto predicateVector = simd::set1(predicate); - simd::template transform_pack<1>(read, write, - [&predicateVector](const typename simd::vector_t value) noexcept - { return simd::movemask_slim(simd::cmpeq(value, predicateVector)); }); + simd::template transform_pack<1>(read, write, [&predicateVector](const typename simd::vector_t value) noexcept + { return simd::movemask_slim(simd::cmpeq(value, predicateVector)); }); } #pragma region Bitwise Operations (constrained) @@ -326,7 +337,7 @@ template struct SimdAlgo final }; template Select128, std::invocable Select256> - SIMDLIB_FORCE_INLINE constexpr static void ChooseSimd(std::size_t element_count, Select128 &&select128, Select256 &&select256) noexcept + constexpr static void SIMD_FLAGS(Neither, ForceInline, Flatten) ChooseSimd(std::size_t element_count, Select128 &&select128, Select256 &&select256) noexcept { if (element_count * read_data_size >= 256) std::invoke(select256, simd_256_tag{}); diff --git a/include/SimdLib/SimdApi.h b/include/SimdLib/SimdApi.h index a698f22..0a48299 100644 --- a/include/SimdLib/SimdApi.h +++ b/include/SimdLib/SimdApi.h @@ -6,12 +6,10 @@ namespace SimdLib { template -inline constexpr bool is_simd_api_available_v [[deprecated("Use SimdLib::is_api_available_v")]] = - is_api_available_v; +inline constexpr bool is_simd_api_available_v [[deprecated("Use SimdLib::is_api_available_v")]] = is_api_available_v; template concept SimdApiAvailable [[deprecated("Use SimdLib::ApiAvailable")]] = ApiAvailable; -template -using SimdApi [[deprecated("Use SimdLib::Api")]] = Api; +template using SimdApi [[deprecated("Use SimdLib::Api")]] = Api; } // namespace SimdLib diff --git a/include/SimdLib/SimdLib.h b/include/SimdLib/SimdLib.h index fe4dbe9..7bf732b 100644 --- a/include/SimdLib/SimdLib.h +++ b/include/SimdLib/SimdLib.h @@ -1,11 +1,17 @@ #pragma once -#include -#include #include -#include -#include +#include +#include +#include +#include +#if SIMDLIB_REGISTER_INTERFACE_AVAILABLE +#include +#include +#endif #include +#include #include -#include +#include +#include #include diff --git a/include/SimdLib/SimdResample.h b/include/SimdLib/SimdResample.h index f55e90a..be1d256 100644 --- a/include/SimdLib/SimdResample.h +++ b/include/SimdLib/SimdResample.h @@ -1,7 +1,7 @@ #pragma once -#include #include +#include #include #include @@ -17,9 +17,7 @@ namespace SimdLib::SimdResample /// `dst.size() == src.size() * 8` and maps each input bit to `0xFF` or `0x00`. /// Packs one bit per source byte, set when the byte is nonzero. -inline void ReduceBytesToBitsBy8_Any( - const std::span src, - const std::span dst) noexcept +inline void ReduceBytesToBitsBy8_Any(const std::span src, const std::span dst) noexcept { SIMDLIB_PRECONDITION(src.size() == dst.size() * 8, "ReduceBytesToBitsBy8_Any requires src.size() == dst.size() * 8"); @@ -53,9 +51,7 @@ inline void ReduceBytesToBitsBy8_Any( } /// Packs one bit per source byte, set when the byte is exactly `0xFF`. -inline void ReduceBytesToBitsBy8_All( - const std::span src, - const std::span dst) noexcept +inline void ReduceBytesToBitsBy8_All(const std::span src, const std::span dst) noexcept { SIMDLIB_PRECONDITION(src.size() == dst.size() * 8, "ReduceBytesToBitsBy8_All requires src.size() == dst.size() * 8"); @@ -87,9 +83,7 @@ inline void ReduceBytesToBitsBy8_All( } /// Packs one bit per source byte, set when the byte has odd parity. -inline void ReduceBytesToBitsBy8_Parity( - const std::span src, - const std::span dst) noexcept +inline void ReduceBytesToBitsBy8_Parity(const std::span src, const std::span dst) noexcept { SIMDLIB_PRECONDITION(src.size() == dst.size() * 8, "ReduceBytesToBitsBy8_Parity requires src.size() == dst.size() * 8"); @@ -98,11 +92,9 @@ inline void ReduceBytesToBitsBy8_Parity( using U16x8 = Api<128, std::uint16_t>; const auto lowNibbleMask = U8x16::set1(std::uint8_t{0x0F}); const auto one = U8x16::set1(std::uint8_t{1}); - const auto parityLut = U8x16::setr( - std::uint8_t{0}, std::uint8_t{1}, std::uint8_t{1}, std::uint8_t{0}, - std::uint8_t{1}, std::uint8_t{0}, std::uint8_t{0}, std::uint8_t{1}, - std::uint8_t{1}, std::uint8_t{0}, std::uint8_t{0}, std::uint8_t{1}, - std::uint8_t{0}, std::uint8_t{1}, std::uint8_t{1}, std::uint8_t{0}); + const auto parityLut = + U8x16::setr(std::uint8_t{0}, std::uint8_t{1}, std::uint8_t{1}, std::uint8_t{0}, std::uint8_t{1}, std::uint8_t{0}, std::uint8_t{0}, std::uint8_t{1}, + std::uint8_t{1}, std::uint8_t{0}, std::uint8_t{0}, std::uint8_t{1}, std::uint8_t{0}, std::uint8_t{1}, std::uint8_t{1}, std::uint8_t{0}); const auto parityMask = [&](const typename U8x16::vector_t value) noexcept { @@ -137,9 +129,7 @@ inline void ReduceBytesToBitsBy8_Parity( } /// Expands each packed source bit to one byte (`1 -> 0xFF`, `0 -> 0x00`). -inline void ExpandBitsToBytesBy8( - const std::span src, - const std::span dst) noexcept +inline void ExpandBitsToBytesBy8(const std::span src, const std::span dst) noexcept { SIMDLIB_PRECONDITION(dst.size() == src.size() * 8, "ExpandBitsToBytesBy8 requires dst.size() == src.size() * 8"); @@ -147,25 +137,19 @@ inline void ExpandBitsToBytesBy8( using U8x16 = Api<128, std::uint8_t>; const auto zero = U8x16::setzero(); const auto allOnes = U8x16::set1(std::uint8_t{0xFF}); - const auto laneMasks = U8x16::setr( - std::uint8_t{1}, std::uint8_t{2}, std::uint8_t{4}, std::uint8_t{8}, - std::uint8_t{16}, std::uint8_t{32}, std::uint8_t{64}, std::uint8_t{0x80}, - std::uint8_t{1}, std::uint8_t{2}, std::uint8_t{4}, std::uint8_t{8}, - std::uint8_t{16}, std::uint8_t{32}, std::uint8_t{64}, std::uint8_t{0x80}); + const auto laneMasks = U8x16::setr(std::uint8_t{1}, std::uint8_t{2}, std::uint8_t{4}, std::uint8_t{8}, std::uint8_t{16}, std::uint8_t{32}, std::uint8_t{64}, + std::uint8_t{0x80}, std::uint8_t{1}, std::uint8_t{2}, std::uint8_t{4}, std::uint8_t{8}, std::uint8_t{16}, + std::uint8_t{32}, std::uint8_t{64}, std::uint8_t{0x80}); const auto expandPair = [&](const std::uint8_t low, const std::uint8_t high) noexcept { - const auto bits = U8x16::setr( - low, low, low, low, low, low, low, low, - high, high, high, high, high, high, high, high); + const auto bits = U8x16::setr(low, low, low, low, low, low, low, low, high, high, high, high, high, high, high, high); const auto equalZero = U8x16::cmpeq(U8x16::bitwise_and(bits, laneMasks), zero); return U8x16::bitwise_andnot(equalZero, allOnes); }; const std::size_t pairCount = src.size() / 2; for (std::size_t pair = 0; pair < pairCount; ++pair) - U8x16::store_unaligned( - expandPair(src[pair * 2], src[pair * 2 + 1]), - std::span(dst.data() + pair * 16, 16)); + U8x16::store_unaligned(expandPair(src[pair * 2], src[pair * 2 + 1]), std::span(dst.data() + pair * 16, 16)); if ((src.size() & 1u) != 0) U8x16::store_half(expandPair(src.back(), 0), dst.data() + pairCount * 16); #else diff --git a/include/SimdLib/SimdVector.h b/include/SimdLib/SimdVector.h index b38057e..ed32da1 100644 --- a/include/SimdLib/SimdVector.h +++ b/include/SimdLib/SimdVector.h @@ -1,6 +1,6 @@ #pragma once -#include #include +#include #include #include #include @@ -60,17 +60,17 @@ class SimdVector final constexpr static inline mask_t inactive_cmp_mask = static_cast(full_cmp_mask & ~active_cmp_mask); - SIMDLIB_FORCE_INLINE constexpr static bool mask_has_any(const mask_t mask) noexcept + constexpr static bool SIMD_FLAGS(Neither, ForceInline, Flatten) mask_has_any(const mask_t mask) noexcept { return (mask & active_cmp_mask) != 0; } - SIMDLIB_FORCE_INLINE constexpr static bool mask_has_all(const mask_t mask) noexcept + constexpr static bool SIMD_FLAGS(Neither, ForceInline, Flatten) mask_has_all(const mask_t mask) noexcept { return (mask & active_cmp_mask) == active_cmp_mask; } - SIMDLIB_FORCE_INLINE constexpr static bool inactive_mask_has_all(const mask_t mask) noexcept + constexpr static bool SIMD_FLAGS(Neither, ForceInline, Flatten) inactive_mask_has_all(const mask_t mask) noexcept { return (mask & inactive_cmp_mask) == inactive_cmp_mask; } @@ -80,7 +80,8 @@ class SimdVector final * @param operation Name of the operation validating the result. * @return `value` unchanged. */ - template SIMDLIB_FORCE_INLINE constexpr static result_t CheckResultInactiveLanesZero(const result_t value, const char *operation) noexcept + template + constexpr static result_t SIMD_FLAGS(InOut, ForceInline, Flatten) CheckResultInactiveLanesZero(const result_t value, const char *operation) noexcept { #if SIMDLIB_ENABLE_CHECKS if constexpr (element_count != simd::element_count && std::same_as, vector_t>) @@ -102,19 +103,21 @@ class SimdVector final * @param fillValue Scalar written into every inactive hardware lane. * @return Register with unchanged active lanes and filled inactive lanes. */ - SIMDLIB_FORCE_INLINE constexpr static vector_t FillInactiveLanes(const vector_t value, const element_t fillValue) noexcept + constexpr static vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) FillInactiveLanes(const vector_t value, const element_t fillValue) noexcept { if constexpr (element_count == simd::element_count) { return value; } - - const auto lanes = simd::to_array(value); - return [&](std::index_sequence, - std::index_sequence) constexpr noexcept -> vector_t + else { - return simd::setr_partial(static_cast(lanes[ActiveIndices])..., ((void)FillIndices, fillValue)...); - }(std::make_index_sequence{}, std::make_index_sequence{}); + const auto lanes = simd::to_array(value); + return [&](std::index_sequence, + std::index_sequence) constexpr noexcept -> vector_t + { + return simd::setr_partial(static_cast(lanes[ActiveIndices])..., ((void)FillIndices, fillValue)...); + }(std::make_index_sequence{}, std::make_index_sequence{}); + } } #pragma endregion @@ -142,19 +145,16 @@ class SimdVector final /** @brief Constructs a new SIMD vector with all elements set to zero. * @return Zero-initialized SIMD vector storage. */ - SIMDLIB_FORCE_INLINE constexpr SimdVector() noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr SimdVector() noexcept { - if (std::is_constant_evaluated()) - m_data = {}; - else - m_data = simd::setzero(); + m_data = simd::setzero(); } /** @brief Constructs a new SIMD vector from a SIMD register. * @param data Source SIMD register. * @return SIMD vector that wraps `data` unchanged. */ - SIMDLIB_FORCE_INLINE constexpr SimdVector(vector_t data) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr SimdVector(vector_t data) noexcept { m_data = data; }; @@ -163,7 +163,7 @@ class SimdVector final * @param v Scalar value broadcast into every register lane. * @return SIMD vector whose lanes are all initialized from `v`. */ - SIMDLIB_FORCE_INLINE constexpr explicit SimdVector(element_t v) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit SimdVector(element_t v) noexcept { if constexpr (element_count == simd::element_count) { @@ -180,7 +180,7 @@ class SimdVector final * @param data Source span containing one full register worth of elements. * @return SIMD vector loaded from `data`. */ - SIMDLIB_FORCE_INLINE constexpr explicit SimdVector(std::span data) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit SimdVector(std::span data) noexcept { m_data = simd::load(std::span(data.data(), data.size())); }; @@ -189,7 +189,7 @@ class SimdVector final * @param data Source span containing one full register worth of elements. * @return SIMD vector loaded from `data`. */ - SIMDLIB_FORCE_INLINE constexpr explicit SimdVector(std::span data) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit SimdVector(std::span data) noexcept { m_data = simd::load(data); }; @@ -198,7 +198,7 @@ class SimdVector final * @param data Source span containing exactly the active logical elements. * @return SIMD vector loaded from `data` without requiring caller-side padding. */ - SIMDLIB_FORCE_INLINE constexpr explicit SimdVector(std::span data) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit SimdVector(std::span data) noexcept requires(element_count != simd::element_count) { m_data = simd::template load_partial(std::span(data)); @@ -208,7 +208,7 @@ class SimdVector final * @param data Source span containing exactly the active logical elements. * @return SIMD vector loaded from `data` without requiring caller-side padding. */ - SIMDLIB_FORCE_INLINE constexpr explicit SimdVector(std::span data) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit SimdVector(std::span data) noexcept requires(element_count != simd::element_count) { m_data = simd::template load_partial(data); @@ -218,7 +218,8 @@ class SimdVector final * @param data Source array containing one full register worth of elements. * @return SIMD vector loaded from `data`. */ - SIMDLIB_FORCE_INLINE constexpr explicit SimdVector(const std::array &data) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit SimdVector( + const std::array &data) noexcept { m_data = simd::construct(data); }; @@ -227,7 +228,7 @@ class SimdVector final * @param data Source array containing exactly the active logical elements. * @return SIMD vector loaded from `data` without requiring caller-side padding. */ - SIMDLIB_FORCE_INLINE constexpr explicit SimdVector(const std::array &data) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit SimdVector(const std::array &data) noexcept requires(element_count != simd::element_count) { m_data = simd::template load_partial(std::span(data)); @@ -240,7 +241,7 @@ class SimdVector final */ template requires(std::is_integral_v && std::is_integral_v && sizeof(source_t) < sizeof(element_t)) - SIMDLIB_FORCE_INLINE constexpr explicit SimdVector(const SimdVector &other) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit SimdVector(const SimdVector &other) noexcept { using source_simd = typename SimdVector::simd; m_data = source_simd::template widen(other.getRegister()); @@ -252,7 +253,7 @@ class SimdVector final */ template ... Args> requires(sizeof...(Args) == element_count) - SIMDLIB_FORCE_INLINE constexpr explicit SimdVector(Args &&...args) noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit SimdVector(Args &&...args) noexcept { m_data = simd::setr_partial(static_cast(std::forward(args))...); } @@ -266,7 +267,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing the per-lane sum. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator+(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) operator+(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::add(m_data, rhs), "SimdVector::operator+(vector_t)"); } @@ -275,7 +276,7 @@ class SimdVector final * @param rhs Scalar value added to every active logical element. * @return Register containing the per-lane sum. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator+(element_t rhs) const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) operator+(element_t rhs) const noexcept { const SimdVector scalarRhs(rhs); return simd::add(m_data, scalarRhs.getRegister()); @@ -285,7 +286,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing the per-lane difference. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator-(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) operator-(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::subtract(m_data, rhs), "SimdVector::operator-(vector_t)"); } @@ -294,7 +295,7 @@ class SimdVector final * @param rhs Scalar value subtracted from every active logical element. * @return Register containing the per-lane difference. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator-(element_t rhs) const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) operator-(element_t rhs) const noexcept { const SimdVector scalarRhs(rhs); return simd::subtract(m_data, scalarRhs.getRegister()); @@ -304,7 +305,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing the per-lane product. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator*(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) operator*(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::multiply(m_data, rhs), "SimdVector::operator*(vector_t)"); } @@ -313,7 +314,7 @@ class SimdVector final * @param rhs Scalar value multiplied into every active logical element. * @return Register containing the per-lane product. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator*(element_t rhs) const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) operator*(element_t rhs) const noexcept { const SimdVector scalarRhs(rhs); return simd::multiply(m_data, scalarRhs.getRegister()); @@ -325,7 +326,7 @@ class SimdVector final * @return SIMD vector containing `(this - minInclusive + 1)` per active lane, widened when needed. */ template - SIMDLIB_FORCE_INLINE auto VECTORCALL size(vector_t minInclusive) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) size(vector_t minInclusive) const noexcept requires(std::is_integral_v && std::is_integral_v && sizeof(target_element_t) >= sizeof(element_t)) { if constexpr (sizeof(target_element_t) > sizeof(element_t)) @@ -349,7 +350,7 @@ class SimdVector final * @return Product of `(this - minInclusive + 1)` over the active logical lanes. */ template - SIMDLIB_FORCE_INLINE auto VECTORCALL area(vector_t minInclusive) const noexcept + auto SIMD_FLAGS(In, ForceInline, Flatten) area(vector_t minInclusive) const noexcept requires(std::is_integral_v && std::is_integral_v && sizeof(target_element_t) >= sizeof(element_t)) { return this->template size(minInclusive).area(); @@ -359,7 +360,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing the per-lane quotient. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator/(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) operator/(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::divide(m_data, FillInactiveLanes(rhs, element_t{1})), "SimdVector::operator/(vector_t)"); } @@ -368,7 +369,7 @@ class SimdVector final * @param rhs Scalar value that divides every active logical element. * @return Register containing the per-lane quotient. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator/(element_t rhs) const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) operator/(element_t rhs) const noexcept { return simd::divide(m_data, simd::set1(rhs)); } @@ -377,7 +378,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing the per-lane remainder. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator%(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) operator%(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::modulus(m_data, FillInactiveLanes(rhs, element_t{1})), "SimdVector::operator%(vector_t)"); } @@ -386,7 +387,7 @@ class SimdVector final * @param rhs Scalar value used as the modulus for every active logical element. * @return Register containing the per-lane remainder. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator%(element_t rhs) const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) operator%(element_t rhs) const noexcept { return simd::modulus(m_data, simd::set1(rhs)); } @@ -394,7 +395,7 @@ class SimdVector final /** @brief Negates each lane of this vector. * @return Register containing the per-lane negation. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL operator-() const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) operator-() const noexcept { return simd::negate(m_data); } @@ -403,7 +404,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator+=(vector_t rhs) noexcept + auto SIMD_FLAGS(In, ForceInline, Flatten) operator+=(vector_t rhs) noexcept -> SimdVector & { m_data = CheckResultInactiveLanesZero(simd::add(m_data, rhs), "SimdVector::operator+=(vector_t)"); return *this; @@ -413,7 +414,7 @@ class SimdVector final * @param rhs Scalar value added to every active logical element. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator+=(element_t rhs) noexcept + auto SIMD_FLAGS(Neither, ForceInline, Flatten) operator+=(element_t rhs) noexcept -> SimdVector & { const SimdVector scalarRhs(rhs); m_data = simd::add(m_data, scalarRhs.getRegister()); @@ -424,7 +425,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator-=(vector_t rhs) noexcept + auto SIMD_FLAGS(In, ForceInline, Flatten) operator-=(vector_t rhs) noexcept -> SimdVector & { m_data = CheckResultInactiveLanesZero(simd::subtract(m_data, rhs), "SimdVector::operator-=(vector_t)"); return *this; @@ -434,7 +435,7 @@ class SimdVector final * @param rhs Scalar value subtracted from every active logical element. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator-=(element_t rhs) noexcept + auto SIMD_FLAGS(Neither, ForceInline, Flatten) operator-=(element_t rhs) noexcept -> SimdVector & { const SimdVector scalarRhs(rhs); m_data = simd::subtract(m_data, scalarRhs.getRegister()); @@ -445,7 +446,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator*=(vector_t rhs) noexcept + auto SIMD_FLAGS(In, ForceInline, Flatten) operator*=(vector_t rhs) noexcept -> SimdVector & { m_data = CheckResultInactiveLanesZero(simd::multiply(m_data, rhs), "SimdVector::operator*=(vector_t)"); return *this; @@ -455,7 +456,7 @@ class SimdVector final * @param rhs Scalar value multiplied into every active logical element. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator*=(element_t rhs) noexcept + auto SIMD_FLAGS(Neither, ForceInline, Flatten) operator*=(element_t rhs) noexcept -> SimdVector & { const SimdVector scalarRhs(rhs); m_data = simd::multiply(m_data, scalarRhs.getRegister()); @@ -466,7 +467,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator/=(vector_t rhs) noexcept + auto SIMD_FLAGS(In, ForceInline, Flatten) operator/=(vector_t rhs) noexcept -> SimdVector & { m_data = CheckResultInactiveLanesZero(simd::divide(m_data, FillInactiveLanes(rhs, element_t{1})), "SimdVector::operator/=(vector_t)"); return *this; @@ -476,7 +477,7 @@ class SimdVector final * @param rhs Scalar value that divides every active logical element. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator/=(element_t rhs) noexcept + auto SIMD_FLAGS(Neither, ForceInline, Flatten) operator/=(element_t rhs) noexcept -> SimdVector & { m_data = simd::divide(m_data, simd::set1(rhs)); return *this; @@ -486,7 +487,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator%=(vector_t rhs) noexcept + auto SIMD_FLAGS(In, ForceInline, Flatten) operator%=(vector_t rhs) noexcept -> SimdVector & { m_data = CheckResultInactiveLanesZero(simd::modulus(m_data, FillInactiveLanes(rhs, element_t{1})), "SimdVector::operator%=(vector_t)"); return *this; @@ -496,7 +497,7 @@ class SimdVector final * @param rhs Scalar value used as the modulus for every active logical element. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator%=(element_t rhs) noexcept + auto SIMD_FLAGS(Neither, ForceInline, Flatten) operator%=(element_t rhs) noexcept -> SimdVector & { m_data = simd::modulus(m_data, simd::set1(rhs)); return *this; @@ -510,7 +511,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Saturated sum of `m_data` and `rhs`. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL add_saturated(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) add_saturated(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::add_saturated(lhsValue, rhsValue); } { return CheckResultInactiveLanesZero(simd::add_saturated(m_data, rhs), "SimdVector::add_saturated(vector_t)"); @@ -520,7 +521,7 @@ class SimdVector final * @param rhs Scalar value added to every active logical element. * @return Saturated sum of `m_data` and the broadcast scalar value. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL add_saturated(element_t rhs) const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) add_saturated(element_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::add_saturated(lhsValue, rhsValue); } { const SimdVector scalarRhs(rhs); @@ -531,7 +532,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Saturated difference of `m_data` and `rhs`. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL subtract_saturated(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) subtract_saturated(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::subtract_saturated(lhsValue, rhsValue); } { return CheckResultInactiveLanesZero(simd::subtract_saturated(m_data, rhs), "SimdVector::subtract_saturated(vector_t)"); @@ -541,7 +542,7 @@ class SimdVector final * @param rhs Scalar value subtracted from every active logical element. * @return Saturated difference of `m_data` and the scalar value. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL subtract_saturated(element_t rhs) const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) subtract_saturated(element_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::subtract_saturated(lhsValue, rhsValue); } { const SimdVector scalarRhs(rhs); @@ -552,7 +553,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Saturated product of `m_data` and `rhs`. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL multiply_saturated(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) multiply_saturated(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::multiply_saturated(lhsValue, rhsValue); } { return CheckResultInactiveLanesZero(simd::multiply_saturated(m_data, rhs), "SimdVector::multiply_saturated(vector_t)"); @@ -562,7 +563,7 @@ class SimdVector final * @param rhs Scalar value multiplied into every active logical element. * @return Saturated product of `m_data` and the scalar value. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL multiply_saturated(element_t rhs) const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) multiply_saturated(element_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::multiply_saturated(lhsValue, rhsValue); } { const SimdVector scalarRhs(rhs); @@ -576,7 +577,7 @@ class SimdVector final /** @brief Inverts every bit in the underlying register. * @return SIMD vector containing the bitwise inverse. */ - SIMDLIB_FORCE_INLINE SimdVector VECTORCALL operator~() const noexcept + SimdVector SIMD_FLAGS(Out, ForceInline, Flatten) operator~() const noexcept { if constexpr (element_count == simd::element_count) { @@ -595,7 +596,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return SIMD vector containing the bitwise AND result. */ - SIMDLIB_FORCE_INLINE SimdVector VECTORCALL operator&(vector_t rhs) const noexcept + SimdVector SIMD_FLAGS(InOut, ForceInline, Flatten) operator&(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::bitwise_and(m_data, rhs), "SimdVector::operator&(vector_t)"); } @@ -604,7 +605,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return SIMD vector containing the bitwise OR result. */ - SIMDLIB_FORCE_INLINE SimdVector VECTORCALL operator|(vector_t rhs) const noexcept + SimdVector SIMD_FLAGS(InOut, ForceInline, Flatten) operator|(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::bitwise_or(m_data, rhs), "SimdVector::operator|(vector_t)"); } @@ -613,7 +614,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return SIMD vector containing the bitwise XOR result. */ - SIMDLIB_FORCE_INLINE SimdVector VECTORCALL operator^(vector_t rhs) const noexcept + SimdVector SIMD_FLAGS(InOut, ForceInline, Flatten) operator^(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::bitwise_xor(m_data, rhs), "SimdVector::operator^(vector_t)"); } @@ -622,7 +623,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator&=(vector_t rhs) noexcept + auto SIMD_FLAGS(In, ForceInline, Flatten) operator&=(vector_t rhs) noexcept -> SimdVector & { m_data = CheckResultInactiveLanesZero(simd::bitwise_and(m_data, rhs), "SimdVector::operator&=(vector_t)"); return *this; @@ -632,7 +633,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator|=(vector_t rhs) noexcept + auto SIMD_FLAGS(In, ForceInline, Flatten) operator|=(vector_t rhs) noexcept -> SimdVector & { m_data = CheckResultInactiveLanesZero(simd::bitwise_or(m_data, rhs), "SimdVector::operator|=(vector_t)"); return *this; @@ -642,7 +643,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE SimdVector &VECTORCALL operator^=(vector_t rhs) noexcept + auto SIMD_FLAGS(In, ForceInline, Flatten) operator^=(vector_t rhs) noexcept -> SimdVector & { m_data = CheckResultInactiveLanesZero(simd::bitwise_xor(m_data, rhs), "SimdVector::operator^=(vector_t)"); return *this; @@ -656,7 +657,7 @@ class SimdVector final * @param shift Shift count applied to every active lane. * @return SIMD vector containing the shifted values. */ - SIMDLIB_FORCE_INLINE constexpr SimdVector VECTORCALL operator<<(int shift) const noexcept + constexpr SimdVector SIMD_FLAGS(Out, ForceInline, Flatten) operator<<(int shift) const noexcept { return simd::shift_left(m_data, shift); } @@ -665,7 +666,7 @@ class SimdVector final * @param shift Shift count applied to every active lane. * @return SIMD vector containing the shifted values using arithmetic or logical shift semantics for the element type. */ - SIMDLIB_FORCE_INLINE constexpr SimdVector VECTORCALL operator>>(int shift) const noexcept + constexpr SimdVector SIMD_FLAGS(Out, ForceInline, Flatten) operator>>(int shift) const noexcept { if constexpr (std::is_signed_v) return simd::shift_right_arithmetic(m_data, shift); @@ -677,7 +678,7 @@ class SimdVector final * @param shift Shift count applied to every active lane. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE constexpr SimdVector &VECTORCALL operator<<=(int shift) noexcept + constexpr auto SIMD_FLAGS(Neither, ForceInline, Flatten) operator<<=(int shift) noexcept -> SimdVector & { m_data = simd::shift_left(m_data, shift); return *this; @@ -687,7 +688,7 @@ class SimdVector final * @param shift Shift count applied to every active lane. * @return Reference to this SIMD vector after the update. */ - SIMDLIB_FORCE_INLINE constexpr SimdVector &VECTORCALL operator>>=(int shift) noexcept + constexpr auto SIMD_FLAGS(Neither, ForceInline, Flatten) operator>>=(int shift) noexcept -> SimdVector & { if constexpr (std::is_signed_v) m_data = simd::shift_right_arithmetic(m_data, shift); @@ -704,7 +705,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return `true` when every active element compares equal. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL operator==(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) operator==(vector_t rhs) const noexcept { return mask_has_all(simd::cmp_eq_mask(m_data, rhs)); } @@ -713,43 +714,43 @@ class SimdVector final * @param rhs Right-hand input register. * @return `true` when every active element is greater than its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL operator>(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) operator>(vector_t rhs) const noexcept { - return mask_has_all(simd::cmp_gt(m_data, rhs)); + return mask_has_all(simd::cmp_gt_mask(m_data, rhs)); } /** @brief Returns true if all elements are greater than or equal to the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when every active element is greater than or equal to its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL operator>=(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) operator>=(vector_t rhs) const noexcept { - return mask_has_all(simd::cmp_ge(m_data, rhs)); + return mask_has_all(simd::cmp_ge_mask(m_data, rhs)); } /** @brief Returns true if all elements are less than the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when every active element is less than its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL operator<(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) operator<(vector_t rhs) const noexcept { - return mask_has_all(simd::cmp_lt(m_data, rhs)); + return mask_has_all(simd::cmp_lt_mask(m_data, rhs)); } /** @brief Returns true if all elements are less than or equal to the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when every active element is less than or equal to its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL operator<=(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) operator<=(vector_t rhs) const noexcept { - return mask_has_all(simd::cmp_le(m_data, rhs)); + return mask_has_all(simd::cmp_le_mask(m_data, rhs)); } /** @brief Returns true if any element equals the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when at least one active element compares equal. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL any_equal(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) any_equal(vector_t rhs) const noexcept { return mask_has_any(simd::cmp_eq_mask(m_data, rhs)); } @@ -758,7 +759,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return `true` when every active element compares equal. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL all_equal(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) all_equal(vector_t rhs) const noexcept { return mask_has_all(simd::cmp_eq_mask(m_data, rhs)); } @@ -767,72 +768,72 @@ class SimdVector final * @param rhs Right-hand input register. * @return `true` when at least one active element is greater than its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL any_greater(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) any_greater(vector_t rhs) const noexcept { - return mask_has_any(simd::cmp_gt(m_data, rhs)); + return mask_has_any(simd::cmp_gt_mask(m_data, rhs)); } /** @brief Returns true if all elements are greater than the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when every active element is greater than its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL all_greater(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) all_greater(vector_t rhs) const noexcept { - return mask_has_all(simd::cmp_gt(m_data, rhs)); + return mask_has_all(simd::cmp_gt_mask(m_data, rhs)); } /** @brief Returns true if any element is greater than or equal to the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when at least one active element is greater than or equal to its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL any_greater_equal(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) any_greater_equal(vector_t rhs) const noexcept { - return mask_has_any(simd::cmp_ge(m_data, rhs)); + return mask_has_any(simd::cmp_ge_mask(m_data, rhs)); } /** @brief Returns true if all elements are greater than or equal to the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when every active element is greater than or equal to its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL all_greater_equal(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) all_greater_equal(vector_t rhs) const noexcept { - return mask_has_all(simd::cmp_ge(m_data, rhs)); + return mask_has_all(simd::cmp_ge_mask(m_data, rhs)); } /** @brief Returns true if any element is less than the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when at least one active element is less than its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL any_less(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) any_less(vector_t rhs) const noexcept { - return mask_has_any(simd::cmp_lt(m_data, rhs)); + return mask_has_any(simd::cmp_lt_mask(m_data, rhs)); } /** @brief Returns true if all elements are less than the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when every active element is less than its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL all_less(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) all_less(vector_t rhs) const noexcept { - return mask_has_all(simd::cmp_lt(m_data, rhs)); + return mask_has_all(simd::cmp_lt_mask(m_data, rhs)); } /** @brief Returns true if any element is less than or equal to the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when at least one active element is less than or equal to its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL any_less_equal(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) any_less_equal(vector_t rhs) const noexcept { - return mask_has_any(simd::cmp_le(m_data, rhs)); + return mask_has_any(simd::cmp_le_mask(m_data, rhs)); } /** @brief Returns true if all elements are less than or equal to the corresponding element in the other vector. * @param rhs Right-hand input register. * @return `true` when every active element is less than or equal to its counterpart. */ - SIMDLIB_FORCE_INLINE constexpr bool VECTORCALL all_less_equal(vector_t rhs) const noexcept + constexpr bool SIMD_FLAGS(In, ForceInline, Flatten) all_less_equal(vector_t rhs) const noexcept { - return mask_has_all(simd::cmp_le(m_data, rhs)); + return mask_has_all(simd::cmp_le_mask(m_data, rhs)); } #pragma endregion @@ -843,7 +844,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register whose lanes are `min(m_data[i], rhs[i])`. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL min(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) min(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::min(m_data, rhs), "SimdVector::min(vector_t)"); } @@ -852,7 +853,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register whose lanes are `max(m_data[i], rhs[i])`. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL max(vector_t rhs) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) max(vector_t rhs) const noexcept { return CheckResultInactiveLanesZero(simd::max(m_data, rhs), "SimdVector::max(vector_t)"); } @@ -864,7 +865,7 @@ class SimdVector final /** @brief Returns a SIMD register containing the absolute value of each element. * @return Register containing the per-element absolute values of `m_data`. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL abs() const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) abs() const noexcept requires requires(vector_t value) { simd::absolute(value); } { return simd::absolute(m_data); @@ -873,25 +874,33 @@ class SimdVector final /** @brief Computes the square root of each element. * @return Register containing the per-lane square roots. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL sqrt() const noexcept + auto SIMD_FLAGS(Out, ForceInline, Flatten) sqrt() const noexcept requires requires(vector_t value) { simd::sqrt(value); } { return simd::sqrt(m_data); } - /** @brief Computes the per-128-bit-lane magnitude when the underlying Simd specialization supports it. - * @return Register containing the lane-local magnitudes broadcast across each lane group. + /** @brief Computes broadcast floating magnitudes or sparse unchecked integer magnitudes for each 128-bit group. + * @return The underlying magnitude register; only each group-leading lane is specified for integer elements. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL magnitude() const noexcept + auto SIMD_FLAGS(Out, ForceInline, Flatten) magnitude() const noexcept requires requires(vector_t value) { simd::magnitude(value); } { return simd::magnitude(m_data); } + /** @brief Computes saturated integer magnitudes followed by canonical overflow-mask lanes. + * @return Each 128-bit group stores its magnitude in lane zero and overflow mask in lane one. + */ + auto SIMD_FLAGS(Out, ForceInline, Flatten) magnitude_checked() const noexcept + requires requires(vector_t value) { simd::magnitude_checked(value); } + { + return simd::magnitude_checked(m_data); + } /** @brief Computes the multiplicative product of the active logical lanes. * @return Product of the declared logical lanes, widened to 32-bit for sub-32-bit integer vectors and reduced modulo the result width. */ - SIMDLIB_FORCE_INLINE area_element_t VECTORCALL area() const noexcept + area_element_t SIMD_FLAGS(Neither, ForceInline, Flatten) area() const noexcept requires std::is_integral_v { if constexpr (element_count == 1) @@ -929,7 +938,7 @@ class SimdVector final /** @brief Normalizes floating-point lanes using the Simd API's lane-local length semantics. * @return Register containing normalized per-lane values. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL normalize() const noexcept + auto SIMD_FLAGS(Out, ForceInline, Flatten) normalize() const noexcept requires requires(vector_t value) { simd::normalize(value); } { return simd::normalize(m_data); @@ -939,7 +948,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing the per-lane averages. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL avg(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) avg(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::avg(lhsValue, rhsValue); } { return CheckResultInactiveLanesZero(simd::avg(m_data, rhs), "SimdVector::avg(vector_t)"); @@ -950,7 +959,7 @@ class SimdVector final * @param addend Register added to the product. * @return Register containing the multiply-add result. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL multiply_add(vector_t rhs, vector_t addend) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) multiply_add(vector_t rhs, vector_t addend) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue, vector_t addValue) { simd::multiply_add(lhsValue, rhsValue, addValue); } { return CheckResultInactiveLanesZero(simd::multiply_add(m_data, rhs, addend), "SimdVector::multiply_add(vector_t, vector_t)"); @@ -960,7 +969,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing pairwise horizontal sums. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL add_horizontal(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) add_horizontal(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::add_horizontal(lhsValue, rhsValue); } { return simd::add_horizontal(m_data, rhs); @@ -970,7 +979,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing pairwise horizontal differences. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL subtract_horizontal(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) subtract_horizontal(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::subtract_horizontal(lhsValue, rhsValue); } { return simd::subtract_horizontal(m_data, rhs); @@ -980,7 +989,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing saturated horizontal sums. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL add_horizontal_saturated(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) add_horizontal_saturated(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::hadd_saturated(lhsValue, rhsValue); } { return simd::hadd_saturated(m_data, rhs); @@ -990,7 +999,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing saturated horizontal differences. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL subtract_horizontal_saturated(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) subtract_horizontal_saturated(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::hsubtract_saturated(lhsValue, rhsValue); } { return simd::hsubtract_saturated(m_data, rhs); @@ -1000,7 +1009,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register whose lane type follows the promoted integer mapping. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL multiply_add_adjacent(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) multiply_add_adjacent(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::multiply_add_adjacent(lhsValue, rhsValue); } { return simd::multiply_add_adjacent(m_data, rhs); @@ -1010,7 +1019,7 @@ class SimdVector final * @param rhs Right-hand input register whose bytes are interpreted as signed. * @return Register containing signed 16-bit accumulation results. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL multiply_add_unsigned_signed_bytes(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) multiply_add_unsigned_signed_bytes(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::multiply_add_unsigned_signed_bytes(lhsValue, rhsValue); } { return CheckResultInactiveLanesZero(simd::multiply_add_unsigned_signed_bytes(m_data, rhs), "SimdVector::multiply_add_unsigned_signed_bytes(vector_t)"); @@ -1020,7 +1029,7 @@ class SimdVector final * @param rhs Right-hand input register interpreted byte-wise. * @return Register containing 64-bit absolute-difference accumulations. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL sum_absolute_byte_differences(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) sum_absolute_byte_differences(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::sum_absolute_byte_differences(lhsValue, rhsValue); } { return CheckResultInactiveLanesZero(simd::sum_absolute_byte_differences(m_data, rhs), "SimdVector::sum_absolute_byte_differences(vector_t)"); @@ -1032,7 +1041,7 @@ class SimdVector final * @return Register containing byte-window absolute-difference accumulations. */ template - SIMDLIB_FORCE_INLINE auto VECTORCALL multi_sum_absolute_byte_differences(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) multi_sum_absolute_byte_differences(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::template multi_sum_absolute_byte_differences(lhsValue, rhsValue); } { return CheckResultInactiveLanesZero(simd::template multi_sum_absolute_byte_differences(m_data, rhs), @@ -1042,7 +1051,7 @@ class SimdVector final /** @brief Returns the first index of the minimum value in the vector. * @return Zero-based index of the first minimum element. */ - SIMDLIB_FORCE_INLINE std::size_t VECTORCALL min_position() const noexcept + std::size_t SIMD_FLAGS(Neither, ForceInline, Flatten) min_position() const noexcept requires requires(vector_t value) { simd::min_position(value); } { return simd::min_position(FillInactiveLanes(m_data, std::numeric_limits::max())); @@ -1051,7 +1060,7 @@ class SimdVector final /** @brief Returns the first index of the maximum value in the vector. * @return Zero-based index of the first maximum element. */ - SIMDLIB_FORCE_INLINE std::size_t VECTORCALL max_position() const noexcept + std::size_t SIMD_FLAGS(Neither, ForceInline, Flatten) max_position() const noexcept requires requires(vector_t value) { simd::max_position(value); } { return simd::max_position(FillInactiveLanes(m_data, std::numeric_limits::lowest())); @@ -1061,7 +1070,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Register containing alternating subtract/add results. */ - SIMDLIB_FORCE_INLINE auto VECTORCALL add_subtract(vector_t rhs) const noexcept + auto SIMD_FLAGS(InOut, ForceInline, Flatten) add_subtract(vector_t rhs) const noexcept requires requires(vector_t lhsValue, vector_t rhsValue) { simd::add_subtract(lhsValue, rhsValue); } { return simd::add_subtract(m_data, rhs); @@ -1071,7 +1080,7 @@ class SimdVector final * @param rhs Right-hand input register. * @return Scalar dot-product result for the active vector dimensions. */ - SIMDLIB_FORCE_INLINE element_t VECTORCALL dot_product(vector_t rhs) const noexcept + element_t SIMD_FLAGS(In, ForceInline, Flatten) dot_product(vector_t rhs) const noexcept requires(std::is_floating_point_v && requires(vector_t lhsValue, vector_t rhsValue) { simd::template dot_product<0x11>(lhsValue, rhsValue); }) { @@ -1081,11 +1090,11 @@ class SimdVector final constexpr int lowActiveCount = element_count < laneElementCount ? element_count : laneElementCount; constexpr int lowMask = (((1 << lowActiveCount) - 1) << 4) | 0x1; const auto partial = simd::template dot_product(m_data, rhs); - element_t result = simd::get_element(partial, 0); + element_t result = simd::extract_slow(partial, 0); if constexpr (simd_width == 256 && element_count > laneElementCount) { - result = static_cast(result + simd::get_element(partial, laneElementCount)); + result = static_cast(result + simd::extract_slow(partial, laneElementCount)); } return result; @@ -1096,11 +1105,11 @@ class SimdVector final constexpr int lowActiveCount = element_count < laneElementCount ? element_count : laneElementCount; constexpr int lowMask = (((1 << lowActiveCount) - 1) << 4) | 0x1; const auto partial = simd::template dot_product(m_data, rhs); - element_t result = simd::get_element(partial, 0); + element_t result = simd::extract_slow(partial, 0); if constexpr (simd_width == 256 && element_count > laneElementCount) { - result = static_cast(result + simd::get_element(partial, laneElementCount)); + result = static_cast(result + simd::extract_slow(partial, laneElementCount)); } return result; @@ -1112,7 +1121,7 @@ class SimdVector final * @param maxValue Register containing the per-element upper bounds. * @return Register containing `m_data` clamped to `[minValue, maxValue]` per lane. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL clamp(vector_t minValue, vector_t maxValue) const noexcept + vector_t SIMD_FLAGS(InOut, ForceInline, Flatten) clamp(vector_t minValue, vector_t maxValue) const noexcept requires requires(vector_t value) { simd::min(value, value); simd::max(value, value); @@ -1129,7 +1138,7 @@ class SimdVector final * @param maxValue Scalar upper bound broadcast to every lane. * @return Register containing `m_data` clamped to `[minValue, maxValue]` per lane. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL clamp(element_t minValue, element_t maxValue) const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) clamp(element_t minValue, element_t maxValue) const noexcept requires requires(vector_t value) { simd::min(value, value); simd::max(value, value); @@ -1141,7 +1150,7 @@ class SimdVector final /** @brief Returns the sign of each element as -1, 0, or 1, or 0 and 1 for unsigned types. * @return Register containing the per-element sign classification of `m_data`. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL sign() const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) sign() const noexcept requires requires(vector_t value) { simd::cmpgt(value, value); simd::bitwise_and(value, value); @@ -1173,7 +1182,7 @@ class SimdVector final /** @brief Implicitly converts this wrapper to the underlying SIMD register. * @return Copy of the wrapped SIMD register. */ - SIMDLIB_FORCE_INLINE VECTORCALL operator vector_t() const noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE SIMDLIB_METHOD_FLAGS_VECTORCALL operator vector_t() const noexcept { return m_data; } @@ -1181,7 +1190,7 @@ class SimdVector final /** @brief Returns a mutable span view over the underlying register storage. * @return Mutable span covering every hardware lane in the register. */ - SIMDLIB_FORCE_INLINE operator std::span() noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE operator std::span() noexcept { return std::span(Detail::register_data(m_data), simd::element_count); } @@ -1189,7 +1198,7 @@ class SimdVector final /** @brief Returns a readonly span view over the underlying register storage. * @return Readonly span covering every hardware lane in the register. */ - SIMDLIB_FORCE_INLINE operator std::span() const noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE operator std::span() const noexcept { return std::span(Detail::register_data(m_data), simd::element_count); } @@ -1197,7 +1206,7 @@ class SimdVector final /** @brief Converts the wrapped SIMD register to a fixed array. * @return Array containing the full underlying register contents in lane order. */ - SIMDLIB_FORCE_INLINE constexpr explicit operator std::array() const noexcept + SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_FORCE_INLINE constexpr explicit operator std::array() const noexcept { return simd::to_array(m_data); } @@ -1205,7 +1214,7 @@ class SimdVector final /** @brief Converts the SIMD vector to an array of elements. * @return Array containing the full underlying register contents in lane order. */ - SIMDLIB_FORCE_INLINE constexpr std::array toArray() const noexcept + constexpr std::array SIMD_FLAGS(Neither, ForceInline, Flatten) toArray() const noexcept { return static_cast>(*this); } @@ -1213,7 +1222,7 @@ class SimdVector final /** @brief Returns a span over the SIMD vector's elements. * @return Mutable span view of the full underlying register storage. */ - SIMDLIB_FORCE_INLINE std::span getSpan() noexcept + std::span SIMD_FLAGS(Neither, ForceInline, Flatten) getSpan() noexcept { return static_cast>(*this); } @@ -1221,7 +1230,7 @@ class SimdVector final /** @brief Returns a readonly span over the SIMD vector's elements. * @return Readonly span view of the full underlying register storage. */ - SIMDLIB_FORCE_INLINE std::span getSpan() const noexcept + std::span SIMD_FLAGS(Neither, ForceInline, Flatten) getSpan() const noexcept { return static_cast>(*this); } @@ -1229,7 +1238,7 @@ class SimdVector final /** @brief Returns the underlying SIMD register. * @return Mutable reference to the wrapped SIMD register. */ - SIMDLIB_FORCE_INLINE vector_t &VECTORCALL getRegister() noexcept + auto SIMD_FLAGS(Neither, ForceInline, Flatten) getRegister() noexcept -> vector_t & { return m_data; } @@ -1237,7 +1246,7 @@ class SimdVector final /** @brief Returns the underlying SIMD register. * @return Copy of the wrapped SIMD register. */ - SIMDLIB_FORCE_INLINE vector_t VECTORCALL getRegister() const noexcept + vector_t SIMD_FLAGS(Out, ForceInline, Flatten) getRegister() const noexcept { return m_data; } @@ -1245,7 +1254,7 @@ class SimdVector final /** @brief Returns a tuple containing the span view used by tuple-like integrations. * @return Tuple containing the readonly span view of this SIMD vector. */ - SIMDLIB_FORCE_INLINE constexpr auto getTuple() const noexcept + constexpr auto SIMD_FLAGS(Neither, ForceInline, Flatten) getTuple() const noexcept { return std::tuple{this->getSpan()}; } @@ -1462,57 +1471,4 @@ template struct hash; -using VectorUInt8 = SimdVector; - -using VectorInt16 = SimdVector; -using VectorUInt16 = SimdVector; - -using VectorInt32 = SimdVector; -using VectorUInt32 = SimdVector; - -using VectorInt64 = SimdVector; -using VectorUInt64 = SimdVector; - -#pragma endregion - -#pragma region Type Aliases (Unsigned) - -using uint8x16 = SimdVector; -using uint8x32 = SimdVector; - -using uint16x8 = SimdVector; -using uint16x16 = SimdVector; - -using uint32x4 = SimdVector; -using uint32x8 = SimdVector; - -using uint64x2 = SimdVector; -using uint64x4 = SimdVector; - #pragma endregion - -#pragma region Type Aliases (Signed) - -using int8x16 = SimdVector; -using int8x32 = SimdVector; - -using int16x8 = SimdVector; -using int16x16 = SimdVector; - -using int32x4 = SimdVector; -using int32x8 = SimdVector; - -using int64x2 = SimdVector; -using int64x4 = SimdVector; - -#pragma endregion - -} // namespace SimdLib diff --git a/include/SimdLib/TemplateTools.h b/include/SimdLib/TemplateTools.h index 4ffcaf5..8409b08 100644 --- a/include/SimdLib/TemplateTools.h +++ b/include/SimdLib/TemplateTools.h @@ -39,8 +39,7 @@ using select_signed_integer_t = #pragma region Concepts template -concept integer_like = std::numeric_limits::is_specialized && std::numeric_limits::is_integer && - !std::same_as, bool>; +concept integer_like = std::numeric_limits::is_specialized && std::numeric_limits::is_integer && !std::same_as, bool>; #pragma endregion diff --git a/include/SimdLib/UInt128.h b/include/SimdLib/UInt128.h index b0d3135..1cdf21f 100644 --- a/include/SimdLib/UInt128.h +++ b/include/SimdLib/UInt128.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include #include #include @@ -31,19 +31,15 @@ class uint128_t final private: alignas(16) std::array m_data{0, 0}; - template - struct simd_element + template struct simd_element { using type = std::uint64_t; }; /** @brief Internal facade used whenever the 128-bit SIMD backend is available. */ - template - using simd = Api<128, typename simd_element::type>; + template using simd = Api<128, typename simd_element::type>; - template - inline static constexpr bool simd_available = - is_api_available_v<128, typename simd_element::type>; + template inline static constexpr bool simd_available = is_api_available_v<128, typename simd_element::type>; public: using block_t = std::uint64_t; @@ -51,32 +47,22 @@ class uint128_t final inline static constexpr std::size_t block_width = std::numeric_limits::digits; constexpr uint128_t() noexcept = default; - constexpr uint128_t(const uint128_t&) noexcept = default; - constexpr uint128_t(uint128_t&&) noexcept = default; - constexpr uint128_t& operator=(const uint128_t&) noexcept = default; - constexpr uint128_t& operator=(uint128_t&&) noexcept = default; + constexpr uint128_t(const uint128_t &) noexcept = default; + constexpr uint128_t(uint128_t &&) noexcept = default; + constexpr uint128_t &operator=(const uint128_t &) noexcept = default; + constexpr uint128_t &operator=(uint128_t &&) noexcept = default; constexpr ~uint128_t() = default; /** @brief Constructs a value from low and high words, in that order. */ - constexpr uint128_t(const std::uint64_t lower, const std::uint64_t upper) noexcept - : m_data{lower, upper} - { - } + constexpr uint128_t(const std::uint64_t lower, const std::uint64_t upper) noexcept : m_data{lower, upper} {} - template - constexpr uint128_t(const T value) noexcept - : m_data{static_cast(value), 0} - { - } + template constexpr uint128_t(const T value) noexcept : m_data{static_cast(value), 0} {} - constexpr uint128_t(const bool value) noexcept - : m_data{static_cast(value), 0} - { - } + constexpr uint128_t(const bool value) noexcept : m_data{static_cast(value), 0} {} /** @brief Loads the stored words into a backend register through Api. */ template - requires(simd_available) + requires(simd_available) [[nodiscard]] auto to_register() const noexcept -> typename simd::vector_t { return simd::load_aligned(std::span{m_data}); @@ -84,41 +70,40 @@ class uint128_t final /** @brief Constructs a value by extracting both words from a backend register through Api. */ template - requires(simd_available) + requires(simd_available) [[nodiscard]] static uint128_t from_register(const typename simd::vector_t value) noexcept { - return uint128_t( - static_cast(simd::template extract<0>(value)), - static_cast(simd::template extract<1>(value))); + return uint128_t(static_cast(simd::template extract<0>(value)), + static_cast(simd::template extract<1>(value))); } - [[nodiscard]] constexpr uint128_t operator+(const uint128_t& rhs) const noexcept + [[nodiscard]] constexpr uint128_t operator+(const uint128_t &rhs) const noexcept { const auto lowResult = add_with_carry(m_data[0], rhs.m_data[0]); const auto highResult = add_with_carry(m_data[1], rhs.m_data[1], lowResult.carry); return uint128_t(lowResult.value, highResult.value); } - [[nodiscard]] constexpr uint128_t operator-(const uint128_t& rhs) const noexcept + [[nodiscard]] constexpr uint128_t operator-(const uint128_t &rhs) const noexcept { const auto lowResult = subtract_with_borrow(m_data[0], rhs.m_data[0]); const auto highResult = subtract_with_borrow(m_data[1], rhs.m_data[1], lowResult.borrow); return uint128_t(lowResult.value, highResult.value); } - constexpr uint128_t& operator+=(const uint128_t& rhs) noexcept + constexpr uint128_t &operator+=(const uint128_t &rhs) noexcept { return *this = *this + rhs; } - constexpr uint128_t& operator-=(const uint128_t& rhs) noexcept + constexpr uint128_t &operator-=(const uint128_t &rhs) noexcept { return *this = *this - rhs; } - [[nodiscard]] constexpr bool operator==(const uint128_t& rhs) const noexcept = default; + [[nodiscard]] constexpr bool operator==(const uint128_t &rhs) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(const uint128_t& rhs) const noexcept + [[nodiscard]] constexpr std::strong_ordering operator<=>(const uint128_t &rhs) const noexcept { if (m_data[1] != rhs.m_data[1]) { @@ -127,19 +112,29 @@ class uint128_t final return m_data[0] <=> rhs.m_data[0]; } + /** + * @brief Compares this value with a built-in integral value. + * @tparam T Integral comparison type containing at most 64 value bits. + * + * @param rhs Scalar value to compare. + * @return `true` when both values represent the same nonnegative integer. + */ template - requires(std::numeric_limits::digits <= 64) + requires(std::numeric_limits::digits <= 64) [[nodiscard]] constexpr bool operator==(const T rhs) const noexcept { if constexpr (std::is_signed_v) { return rhs >= 0 && m_data[1] == 0 && m_data[0] == static_cast(rhs); } - return m_data[1] == 0 && m_data[0] == static_cast(rhs); + else + { + return m_data[1] == 0 && m_data[0] == static_cast(rhs); + } } template - requires(std::numeric_limits::digits <= 64) + requires(std::numeric_limits::digits <= 64) [[nodiscard]] constexpr std::strong_ordering operator<=>(const T rhs) const noexcept { if constexpr (std::is_signed_v) @@ -152,7 +147,7 @@ class uint128_t final return *this <=> uint128_t(static_cast(rhs)); } - [[nodiscard]] constexpr uint128_t operator&(const uint128_t& rhs) const noexcept + [[nodiscard]] constexpr uint128_t operator&(const uint128_t &rhs) const noexcept { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_SSE42 if (!std::is_constant_evaluated()) @@ -163,7 +158,7 @@ class uint128_t final return uint128_t(m_data[0] & rhs.m_data[0], m_data[1] & rhs.m_data[1]); } - [[nodiscard]] constexpr uint128_t operator|(const uint128_t& rhs) const noexcept + [[nodiscard]] constexpr uint128_t operator|(const uint128_t &rhs) const noexcept { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_SSE42 if (!std::is_constant_evaluated()) @@ -174,7 +169,7 @@ class uint128_t final return uint128_t(m_data[0] | rhs.m_data[0], m_data[1] | rhs.m_data[1]); } - [[nodiscard]] constexpr uint128_t operator^(const uint128_t& rhs) const noexcept + [[nodiscard]] constexpr uint128_t operator^(const uint128_t &rhs) const noexcept { #if SIMDLIB_TARGET_X86 && SIMDLIB_HAS_SSE42 if (!std::is_constant_evaluated()) @@ -196,24 +191,23 @@ class uint128_t final return uint128_t(~m_data[0], ~m_data[1]); } - constexpr uint128_t& operator&=(const uint128_t& rhs) noexcept + constexpr uint128_t &operator&=(const uint128_t &rhs) noexcept { return *this = *this & rhs; } - constexpr uint128_t& operator|=(const uint128_t& rhs) noexcept + constexpr uint128_t &operator|=(const uint128_t &rhs) noexcept { return *this = *this | rhs; } - constexpr uint128_t& operator^=(const uint128_t& rhs) noexcept + constexpr uint128_t &operator^=(const uint128_t &rhs) noexcept { return *this = *this ^ rhs; } /** @brief Extracts a contiguous bit range and shifts it to bit zero. */ - [[deprecated("Prefer SimdLib::Bmi::bextr")]] - [[nodiscard]] constexpr uint128_t extract(const std::uint8_t len, const std::uint8_t start) const noexcept + [[deprecated("Prefer SimdLib::Bmi::bextr")]] [[nodiscard]] constexpr uint128_t extract(const std::uint8_t len, const std::uint8_t start) const noexcept { if (len == 0 || start >= 128) { @@ -224,23 +218,21 @@ class uint128_t final } template - requires(len <= 64) - [[deprecated("Prefer SimdLib::Bmi::bextr")]] - [[nodiscard]] constexpr std::uint64_t extract(const std::uint8_t start) const noexcept + requires(len <= 64) + [[deprecated("Prefer SimdLib::Bmi::bextr")]] [[nodiscard]] constexpr std::uint64_t extract(const std::uint8_t start) const noexcept { return static_cast(extract(static_cast(len), start)); } template - requires(start <= 128 && len <= 128) - [[deprecated("Prefer SimdLib::Bmi::bextr")]] - [[nodiscard]] constexpr uint128_t extract() const noexcept + requires(start <= 128 && len <= 128) + [[deprecated("Prefer SimdLib::Bmi::bextr")]] [[nodiscard]] constexpr uint128_t extract() const noexcept { return extract(static_cast(len), static_cast(start)); } /** @brief Computes the absolute difference between two unsigned 128-bit values. */ - [[nodiscard]] constexpr uint128_t abs_diff(const uint128_t& other) const noexcept + [[nodiscard]] constexpr uint128_t abs_diff(const uint128_t &other) const noexcept { return *this > other ? *this - other : other - *this; } @@ -264,8 +256,7 @@ class uint128_t final return uint128_t((std::uint64_t{1} << bitCount) - 1, 0); } - template - [[nodiscard]] static constexpr uint128_t create_mask(const int offset) noexcept + template [[nodiscard]] static constexpr uint128_t create_mask(const int offset) noexcept { static_assert(width >= 0 && width <= 128); if constexpr (width == 0) @@ -284,8 +275,7 @@ class uint128_t final } /** @brief Whole-value left shift. Negative counts are treated as zero; counts of 128 or more produce zero. */ - template - [[nodiscard]] constexpr uint128_t operator<<(const T count) const noexcept + template [[nodiscard]] constexpr uint128_t operator<<(const T count) const noexcept { uint128_t result(*this); result.shift_left(normalize_shift(count)); @@ -293,23 +283,20 @@ class uint128_t final } /** @brief Whole-value right shift. Negative counts are treated as zero; counts of 128 or more produce zero. */ - template - [[nodiscard]] constexpr uint128_t operator>>(const T count) const noexcept + template [[nodiscard]] constexpr uint128_t operator>>(const T count) const noexcept { uint128_t result(*this); result.shift_right(normalize_shift(count)); return result; } - template - constexpr uint128_t& operator<<=(const T count) noexcept + template constexpr uint128_t &operator<<=(const T count) noexcept { shift_left(normalize_shift(count)); return *this; } - template - constexpr uint128_t& operator>>=(const T count) noexcept + template constexpr uint128_t &operator>>=(const T count) noexcept { shift_right(normalize_shift(count)); return *this; @@ -320,12 +307,12 @@ class uint128_t final return uint128_t{} - *this; } - constexpr uint128_t& operator++() noexcept + constexpr uint128_t &operator++() noexcept { return *this += uint128_t{1}; } - constexpr uint128_t& operator--() noexcept + constexpr uint128_t &operator--() noexcept { return *this -= uint128_t{1}; } @@ -344,10 +331,22 @@ class uint128_t final return previous; } - [[nodiscard]] constexpr std::uint64_t& low() noexcept { return m_data[0]; } - [[nodiscard]] constexpr std::uint64_t& high() noexcept { return m_data[1]; } - [[nodiscard]] constexpr std::uint64_t low() const noexcept { return m_data[0]; } - [[nodiscard]] constexpr std::uint64_t high() const noexcept { return m_data[1]; } + [[nodiscard]] constexpr std::uint64_t &low() noexcept + { + return m_data[0]; + } + [[nodiscard]] constexpr std::uint64_t &high() noexcept + { + return m_data[1]; + } + [[nodiscard]] constexpr std::uint64_t low() const noexcept + { + return m_data[0]; + } + [[nodiscard]] constexpr std::uint64_t high() const noexcept + { + return m_data[1]; + } /** @brief Returns the backing word at index zero (low) or one (high). */ [[nodiscard]] constexpr std::uint64_t getBlock(const int index) const noexcept @@ -355,8 +354,7 @@ class uint128_t final return m_data[static_cast(index)]; } - template - [[nodiscard]] constexpr explicit operator T() const noexcept + template [[nodiscard]] constexpr explicit operator T() const noexcept { return static_cast(m_data[0]); } @@ -379,10 +377,8 @@ class uint128_t final bool borrow; }; - [[nodiscard]] static constexpr add_carry_result portable_add_with_carry( - const std::uint64_t lhs, - const std::uint64_t rhs, - const bool carryIn = false) noexcept + [[nodiscard]] static constexpr add_carry_result portable_add_with_carry(const std::uint64_t lhs, const std::uint64_t rhs, + const bool carryIn = false) noexcept { const std::uint64_t partial = lhs + rhs; const bool firstCarry = partial < lhs; @@ -390,10 +386,8 @@ class uint128_t final return {result, firstCarry || result < partial}; } - [[nodiscard]] static constexpr subtract_borrow_result portable_subtract_with_borrow( - const std::uint64_t lhs, - const std::uint64_t rhs, - const bool borrowIn = false) noexcept + [[nodiscard]] static constexpr subtract_borrow_result portable_subtract_with_borrow(const std::uint64_t lhs, const std::uint64_t rhs, + const bool borrowIn = false) noexcept { const std::uint64_t partial = lhs - rhs; const bool firstBorrow = lhs < rhs; @@ -401,17 +395,13 @@ class uint128_t final return {result, firstBorrow || partial < static_cast(borrowIn)}; } - [[nodiscard]] static constexpr add_carry_result add_with_carry( - const std::uint64_t lhs, - const std::uint64_t rhs, - const bool carryIn = false) noexcept + [[nodiscard]] static constexpr add_carry_result add_with_carry(const std::uint64_t lhs, const std::uint64_t rhs, const bool carryIn = false) noexcept { #if SIMDLIB_USE_COMPILER_CARRY_INTRINSICS && SIMDLIB_COMPILER_MSVC && defined(_M_X64) if (!std::is_constant_evaluated()) { std::uint64_t result = 0; - const unsigned char carry = _addcarry_u64( - static_cast(carryIn), lhs, rhs, &result); + const unsigned char carry = _addcarry_u64(static_cast(carryIn), lhs, rhs, &result); return {result, carry != 0}; } #elif SIMDLIB_USE_COMPILER_CARRY_INTRINSICS && (SIMDLIB_COMPILER_CLANG || SIMDLIB_COMPILER_GCC) @@ -420,25 +410,21 @@ class uint128_t final std::uint64_t partial = 0; std::uint64_t result = 0; const bool firstCarry = __builtin_add_overflow(lhs, rhs, &partial); - const bool secondCarry = __builtin_add_overflow( - partial, static_cast(carryIn), &result); + const bool secondCarry = __builtin_add_overflow(partial, static_cast(carryIn), &result); return {result, firstCarry || secondCarry}; } #endif return portable_add_with_carry(lhs, rhs, carryIn); } - [[nodiscard]] static constexpr subtract_borrow_result subtract_with_borrow( - const std::uint64_t lhs, - const std::uint64_t rhs, - const bool borrowIn = false) noexcept + [[nodiscard]] static constexpr subtract_borrow_result subtract_with_borrow(const std::uint64_t lhs, const std::uint64_t rhs, + const bool borrowIn = false) noexcept { #if SIMDLIB_USE_COMPILER_CARRY_INTRINSICS && SIMDLIB_COMPILER_MSVC && defined(_M_X64) if (!std::is_constant_evaluated()) { std::uint64_t result = 0; - const unsigned char borrow = _subborrow_u64( - static_cast(borrowIn), lhs, rhs, &result); + const unsigned char borrow = _subborrow_u64(static_cast(borrowIn), lhs, rhs, &result); return {result, borrow != 0}; } #elif SIMDLIB_USE_COMPILER_CARRY_INTRINSICS && (SIMDLIB_COMPILER_CLANG || SIMDLIB_COMPILER_GCC) @@ -447,16 +433,14 @@ class uint128_t final std::uint64_t partial = 0; std::uint64_t result = 0; const bool firstBorrow = __builtin_sub_overflow(lhs, rhs, &partial); - const bool secondBorrow = __builtin_sub_overflow( - partial, static_cast(borrowIn), &result); + const bool secondBorrow = __builtin_sub_overflow(partial, static_cast(borrowIn), &result); return {result, firstBorrow || secondBorrow}; } #endif return portable_subtract_with_borrow(lhs, rhs, borrowIn); } - template - [[nodiscard]] static constexpr int normalize_shift(const T count) noexcept + template [[nodiscard]] static constexpr int normalize_shift(const T count) noexcept { if constexpr (std::same_as, bool>) { @@ -500,9 +484,7 @@ class uint128_t final m_data = {0, m_data[0] << (count - 64)}; return; } - m_data = { - m_data[0] << count, - (m_data[1] << count) | (m_data[0] >> (64 - count))}; + m_data = {m_data[0] << count, (m_data[1] << count) | (m_data[0] >> (64 - count))}; } constexpr void shift_right(const int count) noexcept @@ -528,13 +510,11 @@ class uint128_t final m_data = {m_data[1] >> (count - 64), 0}; return; } - m_data = { - (m_data[0] >> count) | (m_data[1] << (64 - count)), - m_data[1] >> count}; + m_data = {(m_data[0] >> count) | (m_data[1] << (64 - count)), m_data[1] >> count}; } template - requires(simd_available) + requires(simd_available) [[nodiscard]] static uint128_t store_register(const typename simd::vector_t value) noexcept { uint128_t result; @@ -543,8 +523,8 @@ class uint128_t final } template - requires(simd_available) - [[nodiscard]] uint128_t simd_bitwise_binary(const uint128_t& rhs) const noexcept + requires(simd_available) + [[nodiscard]] uint128_t simd_bitwise_binary(const uint128_t &rhs) const noexcept { const auto lhsRegister = to_register(); const auto rhsRegister = rhs.template to_register(); @@ -563,25 +543,27 @@ class uint128_t final } template - requires(simd_available) + requires(simd_available) [[nodiscard]] uint128_t simd_bitwise_not() const noexcept { const auto value = simd::construct(m_data); return store_register(simd::bitwise_not(value)); } + /** @brief Shifts the complete value left through the SIMD runtime-count slow path. */ template - requires(simd_available) + requires(simd_available) [[nodiscard]] uint128_t simd_shift_left(const int count) const noexcept { - return store_register(simd::bit_shift_left(to_register(), count)); + return store_register(simd::shift_bits_left_slow(to_register(), count)); } + /** @brief Shifts the complete value right through the SIMD runtime-count slow path. */ template - requires(simd_available) + requires(simd_available) [[nodiscard]] uint128_t simd_shift_right(const int count) const noexcept { - return store_register(simd::bit_shift_right(to_register(), count)); + return store_register(simd::shift_bits_right_slow(to_register(), count)); } }; @@ -594,8 +576,8 @@ static_assert(std::is_trivially_copyable_v); namespace std { -template <> -class numeric_limits +/** @brief Supplies standard numeric limits for SimdLib's unsigned 128-bit integer. */ +template <> class numeric_limits : public numeric_limits { public: static constexpr bool is_specialized = true; @@ -613,8 +595,6 @@ class numeric_limits static constexpr bool has_infinity = false; static constexpr bool has_quiet_NaN = false; static constexpr bool has_signaling_NaN = false; - static constexpr float_denorm_style has_denorm = denorm_absent; - static constexpr bool has_denorm_loss = false; static constexpr bool is_iec559 = false; static constexpr bool is_bounded = true; static constexpr bool is_modulo = true; @@ -622,24 +602,47 @@ class numeric_limits static constexpr bool tinyness_before = false; static constexpr float_round_style round_style = round_toward_zero; - [[nodiscard]] static constexpr SimdLib::uint128_t min() noexcept { return {}; } - [[nodiscard]] static constexpr SimdLib::uint128_t lowest() noexcept { return {}; } + [[nodiscard]] static constexpr SimdLib::uint128_t min() noexcept + { + return {}; + } + [[nodiscard]] static constexpr SimdLib::uint128_t lowest() noexcept + { + return {}; + } [[nodiscard]] static constexpr SimdLib::uint128_t max() noexcept { return {numeric_limits::max(), numeric_limits::max()}; } - [[nodiscard]] static constexpr SimdLib::uint128_t epsilon() noexcept { return {}; } - [[nodiscard]] static constexpr SimdLib::uint128_t round_error() noexcept { return {}; } - [[nodiscard]] static constexpr SimdLib::uint128_t infinity() noexcept { return {}; } - [[nodiscard]] static constexpr SimdLib::uint128_t quiet_NaN() noexcept { return {}; } - [[nodiscard]] static constexpr SimdLib::uint128_t signaling_NaN() noexcept { return {}; } - [[nodiscard]] static constexpr SimdLib::uint128_t denorm_min() noexcept { return {}; } + [[nodiscard]] static constexpr SimdLib::uint128_t epsilon() noexcept + { + return {}; + } + [[nodiscard]] static constexpr SimdLib::uint128_t round_error() noexcept + { + return {}; + } + [[nodiscard]] static constexpr SimdLib::uint128_t infinity() noexcept + { + return {}; + } + [[nodiscard]] static constexpr SimdLib::uint128_t quiet_NaN() noexcept + { + return {}; + } + [[nodiscard]] static constexpr SimdLib::uint128_t signaling_NaN() noexcept + { + return {}; + } + [[nodiscard]] static constexpr SimdLib::uint128_t denorm_min() noexcept + { + return {}; + } }; -template <> -struct hash +template <> struct hash { - [[nodiscard]] constexpr std::size_t operator()(const SimdLib::uint128_t& value) const noexcept + [[nodiscard]] constexpr std::size_t operator()(const SimdLib::uint128_t &value) const noexcept { return static_cast(value.low() ^ value.high()); } @@ -649,10 +652,7 @@ struct hash namespace SimdLib::Bmi { /** @brief Extracts a contiguous bit range from a 128-bit value and shifts it to bit zero. */ -[[nodiscard]] constexpr uint128_t bextr( - const uint128_t value, - const std::uint8_t len, - const std::uint8_t start) noexcept +[[nodiscard]] constexpr uint128_t bextr(const uint128_t value, const std::uint8_t len, const std::uint8_t start) noexcept { if (len == 0 || start >= 128) { @@ -700,9 +700,7 @@ namespace SimdLib /** @brief Returns the number of consecutive one bits from the least-significant side. */ [[nodiscard]] constexpr int countr_one(const uint128_t value) noexcept { - return value.low() == std::numeric_limits::max() - ? 64 + std::countr_one(value.high()) - : std::countr_one(value.low()); + return value.low() == std::numeric_limits::max() ? 64 + std::countr_one(value.high()) : std::countr_one(value.low()); } /** @brief Returns the number of consecutive zero bits from the most-significant side. */ @@ -714,9 +712,7 @@ namespace SimdLib /** @brief Returns the number of consecutive one bits from the most-significant side. */ [[nodiscard]] constexpr int countl_one(const uint128_t value) noexcept { - return value.high() == std::numeric_limits::max() - ? 64 + std::countl_one(value.low()) - : std::countl_one(value.high()); + return value.high() == std::numeric_limits::max() ? 64 + std::countl_one(value.low()) : std::countl_one(value.high()); } /** @brief Returns the number of bits required to represent the value. */ diff --git a/tests/Api128.tests.cpp b/tests/Api128.tests.cpp index d587676..1acb72c 100644 --- a/tests/Api128.tests.cpp +++ b/tests/Api128.tests.cpp @@ -4,6 +4,7 @@ #include #include #include +#include using namespace SimdLib::Tests; @@ -17,34 +18,44 @@ TEST_CASE("128-bit constexpr contracts match volatile runtime dispatch", "[simdl } TEST_CASE("128-bit Api specialization matrix", "[simdlib][sse42][availability]") { - require_supported_addition_matrix<128>(); + require_supported_addition_matrix<128>(); +} + +TEST_CASE("128-bit runtime extraction covers every lane and element type", "[simdlib][sse42][extract][runtime]") +{ + require_runtime_extraction_matrix_128(); +} + +TEST_CASE("128-bit runtime insertion covers every lane and element type", "[simdlib][sse42][insert][runtime]") +{ + require_runtime_insertion_matrix_128(); } TEST_CASE("128-bit aligned and unaligned transfer matrix", "[simdlib][sse42][transfer]") { - require_supported_transfer_matrix<128>(); + require_supported_transfer_matrix<128>(); } TEST_CASE("128-bit partial loads accept unaligned prefixes and zero inactive lanes", "[simdlib][sse42][transfer][partial]") { - require_supported_partial_transfer_matrix<128>(); + require_supported_partial_transfer_matrix<128>(); } TEST_CASE("128-bit movemask contracts are byte and element granular", "[simdlib][sse42][movemask]") { - require_supported_movemask_matrix<128>(); + require_supported_movemask_matrix<128>(); } TEST_CASE("128-bit transform_pack preserves packed lane order and exact tails", "[simdlib][sse42][transform-pack]") { require_transform_pack_full_native_word_contract<128>(); - require_transform_pack_mask_contract<128, std::uint8_t, 24>(); - require_transform_pack_mask_contract<128, std::uint8_t, 80>(); - require_transform_pack_mask_contract<128, std::uint64_t, 8>(); - require_transform_pack_width_contract<128, std::uint32_t, 7, 3>(); - require_transform_pack_width_contract<128, std::uint32_t, 19, 9>(); - require_transform_pack_width_contract<128, std::uint64_t, 5, 32>(); - require_transform_pack_type_matrix<128>(); + require_transform_pack_mask_contract<128, std::uint8_t, 24>(); + require_transform_pack_mask_contract<128, std::uint8_t, 80>(); + require_transform_pack_mask_contract<128, std::uint64_t, 8>(); + require_transform_pack_width_contract<128, std::uint32_t, 7, 3>(); + require_transform_pack_width_contract<128, std::uint32_t, 19, 9>(); + require_transform_pack_width_contract<128, std::uint64_t, 5, 32>(); + require_transform_pack_type_matrix<128>(); } TEST_CASE("128-bit public transform overloads preserve exact spans", "[simdlib][sse42][transform]") @@ -54,43 +65,66 @@ TEST_CASE("128-bit public transform overloads preserve exact spans", "[simdlib][ TEST_CASE("128-bit partial construction and float dot product use public Api entry points", "[simdlib][sse42][partial][dot]") { - using integers = SimdLib::Api<128, std::uint32_t>; - const std::array prefix{3, 5}; - REQUIRE(integers::to_array(integers::setr_partial(3U, 5U)) == std::array{3, 5, 0, 0}); - REQUIRE(integers::to_array(integers::template load_partial<2>(prefix)) == std::array{3, 5, 0, 0}); - - using floats = SimdLib::Api<128, float>; - const auto dot = floats::template dot_product<0xFF>(floats::set1(1.0F), floats::set1(2.0F)); - REQUIRE(floats::to_array(dot) == std::array{8.0F, 8.0F, 8.0F, 8.0F}); - const auto partialDot = floats::template dot_product<0x11>(floats::set1(1.0F), floats::set1(2.0F)); - REQUIRE(floats::to_array(partialDot) == std::array{2.0F, 0.0F, 0.0F, 0.0F}); + using integers = SimdLib::Api<128, std::uint32_t>; + const std::array prefix{3, 5}; + REQUIRE(integers::to_array(integers::setr_partial(3U, 5U)) == std::array{3, 5, 0, 0}); + REQUIRE(integers::to_array(integers::template load_partial<2>(prefix)) == std::array{3, 5, 0, 0}); + + using floats = SimdLib::Api<128, float>; + const auto dot = floats::template dot_product<0xFF>(floats::set1(1.0F), floats::set1(2.0F)); + REQUIRE(floats::to_array(dot) == std::array{8.0F, 8.0F, 8.0F, 8.0F}); + const auto partialDot = floats::template dot_product<0x11>(floats::set1(1.0F), floats::set1(2.0F)); + REQUIRE(floats::to_array(partialDot) == std::array{2.0F, 0.0F, 0.0F, 0.0F}); +} + +TEST_CASE("128-bit signed and unsigned 64-bit setr preserves forward lane order and exact bit patterns", "[simdlib][sse42][setr][int64][uint64]") +{ + using signed_words = SimdLib::Api<128, std::int64_t>; + volatile std::int64_t signed_low_source = std::numeric_limits::lowest(); + volatile std::int64_t signed_high_source = std::numeric_limits::max(); + const std::int64_t signed_low = signed_low_source; + const std::int64_t signed_high = signed_high_source; + REQUIRE(signed_words::to_array(signed_words::setr(signed_low, signed_high)) == std::array{signed_low, signed_high}); + + using unsigned_words = SimdLib::Api<128, std::uint64_t>; + volatile std::uint64_t unsigned_low_source = 0x8000'0000'0000'0001ULL; + volatile std::uint64_t unsigned_high_source = 0xFEDC'BA98'7654'3210ULL; + const std::uint64_t unsigned_low = unsigned_low_source; + const std::uint64_t unsigned_high = unsigned_high_source; + REQUIRE(unsigned_words::to_array(unsigned_words::setr(unsigned_low, unsigned_high)) == std::array{unsigned_low, unsigned_high}); } + TEST_CASE("128-bit arithmetic and int8 division match scalar results", "[simdlib][sse42][arithmetic]") { - using integers = SimdLib::Api<128, std::int32_t>; - const auto lhs = integers::setr(4, 8, 12, 16); - const auto rhs = integers::setr(1, 2, 3, 4); - REQUIRE(integers::to_array(integers::subtract(lhs, rhs)) == std::array{3, 6, 9, 12}); - REQUIRE(integers::to_array(integers::multiply(lhs, rhs)) == std::array{4, 16, 36, 64}); - - using bytes = SimdLib::Api<128, std::int8_t>; - const auto quotients = bytes::divide(bytes::set1(24), bytes::set1(6)); - REQUIRE(bytes::to_array(quotients) == std::array{4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}); + using integers = SimdLib::Api<128, std::int32_t>; + const auto lhs = integers::setr(4, 8, 12, 16); + const auto rhs = integers::setr(1, 2, 3, 4); + REQUIRE(integers::to_array(integers::subtract(lhs, rhs)) == std::array{3, 6, 9, 12}); + REQUIRE(integers::to_array(integers::multiply(lhs, rhs)) == std::array{4, 16, 36, 64}); + + using bytes = SimdLib::Api<128, std::int8_t>; + const auto quotients = bytes::divide(bytes::set1(24), bytes::set1(6)); + REQUIRE(bytes::to_array(quotients) == std::array{4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}); +} + +TEST_CASE("128-bit integer remainder matches scalar semantics for every lane and width", "[simdlib][sse42][integer][remainder]") +{ + require_integer_remainder_matrix<128>(); } TEST_CASE("128-bit comparisons and saturation match scalar semantics", "[simdlib][sse42][comparison][saturation]") { require_supported_comparison_matrix<128>(); - using words = SimdLib::Api<128, std::int32_t>; - const auto lhs = words::setr(1, 2, 3, 4); - const auto rhs = words::setr(1, 0, 3, 9); - REQUIRE(words::cmp_eq_mask(lhs, rhs) == 0x00000F0Fu); + using words = SimdLib::Api<128, std::int32_t>; + const auto lhs = words::setr(1, 2, 3, 4); + const auto rhs = words::setr(1, 0, 3, 9); + REQUIRE(words::cmp_eq_mask(lhs, rhs) == 0x00000F0Fu); - using bytes = SimdLib::Api<128, std::uint8_t>; - REQUIRE(bytes::to_array(bytes::add_saturated(bytes::set1(250), bytes::set1(10))) == - std::array{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}); - REQUIRE(bytes::to_array(bytes::subtract_saturated(bytes::set1(5), bytes::set1(10))) == std::array{}); + using bytes = SimdLib::Api<128, std::uint8_t>; + REQUIRE(bytes::to_array(bytes::add_saturated(bytes::set1(250), bytes::set1(10))) == + std::array{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}); + REQUIRE(bytes::to_array(bytes::subtract_saturated(bytes::set1(5), bytes::set1(10))) == std::array{}); } TEST_CASE("128-bit integer extrema and position matrix uses public Api entry points", "[simdlib][sse42][extrema][position]") @@ -124,12 +158,10 @@ TEST_CASE("128-bit signed integer and float conversion gates preserve lane value using integers = SimdLib::Api<128, std::int32_t>; using floats = SimdLib::Api<128, float>; const auto integer_values = integers::setr(-7, 0, 42, 1'000'000); - REQUIRE(floats::to_array(integers::convert_to_float(integer_values)) == - std::array{-7.0f, 0.0f, 42.0f, 1'000'000.0f}); + REQUIRE(floats::to_array(integers::convert_to_float(integer_values)) == std::array{-7.0f, 0.0f, 42.0f, 1'000'000.0f}); const auto float_values = floats::setr(-7.0f, 0.0f, 42.0f, 1'000'000.0f); - REQUIRE(integers::to_array(floats::convert_to_int(float_values)) == - std::array{-7, 0, 42, 1'000'000}); + REQUIRE(integers::to_array(floats::convert_to_int(float_values)) == std::array{-7, 0, 42, 1'000'000}); } TEST_CASE("128-bit public 64-bit arithmetic contract", "[simdlib][sse42][int64][arithmetic]") @@ -139,55 +171,68 @@ TEST_CASE("128-bit public 64-bit arithmetic contract", "[simdlib][sse42][int64][ TEST_CASE("128-bit widening and horizontal arithmetic match scalar references", "[simdlib][sse42][widen][horizontal]") { - using source = SimdLib::Api<128, std::int8_t>; - using target = SimdLib::Api<128, std::int16_t>; - const auto widened = source::template widen(source::setr(-4, -3, -2, -1, 0, 1, 2, 3, 90, 91, 92, 93, 94, 95, 96, 97)); - REQUIRE(target::to_array(widened) == std::array{-4, -3, -2, -1, 0, 1, 2, 3}); - - using lanes = SimdLib::Api<128, std::int32_t>; - const auto horizontal = lanes::add_horizontal(lanes::setr(1, 2, 3, 4), lanes::setr(5, 6, 7, 8)); - REQUIRE(lanes::to_array(horizontal) == std::array{3, 7, 11, 15}); + using source = SimdLib::Api<128, std::int8_t>; + using target = SimdLib::Api<128, std::int16_t>; + const auto widened = source::template widen(source::setr(-4, -3, -2, -1, 0, 1, 2, 3, 90, 91, 92, 93, 94, 95, 96, 97)); + REQUIRE(target::to_array(widened) == std::array{-4, -3, -2, -1, 0, 1, 2, 3}); + + using lanes = SimdLib::Api<128, std::int32_t>; + const auto horizontal = lanes::add_horizontal(lanes::setr(1, 2, 3, 4), lanes::setr(5, 6, 7, 8)); + REQUIRE(lanes::to_array(horizontal) == std::array{3, 7, 11, 15}); } TEST_CASE("128-bit lane and whole-register shifts are distinct", "[simdlib][sse42][shift]") { - using simd = SimdLib::Api<128, std::uint64_t>; - const auto input = simd::setr(0x0123456789ABCDEFULL, 0xFEDCBA9876543210ULL); - REQUIRE(simd::to_array(simd::shift_left(input, 4)) == - std::array{0x123456789ABCDEF0ULL, 0xEDCBA98765432100ULL}); - - const std::array counts{0, 1, 63, 64, 65, 127, 128, 129, 255}; - const auto source = simd::to_array(input); - for (const int count : counts) - { - std::array left{}; - std::array right{}; - if (count == 0) - { - left = source; - right = source; - } - else if (count < 64) - { - left = {source[0] << count, (source[1] << count) | (source[0] >> (64 - count))}; - right = {(source[0] >> count) | (source[1] << (64 - count)), source[1] >> count}; - } - else if (count == 64) - { - left = {0, source[0]}; - right = {source[1], 0}; - } - else if (count < 128) - { - left = {0, source[0] << (count - 64)}; - right = {source[1] >> (count - 64), 0}; - } - REQUIRE(simd::to_array(simd::bit_shift_left(input, count)) == left); - REQUIRE(simd::to_array(simd::bit_shift_right(input, count)) == right); - } - - REQUIRE(simd::to_array(simd::template bit_shift_left<64>(input)) == std::array{0, source[0]}); - REQUIRE(simd::to_array(simd::template bit_shift_right<128>(input)) == std::array{}); + using simd = SimdLib::Api<128, std::uint64_t>; + const auto input = simd::setr(0x0123456789ABCDEFULL, 0xFEDCBA9876543210ULL); + REQUIRE(simd::to_array(simd::shift_left(input, 4)) == std::array{0x123456789ABCDEF0ULL, 0xEDCBA98765432100ULL}); + + constexpr std::array counts{std::numeric_limits::lowest(), -1, 0, 1, 63, 64, 65, 127, 128, 129, 255, std::numeric_limits::max()}; + const auto source = simd::to_array(input); + for (const int count : counts) + { + std::array left{}; + std::array right{}; + if (count <= 0) + { + left = source; + right = source; + } + else if (count < 64) + { + left = {source[0] << count, (source[1] << count) | (source[0] >> (64 - count))}; + right = {(source[0] >> count) | (source[1] << (64 - count)), source[1] >> count}; + } + else if (count == 64) + { + left = {0, source[0]}; + right = {source[1], 0}; + } + else if (count < 128) + { + left = {0, source[0] << (count - 64)}; + right = {source[1] >> (count - 64), 0}; + } + REQUIRE(simd::to_array(simd::shift_bits_left_slow(input, count)) == left); + REQUIRE(simd::to_array(simd::shift_bits_right_slow(input, count)) == right); + } + + REQUIRE(simd::to_array(simd::template shift_bits_left<0>(input)) == source); + REQUIRE(simd::to_array(simd::template shift_bits_left<1>(input)) == std::array{source[0] << 1, (source[1] << 1) | (source[0] >> 63)}); + REQUIRE(simd::to_array(simd::template shift_bits_left<63>(input)) == std::array{source[0] << 63, (source[1] << 63) | (source[0] >> 1)}); + REQUIRE(simd::to_array(simd::template shift_bits_left<64>(input)) == std::array{0, source[0]}); + REQUIRE(simd::to_array(simd::template shift_bits_left<65>(input)) == std::array{0, source[0] << 1}); + REQUIRE(simd::to_array(simd::template shift_bits_left<127>(input)) == std::array{0, source[0] << 63}); + REQUIRE(simd::to_array(simd::template shift_bits_left<128>(input)) == std::array{}); + REQUIRE(simd::to_array(simd::template shift_bits_left<129>(input)) == std::array{}); + REQUIRE(simd::to_array(simd::template shift_bits_right<0>(input)) == source); + REQUIRE(simd::to_array(simd::template shift_bits_right<1>(input)) == std::array{(source[0] >> 1) | (source[1] << 63), source[1] >> 1}); + REQUIRE(simd::to_array(simd::template shift_bits_right<63>(input)) == std::array{(source[0] >> 63) | (source[1] << 1), source[1] >> 63}); + REQUIRE(simd::to_array(simd::template shift_bits_right<64>(input)) == std::array{source[1], 0}); + REQUIRE(simd::to_array(simd::template shift_bits_right<65>(input)) == std::array{source[1] >> 1, 0}); + REQUIRE(simd::to_array(simd::template shift_bits_right<127>(input)) == std::array{source[1] >> 63, 0}); + REQUIRE(simd::to_array(simd::template shift_bits_right<128>(input)) == std::array{}); + REQUIRE(simd::to_array(simd::template shift_bits_right<129>(input)) == std::array{}); } TEST_CASE("128-bit public byte operations cover lane shifts and byte-shift boundaries", "[simdlib][sse42][byte][shift]") @@ -197,18 +242,26 @@ TEST_CASE("128-bit public byte operations cover lane shifts and byte-shift bound for (std::size_t index = 0; index < source.size(); ++index) source[index] = static_cast(index + 1); const auto input = bytes::construct(source); - REQUIRE(bytes::to_array(bytes::set1(0x81)) == std::array{0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81}); - REQUIRE(bytes::to_array(bytes::multiply(bytes::set1(0x81), bytes::set1(2))) == std::array{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}); + REQUIRE(bytes::to_array(bytes::set1(0x81)) == + std::array{0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81}); + REQUIRE(bytes::to_array(bytes::multiply(bytes::set1(0x81), bytes::set1(2))) == + std::array{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}); REQUIRE(bytes::to_array(bytes::shift_left(bytes::set1(0x81), 1))[0] == 0x02); REQUIRE(bytes::to_array(bytes::shift_right(bytes::set1(0x81), 1))[0] == 0x40); using signed_bytes = SimdLib::Api<128, std::int8_t>; REQUIRE(signed_bytes::to_array(signed_bytes::shift_right_arithmetic(signed_bytes::set1(-126), 1))[0] == -63); - for (const int count : std::array{-1, 0, 1, 15, 16, 17}) + constexpr std::array counts{std::numeric_limits::lowest(), -17, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + std::numeric_limits::max()}; + for (const int count : counts) { std::array left{}; std::array right{}; - if (count <= 0) { left = source; right = source; } + if (count <= 0) + { + left = source; + right = source; + } else if (count < static_cast(bytes::byte_count)) { for (std::size_t index = static_cast(count); index < source.size(); ++index) @@ -216,23 +269,23 @@ TEST_CASE("128-bit public byte operations cover lane shifts and byte-shift bound for (std::size_t index = 0; index + static_cast(count) < source.size(); ++index) right[index] = source[index + static_cast(count)]; } - REQUIRE(bytes::to_array(bytes::byte_shift_left(input, count)) == left); - REQUIRE(bytes::to_array(bytes::byte_shift_right(input, count)) == right); + REQUIRE(bytes::to_array(bytes::shift_bytes_left_slow(input, count)) == left); + REQUIRE(bytes::to_array(bytes::shift_bytes_right_slow(input, count)) == right); } } TEST_CASE("128-bit shuffle, blend, and position helpers match scalar references", "[simdlib][sse42][shuffle][blend][position]") { - using words = SimdLib::Api<128, std::int32_t>; - const auto lhs = words::setr(10, 20, 30, 40); - const auto rhs = words::setr(1, 2, 3, 4); - REQUIRE(words::to_array(words::shuffle_32(lhs, 0b00'01'10'11)) == std::array{40, 30, 20, 10}); - REQUIRE(words::to_array(words::blend(lhs, rhs, 0b0101)) == std::array{1, 20, 3, 40}); - - using positions = SimdLib::Api<128, std::uint16_t>; - const auto values = positions::setr(8, 4, 7, 1, 9, 2, 6, 3); - REQUIRE(positions::min_position(values) == 3); - REQUIRE(positions::max_position(values) == 4); + using words = SimdLib::Api<128, std::int32_t>; + const auto lhs = words::setr(10, 20, 30, 40); + const auto rhs = words::setr(1, 2, 3, 4); + REQUIRE(words::to_array(words::shuffle_32_slow(lhs, 0b00'01'10'11)) == std::array{40, 30, 20, 10}); + REQUIRE(words::to_array(words::blend_slow(lhs, rhs, 0b0101)) == std::array{1, 20, 3, 40}); + + using positions = SimdLib::Api<128, std::uint16_t>; + const auto values = positions::setr(8, 4, 7, 1, 9, 2, 6, 3); + REQUIRE(positions::min_position(values) == 3); + REQUIRE(positions::max_position(values) == 4); } TEST_CASE("128-bit Api documentation examples produce their documented results", "[simdlib][sse42][documentation]") @@ -254,22 +307,22 @@ TEST_CASE("128-bit Api documentation examples produce their documented results", std::array{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}); require_documented_register(ApiT::add_subtract(ApiT::set1(10.0F), ApiT::setr(1.0F, 2.0F, 3.0F, 4.0F)), std::array{9.0F, 12.0F, 7.0F, 14.0F}); require_documented_register(U8::avg(U8::set1(2), U8::set1(6)), std::array{4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}); - require_documented_register(U32::bit_shift_left(U32::set1(3), 1), std::array{6U, 6U, 6U, 6U}); - require_documented_register(U32::bit_shift_right(U32::set1(8), 1), std::array{4U, 4U, 4U, 4U}); + require_documented_register(U32::shift_bits_left_slow(U32::set1(3), 1), std::array{6U, 6U, 6U, 6U}); + require_documented_register(U32::shift_bits_right_slow(U32::set1(8), 1), std::array{4U, 4U, 4U, 4U}); require_documented_register(U32::bitwise_and(U32::set1(12), U32::set1(10)), std::array{8U, 8U, 8U, 8U}); require_documented_register(U32::bitwise_andnot(U32::set1(12), U32::set1(10)), std::array{2U, 2U, 2U, 2U}); require_documented_register(U32::bitwise_not(U32::setzero()), std::array{0xFFFFFFFFU, 0xFFFFFFFFU, 0xFFFFFFFFU, 0xFFFFFFFFU}); require_documented_register(U32::bitwise_or(U32::set1(12), U32::set1(10)), std::array{14U, 14U, 14U, 14U}); require_documented_register(U32::bitwise_xor(U32::set1(12), U32::set1(10)), std::array{6U, 6U, 6U, 6U}); - require_documented_register(I32::blend(I32::setr(10, 20, 30, 40), I32::setr(1, 2, 3, 4), 0b0101), std::array{1, 20, 3, 40}); - require_documented_register(U8::byte_shift_left(U8::set1(7), 1), std::array{0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7}); - require_documented_register(U8::byte_shift_right(U8::set1(7), 1), std::array{7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0}); - REQUIRE(ApiT::cmp_eq(ApiT::set1(2.0F), ApiT::set1(2.0F)) == 0xFFFFU); + require_documented_register(I32::blend_slow(I32::setr(10, 20, 30, 40), I32::setr(1, 2, 3, 4), 0b0101), std::array{1, 20, 3, 40}); + require_documented_register(U8::shift_bytes_left_slow(U8::set1(7), 1), std::array{0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7}); + require_documented_register(U8::shift_bytes_right_slow(U8::set1(7), 1), std::array{7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0}); + REQUIRE(ApiT::cmp_eq_mask(ApiT::set1(2.0F), ApiT::set1(2.0F)) == 0xFFFFU); REQUIRE(ApiT::cmp_eq_mask(ApiT::set1(2.0F), ApiT::set1(2.0F)) == 0xFFFFU); - REQUIRE(ApiT::cmp_ge(ApiT::set1(2.0F), ApiT::set1(2.0F)) == 0xFFFFU); - REQUIRE(ApiT::cmp_gt(ApiT::set1(3.0F), ApiT::set1(2.0F)) == 0xFFFFU); - REQUIRE(ApiT::cmp_le(ApiT::set1(2.0F), ApiT::set1(2.0F)) == 0xFFFFU); - REQUIRE(ApiT::cmp_lt(ApiT::set1(2.0F), ApiT::set1(3.0F)) == 0xFFFFU); + REQUIRE(ApiT::cmp_ge_mask(ApiT::set1(2.0F), ApiT::set1(2.0F)) == 0xFFFFU); + REQUIRE(ApiT::cmp_gt_mask(ApiT::set1(3.0F), ApiT::set1(2.0F)) == 0xFFFFU); + REQUIRE(ApiT::cmp_le_mask(ApiT::set1(2.0F), ApiT::set1(2.0F)) == 0xFFFFU); + REQUIRE(ApiT::cmp_lt_mask(ApiT::set1(2.0F), ApiT::set1(3.0F)) == 0xFFFFU); require_documented_register(I16::compress(I16::set1(300), I16::set1(-300)), std::array{127, 127, 127, 127, 127, 127, 127, 127, -128, -128, -128, -128, -128, -128, -128, -128}); require_documented_register(ApiT::construct({1.0F, 2.0F, 0.0F, 0.0F}), std::array{1.0F, 2.0F, 0.0F, 0.0F}); @@ -284,7 +337,7 @@ TEST_CASE("128-bit Api documentation examples produce their documented results", std::array{32767, 32767, 32767, 32767, 20000, 20000, 20000, 20000}); require_documented_register(I16::hsubtract_saturated(I16::setr_partial(30000, -10000, -30000, 10000), I16::setr_partial(20000, -20000, 10000, -10000)), std::array{32767, -32768, 0, 0, 32767, 20000, 0, 0}); - require_documented_register(I32::insert(I32::setzero(), 9, 0), std::array{9, 0, 0, 0}); + require_documented_register(I32::insert_slow(I32::setzero(), 9, 0), std::array{9, 0, 0, 0}); alignas(ApiT::byte_count) const std::array input{1.0F, 2.0F}; require_documented_register(ApiT::load(input), std::array{1.0F, 2.0F, 0.0F, 0.0F}); require_documented_register(ApiT::load_aligned(input), std::array{1.0F, 2.0F, 0.0F, 0.0F}); @@ -293,9 +346,9 @@ TEST_CASE("128-bit Api documentation examples produce their documented results", require_documented_register(ApiT::load_unsafe(input), std::array{1.0F, 2.0F, 0.0F, 0.0F}); require_documented_register(ApiT::magnitude(ApiT::setr_partial(3.0F, 4.0F)), std::array{5.0F, 5.0F, 5.0F, 5.0F}); require_documented_register(ApiT::max(ApiT::setr(2.0F, 8.0F, 4.0F, 9.0F), ApiT::setr(5.0F, 3.0F, 7.0F, 1.0F)), std::array{5.0F, 8.0F, 7.0F, 9.0F}); - REQUIRE(U16::max_position(U16::insert(U16::set1(4), 9, 3)) == 3); + REQUIRE(U16::max_position(U16::insert_slow(U16::set1(4), 9, 3)) == 3); require_documented_register(ApiT::min(ApiT::setr(2.0F, 8.0F, 4.0F, 9.0F), ApiT::setr(5.0F, 3.0F, 7.0F, 1.0F)), std::array{2.0F, 3.0F, 4.0F, 1.0F}); - REQUIRE(U16::min_position(U16::insert(U16::set1(4), 1, 3)) == 3); + REQUIRE(U16::min_position(U16::insert_slow(U16::set1(4), 1, 3)) == 3); require_documented_register(U32::modulus(U32::set1(7), U32::set1(3)), std::array{1U, 1U, 1U, 1U}); REQUIRE(ApiT::movemask(ApiT::set1(-0.0F)) == 0x8888U); REQUIRE(ApiT::movemask_slim(ApiT::set1(-0.0F)) == 0xFU); @@ -322,9 +375,9 @@ TEST_CASE("128-bit Api documentation examples produce their documented results", require_documented_register(I32::shift_right(I32::set1(8), 1), std::array{4, 4, 4, 4}); require_documented_register(I32::shift_right_arithmetic(I32::set1(-8), 1), std::array{-4, -4, -4, -4}); require_documented_register(U8::shuffle(U8::set1(7), U8::set1(0x80)), std::array{}); - const auto high = I16::byte_shift_left(I16::setr_partial(1, 2, 3, 4), 8); - require_documented_register(I16::shuffle_hi(high, 0b0001'1011), std::array{0, 0, 0, 0, 4, 3, 2, 1}); - require_documented_register(I16::shuffle_lo(I16::setr_partial(1, 2, 3, 4), 0b0001'1011), std::array{4, 3, 2, 1, 0, 0, 0, 0}); + const auto high = I16::shift_bytes_left_slow(I16::setr_partial(1, 2, 3, 4), 8); + require_documented_register(I16::shuffle_hi_slow(high, 0b0001'1011), std::array{0, 0, 0, 0, 4, 3, 2, 1}); + require_documented_register(I16::shuffle_lo_slow(I16::setr_partial(1, 2, 3, 4), 0b0001'1011), std::array{4, 3, 2, 1, 0, 0, 0, 0}); require_documented_register(ApiT::sqrt(ApiT::setr_partial(4.0F, 9.0F)), std::array{2.0F, 3.0F, 0.0F, 0.0F}); alignas(ApiT::byte_count) std::array stored{}; ApiT::store(ApiT::setr_partial(1.0F, 2.0F), stored); @@ -344,7 +397,7 @@ TEST_CASE("128-bit Api documentation examples produce their documented results", ApiT::transform(std::array{1.0F, 2.0F, 3.0F}, transformed, [](auto lanes) { return ApiT::add(lanes, ApiT::set1(10.0F)); }); REQUIRE(transformed == std::array{11.0F, 12.0F, 13.0F}); std::array packed{}; - ApiT::transform_pack<1>(std::span{std::array{1.0F, -2.0F, 3.0F, -4.0F}}, packed, + ApiT::transform_pack<1>(std::span{std::array{1.0F, -2.0F, 3.0F, -4.0F}}, std::span{packed}, [](auto lanes) { return ApiT::movemask_slim(lanes); }); REQUIRE(packed[0] == 0b0000'1010); require_documented_register(ApiT::unpack_hi(ApiT::setr(1.0F, 2.0F, 3.0F, 4.0F), ApiT::setr(5.0F, 6.0F, 7.0F, 8.0F)), diff --git a/tests/Api256.tests.cpp b/tests/Api256.tests.cpp index 0fce359..6c74ccc 100644 --- a/tests/Api256.tests.cpp +++ b/tests/Api256.tests.cpp @@ -15,22 +15,32 @@ TEST_CASE("256-bit constexpr contracts match volatile runtime dispatch", "[simdl } TEST_CASE("256-bit Api specialization matrix", "[simdlib][avx2][availability]") { - require_supported_addition_matrix<256>(); + require_supported_addition_matrix<256>(); +} + +TEST_CASE("256-bit runtime extraction covers every lane and element type", "[simdlib][avx2][extract][runtime]") +{ + require_runtime_extraction_matrix_256(); +} + +TEST_CASE("256-bit runtime insertion covers every lane and element type", "[simdlib][avx2][insert][runtime]") +{ + require_runtime_insertion_matrix_256(); } TEST_CASE("256-bit aligned and unaligned transfer matrix", "[simdlib][avx2][transfer]") { - require_supported_transfer_matrix<256>(); + require_supported_transfer_matrix<256>(); } TEST_CASE("256-bit partial loads accept unaligned prefixes and zero inactive lanes", "[simdlib][avx2][transfer][partial]") { - require_supported_partial_transfer_matrix<256>(); + require_supported_partial_transfer_matrix<256>(); } TEST_CASE("256-bit movemask contracts are byte and element granular", "[simdlib][avx2][movemask]") { - require_supported_movemask_matrix<256>(); + require_supported_movemask_matrix<256>(); } TEST_CASE("256-bit transform_pack preserves packed lane order and exact tails", "[simdlib][avx2][transform-pack]") @@ -53,44 +63,49 @@ TEST_CASE("256-bit public transform overloads preserve exact spans", "[simdlib][ TEST_CASE("256-bit float and double dot products use public Api entry points", "[simdlib][avx2][dot]") { - using floats = SimdLib::Api<256, float>; - const auto floatDot = floats::template dot_product<0xFF>(floats::set1(1.0F), floats::set1(2.0F)); - REQUIRE(floats::to_array(floatDot) == std::array{8.0F, 8.0F, 8.0F, 8.0F, 8.0F, 8.0F, 8.0F, 8.0F}); + using floats = SimdLib::Api<256, float>; + const auto floatDot = floats::template dot_product<0xFF>(floats::set1(1.0F), floats::set1(2.0F)); + REQUIRE(floats::to_array(floatDot) == std::array{8.0F, 8.0F, 8.0F, 8.0F, 8.0F, 8.0F, 8.0F, 8.0F}); - using doubles = SimdLib::Api<256, double>; - const auto doubleDot = doubles::template dot_product<0xFF>(doubles::set1(1.0), doubles::set1(2.0)); - REQUIRE(doubles::to_array(doubleDot) == std::array{4.0, 4.0, 4.0, 4.0}); + using doubles = SimdLib::Api<256, double>; + const auto doubleDot = doubles::template dot_product<0xFF>(doubles::set1(1.0), doubles::set1(2.0)); + REQUIRE(doubles::to_array(doubleDot) == std::array{4.0, 4.0, 4.0, 4.0}); - const auto floatPartialDot = floats::template dot_product<0x11>(floats::set1(1.0F), floats::set1(2.0F)); - REQUIRE(floats::to_array(floatPartialDot) == std::array{2.0F, 0.0F, 0.0F, 0.0F, 2.0F, 0.0F, 0.0F, 0.0F}); - const auto doublePartialDot = doubles::template dot_product<0x11>(doubles::set1(1.0), doubles::set1(2.0)); - REQUIRE(doubles::to_array(doublePartialDot) == std::array{2.0, 0.0, 2.0, 0.0}); + const auto floatPartialDot = floats::template dot_product<0x11>(floats::set1(1.0F), floats::set1(2.0F)); + REQUIRE(floats::to_array(floatPartialDot) == std::array{2.0F, 0.0F, 0.0F, 0.0F, 2.0F, 0.0F, 0.0F, 0.0F}); + const auto doublePartialDot = doubles::template dot_product<0x11>(doubles::set1(1.0), doubles::set1(2.0)); + REQUIRE(doubles::to_array(doublePartialDot) == std::array{2.0, 0.0, 2.0, 0.0}); } TEST_CASE("256-bit byte function-pointer transforms use public Api entry points", "[simdlib][avx2][transform][byte]") { - using bytes = SimdLib::Api<256, std::uint8_t>; - std::array lhs{}; - std::array rhs{}; - std::array output{}; - for (std::size_t index = 0; index < lhs.size(); ++index) - { - lhs[index] = static_cast(index + 1); - rhs[index] = static_cast(0xA0U + index); - } - - bytes::transform(std::span(lhs), std::span(output), bytes::bitwise_not); - for (std::size_t index = 0; index < output.size(); ++index) - REQUIRE(output[index] == static_cast(~lhs[index])); - - bytes::transform(std::span(lhs), std::span(rhs), std::span(output), bytes::bitwise_xor); - for (std::size_t index = 0; index < output.size(); ++index) - REQUIRE(output[index] == static_cast(lhs[index] ^ rhs[index])); + using bytes = SimdLib::Api<256, std::uint8_t>; + std::array lhs{}; + std::array rhs{}; + std::array output{}; + for (std::size_t index = 0; index < lhs.size(); ++index) + { + lhs[index] = static_cast(index + 1); + rhs[index] = static_cast(0xA0U + index); + } + + bytes::transform(std::span(lhs), std::span(output), bytes::bitwise_not); + for (std::size_t index = 0; index < output.size(); ++index) + REQUIRE(output[index] == static_cast(~lhs[index])); + + bytes::transform(std::span(lhs), std::span(rhs), std::span(output), bytes::bitwise_xor); + for (std::size_t index = 0; index < output.size(); ++index) + REQUIRE(output[index] == static_cast(lhs[index] ^ rhs[index])); } TEST_CASE("256-bit integer extrema and position matrix uses public Api entry points", "[simdlib][avx2][extrema][position]") { require_integer_extrema_position_matrix<256>(); } +TEST_CASE("256-bit integer remainder matches scalar semantics for every lane and width", "[simdlib][avx2][integer][remainder]") +{ + require_integer_remainder_matrix<256>(); +} + TEST_CASE("256-bit public integer operation matrix", "[simdlib][avx2][integer][operations]") { require_integer_operation_matrix<256>(); @@ -120,13 +135,13 @@ TEST_CASE("256-bit arithmetic, horizontal operations, shuffles, and blends match { require_supported_comparison_matrix<256>(); - using simd = SimdLib::Api<256, std::int32_t>; - const auto lhs = simd::setr(1, 2, 3, 4, 5, 6, 7, 8); - const auto rhs = simd::setr(8, 7, 6, 5, 4, 3, 2, 1); - REQUIRE(simd::to_array(simd::multiply(lhs, rhs)) == std::array{8, 14, 18, 20, 20, 18, 14, 8}); - REQUIRE(simd::to_array(simd::add_horizontal(lhs, rhs)) == std::array{3, 7, 15, 11, 11, 15, 7, 3}); - REQUIRE(simd::to_array(simd::shuffle_32(lhs, 0b00'01'10'11)) == std::array{4, 3, 2, 1, 8, 7, 6, 5}); - REQUIRE(simd::to_array(simd::blend(lhs, rhs, 0b01010101)) == std::array{8, 2, 6, 4, 4, 6, 2, 8}); + using simd = SimdLib::Api<256, std::int32_t>; + const auto lhs = simd::setr(1, 2, 3, 4, 5, 6, 7, 8); + const auto rhs = simd::setr(8, 7, 6, 5, 4, 3, 2, 1); + REQUIRE(simd::to_array(simd::multiply(lhs, rhs)) == std::array{8, 14, 18, 20, 20, 18, 14, 8}); + REQUIRE(simd::to_array(simd::add_horizontal(lhs, rhs)) == std::array{3, 7, 15, 11, 11, 15, 7, 3}); + REQUIRE(simd::to_array(simd::shuffle_32_slow(lhs, 0b00'01'10'11)) == std::array{4, 3, 2, 1, 8, 7, 6, 5}); + REQUIRE(simd::to_array(simd::blend_slow(lhs, rhs, 0b01010101)) == std::array{8, 2, 6, 4, 4, 6, 2, 8}); } TEST_CASE("256-bit public 64-bit arithmetic contract", "[simdlib][avx2][int64][arithmetic]") @@ -137,8 +152,11 @@ TEST_CASE("256-bit public 64-bit arithmetic contract", "[simdlib][avx2][int64][a TEST_CASE("256-bit public byte operations cover multiplication and lane shifts", "[simdlib][avx2][byte][shift]") { using bytes = SimdLib::Api<256, std::uint8_t>; - REQUIRE(bytes::to_array(bytes::set1(0x81)) == std::array{0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81}); - REQUIRE(bytes::to_array(bytes::multiply(bytes::set1(0x81), bytes::set1(2))) == std::array{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}); + REQUIRE(bytes::to_array(bytes::set1(0x81)) == std::array{0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, + 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, + 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81}); + REQUIRE(bytes::to_array(bytes::multiply(bytes::set1(0x81), bytes::set1(2))) == + std::array{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}); REQUIRE(bytes::to_array(bytes::shift_left(bytes::set1(0x81), 1))[0] == 0x02); REQUIRE(bytes::to_array(bytes::shift_right(bytes::set1(0x81), 1))[0] == 0x40); using signed_bytes = SimdLib::Api<256, std::int8_t>; @@ -147,10 +165,10 @@ TEST_CASE("256-bit public byte operations cover multiplication and lane shifts", TEST_CASE("256-bit SimdVector preserves arithmetic and storage", "[simdlib][avx2][vector]") { - using vector = SimdLib::SimdVector; - const vector lhs{std::array{1, 2, 3, 4, 5, 6, 7, 8}}; - const vector rhs{2}; - REQUIRE(vector{lhs * rhs}.toArray() == std::array{2, 4, 6, 8, 10, 12, 14, 16}); + using vector = SimdLib::SimdVector; + const vector lhs{std::array{1, 2, 3, 4, 5, 6, 7, 8}}; + const vector rhs{2}; + REQUIRE(vector{lhs * rhs}.toArray() == std::array{2, 4, 6, 8, 10, 12, 14, 16}); } TEST_CASE("256-bit Api documentation examples produce their documented results", "[simdlib][avx2][documentation]") diff --git a/tests/Bmi.tests.cpp b/tests/Bmi.tests.cpp index f9d89e0..c0e9cac 100644 --- a/tests/Bmi.tests.cpp +++ b/tests/Bmi.tests.cpp @@ -326,8 +326,7 @@ void mix_digest(std::uint64_t &digest, const std::uint64_t value) digest *= 1099511628211ULL; } -template -void require_signed_bit_pattern_contract(const signed_t source, const signed_t rhs) +template void require_signed_bit_pattern_contract(const signed_t source, const signed_t rhs) { using unsigned_t = std::make_unsigned_t; const unsigned_t source_bits = std::bit_cast(source); @@ -360,10 +359,9 @@ void require_signed_bit_pattern_contract(const signed_t source, const signed_t r TEST_CASE("BMI signed helpers preserve two's-complement bit patterns", "[simdlib][bmi][signed][regression]") { - require_signed_bit_pattern_contract(std::bit_cast(0xF234'5678u), - std::bit_cast(0x8ACE'1357u)); + require_signed_bit_pattern_contract(std::bit_cast(0xF234'5678u), std::bit_cast(0x8ACE'1357u)); require_signed_bit_pattern_contract(std::bit_cast(0xF234'5678'9ABC'DEF0ull), - std::bit_cast(0x8ACE'1357'2468'BDF1ull)); + std::bit_cast(0x8ACE'1357'2468'BDF1ull)); } TEST_CASE("BMI absolute value handles signed boundaries without arithmetic overflow", "[simdlib][bmi][signed][abs]") diff --git a/tests/CompleteRegisterShift.tests.cpp b/tests/CompleteRegisterShift.tests.cpp new file mode 100644 index 0000000..fbf26b5 --- /dev/null +++ b/tests/CompleteRegisterShift.tests.cpp @@ -0,0 +1,104 @@ +#include "TestSupport.h" + +#include + +#include +#include +#include +#include + +#ifndef SIMDLIB_COMPLETE_SHIFT_TEST_WIDTH +#error "SIMDLIB_COMPLETE_SHIFT_TEST_WIDTH must select the tested register width" +#endif + +namespace +{ + +/** + * @brief Produces the scalar reference for a complete-register left byte shift. + * @tparam count Compile-time byte count. + * @tparam byte_count Complete register width in bytes. + * @param source Source bytes in low-to-high register order. + * @return Shifted bytes with zero-filled low positions. + */ +template +[[nodiscard]] constexpr std::array shift_bytes_left_oracle(const std::array &source) noexcept +{ + std::array result{}; + if constexpr (count < byte_count) + for (std::size_t index = count; index < byte_count; ++index) + result[index] = source[index - count]; + return result; +} + +/** + * @brief Produces the scalar reference for a complete-register right byte shift. + * @tparam count Compile-time byte count. + * @tparam byte_count Complete register width in bytes. + * @param source Source bytes in low-to-high register order. + * @return Shifted bytes with zero-filled high positions. + */ +template +[[nodiscard]] constexpr std::array shift_bytes_right_oracle(const std::array &source) noexcept +{ + std::array result{}; + if constexpr (count < byte_count) + for (std::size_t index = 0; index + count < byte_count; ++index) + result[index] = source[index + count]; + return result; +} + +/** + * @brief Verifies one immediate byte count against scalar and whole-bit-string references. + * @tparam count Compile-time byte count. + * @tparam element_t Integral lane interpretation used to prove cross-element behavior. + */ +template void require_immediate_byte_shift_count() +{ + using api = SimdLib::Api; + constexpr std::size_t byte_count = api::byte_count; + volatile std::uint8_t runtime_seed = 1; + std::array source_bytes{}; + for (std::size_t index = 0; index < byte_count; ++index) + source_bytes[index] = static_cast(runtime_seed + index * 7); + const auto source_lanes = std::bit_cast>(source_bytes); + const auto source = api::construct(source_lanes); + const auto left = std::bit_cast>(api::to_array(api::template shift_bytes_left(count)>(source))); + const auto right = std::bit_cast>(api::to_array(api::template shift_bytes_right(count)>(source))); + REQUIRE(left == shift_bytes_left_oracle(source_bytes)); + REQUIRE(right == shift_bytes_right_oracle(source_bytes)); + if constexpr (SIMDLIB_COMPLETE_SHIFT_TEST_WIDTH == 128) + { + const auto bit_left = + std::bit_cast>(api::to_array(api::template shift_bits_left(count * 8)>(source))); + const auto bit_right = + std::bit_cast>(api::to_array(api::template shift_bits_right(count * 8)>(source))); + REQUIRE(left == bit_left); + REQUIRE(right == bit_right); + } +} + +/** @brief Verifies every required immediate byte count for one lane interpretation. */ +template void require_immediate_byte_shift_counts() +{ + require_immediate_byte_shift_count<0, element_t>(); + require_immediate_byte_shift_count<1, element_t>(); + require_immediate_byte_shift_count<7, element_t>(); + require_immediate_byte_shift_count<8, element_t>(); + require_immediate_byte_shift_count<15, element_t>(); + require_immediate_byte_shift_count<16, element_t>(); + require_immediate_byte_shift_count<17, element_t>(); + require_immediate_byte_shift_count<31, element_t>(); + require_immediate_byte_shift_count<32, element_t>(); + require_immediate_byte_shift_count<33, element_t>(); +} + +} // namespace + +TEST_CASE("Immediate complete-register byte shifts match independent references", "[simdlib][shift][byte][immediate]") +{ + require_immediate_byte_shift_counts(); + require_immediate_byte_shift_counts(); + require_immediate_byte_shift_counts(); + require_immediate_byte_shift_counts(); +} \ No newline at end of file diff --git a/tests/Format.tests.cpp b/tests/Format.tests.cpp index 5d9336a..cd5ce71 100644 --- a/tests/Format.tests.cpp +++ b/tests/Format.tests.cpp @@ -124,21 +124,11 @@ TEST_CASE("uint128_t formatting supports documented integer presentation control TEST_CASE("uint128_t alternate octal formatting covers alignment padding and width branches", "[format][uint128][octal][parity]") { const std::array cases{ - octal_format_case{0, "{:#o}", "0"}, - octal_format_case{9, "{:#o}", "011"}, - octal_format_case{0, "{:#5o}", " 0"}, - octal_format_case{9, "{:#5o}", " 011"}, - octal_format_case{0, "{:>#5o}", " 0"}, - octal_format_case{9, "{:>#5o}", " 011"}, - octal_format_case{0, "{:<#5o}", "0 "}, - octal_format_case{9, "{:<#5o}", "011 "}, - octal_format_case{0, "{:#05o}", "00000"}, - octal_format_case{9, "{:#05o}", "00011"}, - octal_format_case{0, "{:>#05o}", " 0"}, - octal_format_case{9, "{:>#05o}", " 011"}, - octal_format_case{0, "{:#1o}", "0"}, - octal_format_case{9, "{:#2o}", "011"}, - octal_format_case{0, "{:#01o}", "0"}, + octal_format_case{0, "{:#o}", "0"}, octal_format_case{9, "{:#o}", "011"}, octal_format_case{0, "{:#5o}", " 0"}, + octal_format_case{9, "{:#5o}", " 011"}, octal_format_case{0, "{:>#5o}", " 0"}, octal_format_case{9, "{:>#5o}", " 011"}, + octal_format_case{0, "{:<#5o}", "0 "}, octal_format_case{9, "{:<#5o}", "011 "}, octal_format_case{0, "{:#05o}", "00000"}, + octal_format_case{9, "{:#05o}", "00011"}, octal_format_case{0, "{:>#05o}", " 0"}, octal_format_case{9, "{:>#05o}", " 011"}, + octal_format_case{0, "{:#1o}", "0"}, octal_format_case{9, "{:#2o}", "011"}, octal_format_case{0, "{:#01o}", "0"}, octal_format_case{9, "{:#02o}", "011"}, }; @@ -153,14 +143,15 @@ TEST_CASE("uint128_t alternate octal formatting covers alignment padding and wid TEST_CASE("uint128_t formatting matches the standard uint64 formatter within the scalar range", "[format][uint128][parity]") { - const std::array values{ - std::uint64_t{0}, std::uint64_t{1}, std::uint64_t{9}, std::uint64_t{42}, - std::uint64_t{0x1234'5678'9ABC'DEF0}, std::numeric_limits::max()}; - const std::array formats{ - "{}", "{:d}", "{:x}", "{:X}", "{:b}", "{:B}", "{:o}", - "{:+}", "{: }", "{:-}", "{:#d}", "{:#x}", "{:#X}", "{:#b}", "{:#B}", "{:#o}", - "{:024x}", "{:*>30x}", "{:>24x}", "{:*<24x}", "{:*^24x}", - "{:#024x}", "{:+024x}", "{:0>24x}"}; + const std::array values{std::uint64_t{0}, + std::uint64_t{1}, + std::uint64_t{9}, + std::uint64_t{42}, + std::uint64_t{0x1234'5678'9ABC'DEF0}, + std::numeric_limits::max()}; + const std::array formats{"{}", "{:d}", "{:x}", "{:X}", "{:b}", "{:B}", "{:o}", "{:+}", + "{: }", "{:-}", "{:#d}", "{:#x}", "{:#X}", "{:#b}", "{:#B}", "{:#o}", + "{:024x}", "{:*>30x}", "{:>24x}", "{:*<24x}", "{:*^24x}", "{:#024x}", "{:+024x}", "{:0>24x}"}; for (std::uint64_t scalar : values) { @@ -168,8 +159,7 @@ TEST_CASE("uint128_t formatting matches the standard uint64 formatter within the for (const std::string_view format : formats) { CAPTURE(scalar, std::string(format)); - CHECK(std::vformat(format, std::make_format_args(wide)) == - std::vformat(format, std::make_format_args(scalar))); + CHECK(std::vformat(format, std::make_format_args(wide)) == std::vformat(format, std::make_format_args(scalar))); } } } @@ -177,12 +167,8 @@ TEST_CASE("uint128_t formatting matches the standard uint64 formatter within the TEST_CASE("uint128_t formatting rejects unsupported specifications", "[format][uint128]") { const std::array cases{ - invalid_format_case{"opening brace fill", "{<5}"}, - invalid_format_case{"precision", ".2}"}, - invalid_format_case{"dynamic width", "{}"}, - invalid_format_case{"nested replacement field", ">{}"}, - invalid_format_case{"locale", "L}"}, - invalid_format_case{"unsupported presentation", "q}"}, + invalid_format_case{"opening brace fill", "{<5}"}, invalid_format_case{"precision", ".2}"}, invalid_format_case{"dynamic width", "{}"}, + invalid_format_case{"nested replacement field", ">{}"}, invalid_format_case{"locale", "L}"}, invalid_format_case{"unsupported presentation", "q}"}, invalid_format_case{"trailing specification", "dx}"}, }; for (const auto &test : cases) @@ -190,8 +176,7 @@ TEST_CASE("uint128_t formatting rejects unsupported specifications", "[format][u require_parse_rejected(test); } - const invalid_format_case overflow{ - "width overflow", "184467440737095516160}"}; + const invalid_format_case overflow{"width overflow", "184467440737095516160}"}; require_parse_rejected(overflow); const SimdLib::uint128_t value{1}; diff --git a/tests/ImmediateControlSlowPaths.tests.cpp b/tests/ImmediateControlSlowPaths.tests.cpp new file mode 100644 index 0000000..c488ca0 --- /dev/null +++ b/tests/ImmediateControlSlowPaths.tests.cpp @@ -0,0 +1,217 @@ +#include "TestSupport.h" + +#include + +#include +#include +#include +#include +#include + +#ifndef SIMDLIB_IMMEDIATE_CONTROL_TEST_WIDTH +#error "SIMDLIB_IMMEDIATE_CONTROL_TEST_WIDTH must select the tested register width" +#endif + +namespace SimdLib::Tests +{ + +/** + * @brief Creates distinct lane values suitable for immediate-control reference comparisons. + * @tparam api_t Api specialization whose lane array is produced. + * @param offset Offset added to each logical lane index. + * @return Array containing monotonically increasing, exactly representable lane values. + */ +template constexpr std::array make_control_values(const int offset) +{ + std::array result{}; + for (std::size_t index = 0; index < result.size(); ++index) + result[index] = static_cast(offset + static_cast(index)); + return result; +} + +/** + * @brief Verifies every runtime blend control byte for one supported lane type and width. + * @tparam Width SIMD register width in bits. + * @tparam Element Lane type accepted by the immediate blend family. + */ +template void require_blend_slow_controls() +{ + using api = SimdLib::Api; + const auto left = make_control_values(1); + const auto right = make_control_values(65); + const auto lhs = api::construct(left); + const auto rhs = api::construct(right); + for (unsigned int control = 0; control <= 0xFFu; ++control) + { + auto expected = left; + for (std::size_t index = 0; index < expected.size(); ++index) + { + if ((control & (1u << (index % 8))) != 0) + expected[index] = right[index]; + } + const volatile int runtime_control = static_cast(control); + REQUIRE(api::to_array(api::blend_slow(lhs, rhs, runtime_control)) == expected); + } +} + +/** + * @brief Verifies every runtime low- and high-half 16-bit shuffle control byte. + * @tparam Width SIMD register width in bits. + */ +template void require_half_shuffle_slow_controls() +{ + using api = SimdLib::Api; + const auto source = make_control_values(1); + const auto value = api::construct(source); + for (unsigned int control = 0; control <= 0xFFu; ++control) + { + auto low = source; + auto high = source; + for (std::size_t group = 0; group < source.size(); group += 8) + { + for (std::size_t index = 0; index < 4; ++index) + { + const std::size_t selected = (control >> (index * 2)) & 0x3u; + low[group + index] = source[group + selected]; + high[group + 4 + index] = source[group + 4 + selected]; + } + } + const volatile int runtime_control = static_cast(control); + REQUIRE(api::to_array(api::shuffle_lo_slow(value, runtime_control)) == low); + REQUIRE(api::to_array(api::shuffle_hi_slow(value, runtime_control)) == high); + } +} + +/** + * @brief Verifies every runtime 32-bit shuffle control byte. + * @tparam Width SIMD register width in bits. + */ +template void require_shuffle_32_slow_controls() +{ + using api = SimdLib::Api; + const auto source = make_control_values(1); + const auto value = api::construct(source); + for (unsigned int control = 0; control <= 0xFFu; ++control) + { + std::array expected{}; + for (std::size_t group = 0; group < source.size(); group += 4) + { + for (std::size_t index = 0; index < 4; ++index) + expected[group + index] = source[group + ((control >> (index * 2)) & 0x3u)]; + } + const volatile std::uint32_t runtime_control = control; + REQUIRE(api::to_array(api::shuffle_32_slow(value, runtime_control)) == expected); + } +} + +/** + * @brief Verifies every runtime floating-point shuffle control byte for one lane type. + * @tparam Width SIMD register width in bits. + * @tparam Element Floating-point lane type. + */ +template void require_floating_shuffle_slow_controls() +{ + using api = SimdLib::Api; + const auto left = make_control_values(1); + const auto right = make_control_values(65); + const auto lhs = api::construct(left); + const auto rhs = api::construct(right); + for (unsigned int control = 0; control <= 0xFFu; ++control) + { + std::array expected{}; + if constexpr (std::is_same_v) + { + for (std::size_t group = 0; group < expected.size(); group += 4) + { + expected[group] = left[group + (control & 0x3u)]; + expected[group + 1] = left[group + ((control >> 2) & 0x3u)]; + expected[group + 2] = right[group + ((control >> 4) & 0x3u)]; + expected[group + 3] = right[group + ((control >> 6) & 0x3u)]; + } + } + else + { + for (std::size_t group = 0; group < expected.size(); group += 2) + { + const unsigned int group_control = control >> group; + expected[group] = left[group + (group_control & 0x1u)]; + expected[group + 1] = right[group + ((group_control >> 1) & 0x1u)]; + } + } + const volatile int runtime_control = static_cast(control); + REQUIRE(api::to_array(api::shuffle_slow(lhs, rhs, runtime_control)) == expected); + } +} + +/** @brief Verifies every immediate-control emulation family for one register width. */ +template void require_immediate_control_slow_matrix() +{ + require_blend_slow_controls(); + require_blend_slow_controls(); + require_blend_slow_controls(); + require_blend_slow_controls(); + require_blend_slow_controls(); + require_blend_slow_controls(); + require_half_shuffle_slow_controls(); + require_shuffle_32_slow_controls(); + require_floating_shuffle_slow_controls(); + require_floating_shuffle_slow_controls(); +} + +/** @brief Verifies every valid complete-register bit count and its documented boundaries. */ +inline void require_complete_register_shift_slow_controls() +{ + using api = SimdLib::Api<128, std::uint64_t>; + const std::array source{0x0123456789ABCDEFULL, 0xFEDCBA9876543210ULL}; + const auto value = api::construct(source); + for (int count = -1; count <= 129; ++count) + { + std::array left{}; + std::array right{}; + if (count <= 0) + { + left = source; + right = source; + } + else if (count < 64) + { + left = {source[0] << count, (source[1] << count) | (source[0] >> (64 - count))}; + right = {(source[0] >> count) | (source[1] << (64 - count)), source[1] >> count}; + } + else if (count == 64) + { + left = {0, source[0]}; + right = {source[1], 0}; + } + else if (count < 128) + { + left = {0, source[0] << (count - 64)}; + right = {source[1] >> (count - 64), 0}; + } + const volatile int runtime_count = count; + REQUIRE(api::to_array(api::shift_bits_left_slow(value, runtime_count)) == left); + REQUIRE(api::to_array(api::shift_bits_right_slow(value, runtime_count)) == right); + } + const volatile int minimum_count = std::numeric_limits::lowest(); + const volatile int maximum_count = std::numeric_limits::max(); + REQUIRE(api::to_array(api::shift_bits_left_slow(value, minimum_count)) == source); + REQUIRE(api::to_array(api::shift_bits_right_slow(value, minimum_count)) == source); + REQUIRE(api::to_array(api::shift_bits_left_slow(value, maximum_count)) == std::array{}); + REQUIRE(api::to_array(api::shift_bits_right_slow(value, maximum_count)) == std::array{}); +} + +} // namespace SimdLib::Tests + +using namespace SimdLib::Tests; + +TEST_CASE("Runtime immediate-control substitutes cover every control byte", "[simdlib][immediate-control][slow]") +{ + require_immediate_control_slow_matrix(); +} + +#if SIMDLIB_IMMEDIATE_CONTROL_TEST_WIDTH == 128 +TEST_CASE("Complete-register slow shifts cover every valid count", "[simdlib][immediate-control][slow][shift]") +{ + require_complete_register_shift_slow_controls(); +} +#endif \ No newline at end of file diff --git a/tests/LogicalShuffleApi.tests.cpp b/tests/LogicalShuffleApi.tests.cpp new file mode 100644 index 0000000..a0e0c4e --- /dev/null +++ b/tests/LogicalShuffleApi.tests.cpp @@ -0,0 +1,141 @@ +#include "LogicalShuffleTestSupport.h" + +#include +#include + +#include + +#include +#include +#include + +#ifndef SIMDLIB_LOGICAL_SHUFFLE_TEST_WIDTH +#error "SIMDLIB_LOGICAL_SHUFFLE_TEST_WIDTH must select the Api test width" +#endif + +namespace +{ + +using namespace SimdLib::Tests::LogicalShuffle; + +/** + * @brief Invokes one Api logical shuffle by expanding a selector array. + * @tparam api_t Api specialization under test. + * @tparam selectors Logical source-lane selectors. + * @tparam positions Output lane positions. + * @param value Source native register. + * @return Native register returned by the logical shuffle. + */ +template +[[nodiscard]] auto invoke_api_shuffle(typename api_t::vector_t value, std::index_sequence) noexcept +{ + return api_t::template shuffle(value); +} + +/** + * @brief Reports whether one Api exposes a complete logical selector sequence. + * @tparam api_t Api specialization under test. + * @tparam selectors Logical source-lane selectors. + * @tparam positions Output lane positions. + * @return True when the selector-pack overload participates in overload resolution. + */ +template +[[nodiscard]] consteval bool api_accepts_shuffle_impl(std::index_sequence) noexcept +{ + return SimdLib::IApi::Shuffle; +} + +/** + * @brief Reports whether an Api retains its native register-selector byte shuffle overload. + * @tparam api_t Api specialization under test. + */ +template +concept accepts_register_selector_shuffle = requires(typename api_t::vector_t value) { api_t::shuffle(value, value); }; + +/** + * @brief Reports whether an Api exposes its scalar-control floating shuffle slow path. + * @tparam api_t Api specialization under test. + */ +template +concept accepts_scalar_control_shuffle_slow = requires(typename api_t::vector_t value) { api_t::shuffle_slow(value, value, 0); }; + +/** + * @brief Reports whether one Api exposes its complete identity logical shuffle. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return True when the desired logical-shuffle interface is available. + */ +template [[nodiscard]] consteval bool api_accepts_identity_shuffle() noexcept +{ + using api_t = SimdLib::Api; + constexpr auto selectors = identity_selectors(); + return api_accepts_shuffle_impl(std::make_index_sequence{}); +} + +/** + * @brief Compares one Api shuffle result against the independent scalar oracle. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @tparam selectors Logical source-lane selectors. + */ +template void require_api_shuffle() noexcept +{ + using api_t = SimdLib::Api; + constexpr auto source = distinct_lanes(); + const auto actual = api_t::to_array(invoke_api_shuffle(api_t::construct(source), std::make_index_sequence{})); + constexpr auto expected = logical_shuffle_oracle(source); + REQUIRE(same_object_representations(actual, expected)); +} + +/** + * @brief Exercises every required logical selector pattern for one Api specialization. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + */ +template void require_api_shuffle_suite() noexcept +{ + require_api_shuffle()>(); + require_api_shuffle()>(); + require_api_shuffle()>(); + require_api_shuffle()>(); + require_api_shuffle()>(); + require_api_shuffle()>(); + require_api_shuffle()>(); + if constexpr (bits == 256) + { + require_api_shuffle()>(); + require_api_shuffle()>(); + require_api_shuffle()>(); + require_api_shuffle()>(); + } +} + +static_assert(api_accepts_identity_shuffle()); +static_assert(api_accepts_identity_shuffle()); +static_assert(api_accepts_identity_shuffle()); +static_assert(api_accepts_identity_shuffle()); +static_assert(api_accepts_identity_shuffle()); +static_assert(api_accepts_identity_shuffle()); +static_assert(api_accepts_identity_shuffle()); +static_assert(api_accepts_identity_shuffle()); +static_assert(api_accepts_identity_shuffle()); +static_assert(api_accepts_identity_shuffle()); +static_assert(accepts_register_selector_shuffle>); +static_assert(accepts_scalar_control_shuffle_slow>); +static_assert(accepts_scalar_control_shuffle_slow>); + +TEST_CASE("Api logical shuffle matches an independent object-representation oracle", "[simdlib][logical-shuffle]") +{ + require_api_shuffle_suite(); + require_api_shuffle_suite(); + require_api_shuffle_suite(); + require_api_shuffle_suite(); + require_api_shuffle_suite(); + require_api_shuffle_suite(); + require_api_shuffle_suite(); + require_api_shuffle_suite(); + require_api_shuffle_suite(); + require_api_shuffle_suite(); +} + +} // namespace diff --git a/tests/LogicalShuffleImpl128.tests.cpp b/tests/LogicalShuffleImpl128.tests.cpp new file mode 100644 index 0000000..82cda12 --- /dev/null +++ b/tests/LogicalShuffleImpl128.tests.cpp @@ -0,0 +1,93 @@ +#include "LogicalShuffleTestSupport.h" + +#include + +#include + +#include +#include +#include + +namespace +{ + +using namespace SimdLib::Tests::LogicalShuffle; + +/** + * @brief Invokes one 128-bit mapping-layer logical shuffle. + * @tparam element_t Logical lane type. + * @tparam selectors Logical source-lane selectors. + * @tparam positions Output lane positions. + * @param value Source native register. + * @return Native register returned by the selected element specialization. + */ +template +[[nodiscard]] auto invoke_mapping_shuffle(typename SimdLib::Api<128, element_t>::vector_t value, std::index_sequence) noexcept +{ + using mapping_t = SimdLib::Detail::SimdMappings<128, element_t>; + return mapping_t::template shuffle(value); +} + +/** + * @brief Reports whether one mapping exposes the supplied logical selector sequence. + * @tparam mapping_t Mapping specialization under test. + * @tparam indices Logical source-lane selectors. + */ +template +concept accepts_mapping_shuffle = requires(typename mapping_t::vector_t value) { mapping_t::template shuffle(value); }; + +/** + * @brief Compares one mapping shuffle result against the independent scalar oracle. + * @tparam element_t Logical lane type. + * @tparam selectors Logical source-lane selectors. + */ +template void require_mapping_shuffle() noexcept +{ + using api_t = SimdLib::Api<128, element_t>; + constexpr auto source = distinct_lanes(); + const auto actual = + api_t::to_array(invoke_mapping_shuffle(api_t::construct(source), std::make_index_sequence{})); + constexpr auto expected = logical_shuffle_oracle(source); + REQUIRE(same_object_representations(actual, expected)); +} + +/** + * @brief Exercises every required selector pattern for one 128-bit element specialization. + * @tparam element_t Logical lane type. + */ +template void require_mapping_shuffle_suite() noexcept +{ + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); +} + +using int32_mapping = SimdLib::Detail::SimdMappings<128, std::int32_t>; +static_assert(accepts_mapping_shuffle); +static_assert(!accepts_mapping_shuffle); +static_assert(!accepts_mapping_shuffle); +static_assert(SimdLib::Detail::encode_logical_shuffle_32_immediate<3, 2, 1, 0>() == 0x1B); +static_assert(SimdLib::Detail::encode_logical_shuffle_16_byte(3, 0) == 6); +static_assert(SimdLib::Detail::encode_logical_shuffle_16_byte(3, 1) == 7); +static_assert(SimdLib::Detail::encode_logical_shuffle_64_immediate<1, 0>() == 0x4E); +static_assert(SimdLib::Detail::encode_logical_shuffle_double_immediate<1, 0>() == 0x01); + +TEST_CASE("128-bit mapping logical shuffle matches an independent object-representation oracle", "[simdlib][logical-shuffle][backend]") +{ + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); +} + +} // namespace diff --git a/tests/LogicalShuffleImpl256.tests.cpp b/tests/LogicalShuffleImpl256.tests.cpp new file mode 100644 index 0000000..d1eedad --- /dev/null +++ b/tests/LogicalShuffleImpl256.tests.cpp @@ -0,0 +1,111 @@ +#include "LogicalShuffleTestSupport.h" + +#include + +#include + +#include +#include +#include + +namespace +{ + +using namespace SimdLib::Tests::LogicalShuffle; + +/** + * @brief Invokes one 256-bit mapping-layer logical shuffle. + * @tparam element_t Logical lane type. + * @tparam selectors Logical source-lane selectors. + * @tparam positions Output lane positions. + * @param value Source native register. + * @return Native register returned by the selected element specialization. + */ +template +[[nodiscard]] auto invoke_mapping_shuffle(typename SimdLib::Api<256, element_t>::vector_t value, std::index_sequence) noexcept +{ + using mapping_t = SimdLib::Detail::SimdMappings<256, element_t>; + return mapping_t::template shuffle(value); +} + +/** + * @brief Reports whether one mapping exposes the supplied logical selector sequence. + * @tparam mapping_t Mapping specialization under test. + * @tparam indices Logical source-lane selectors. + */ +template +concept accepts_mapping_shuffle = requires(typename mapping_t::vector_t value) { mapping_t::template shuffle(value); }; + +/** + * @brief Reports whether one mapping accepts a complete selector array. + * @tparam mapping_t Mapping specialization under test. + * @tparam selectors Complete selector array. + * @tparam positions Output lane positions. + * @return True when the mapping accepts the expanded selector pack. + */ +template +[[nodiscard]] consteval bool mapping_accepts_selectors(std::index_sequence) noexcept +{ + return accepts_mapping_shuffle; +} + +/** + * @brief Compares one mapping shuffle result against the independent scalar oracle. + * @tparam element_t Logical lane type. + * @tparam selectors Logical source-lane selectors. + */ +template void require_mapping_shuffle() noexcept +{ + using api_t = SimdLib::Api<256, element_t>; + constexpr auto source = distinct_lanes(); + const auto actual = + api_t::to_array(invoke_mapping_shuffle(api_t::construct(source), std::make_index_sequence{})); + constexpr auto expected = logical_shuffle_oracle(source); + REQUIRE(same_object_representations(actual, expected)); +} + +/** + * @brief Exercises local, cross-half, and mixed selector patterns for one 256-bit specialization. + * @tparam element_t Logical lane type. + */ +template void require_mapping_shuffle_suite() noexcept +{ + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); + require_mapping_shuffle()>(); +} + +using byte_mapping = SimdLib::Detail::SimdMappings<256, std::int8_t>; +using dword_mapping = SimdLib::Detail::SimdMappings<256, std::int32_t>; +constexpr auto byte_half_swap = swap_half_selectors(); +static_assert(mapping_accepts_selectors(std::make_index_sequence{})); +static_assert(accepts_mapping_shuffle); +static_assert(!accepts_mapping_shuffle); +static_assert(!accepts_mapping_shuffle); +static_assert(SimdLib::Detail::logical_shuffle_256_has_cross_half_selector<16, byte_half_swap>()); +static_assert(!SimdLib::Detail::logical_shuffle_256_has_local_half_selector<16, byte_half_swap>()); +static_assert(SimdLib::Detail::encode_logical_shuffle_256_byte(1, true, 0, byte_half_swap[0], 0) == 0); + +TEST_CASE("256-bit mapping logical shuffle supports full-register lane selection", "[simdlib][logical-shuffle][backend]") +{ + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); + require_mapping_shuffle_suite(); +} + +} // namespace diff --git a/tests/LogicalShuffleRegister.tests.cpp b/tests/LogicalShuffleRegister.tests.cpp new file mode 100644 index 0000000..19281ac --- /dev/null +++ b/tests/LogicalShuffleRegister.tests.cpp @@ -0,0 +1,236 @@ +#include "LogicalShuffleTestSupport.h" + +#include +#include + +#include + +#include +#include +#include + +#ifndef SIMDLIB_REGISTER_TEST_ENABLE_256 +#define SIMDLIB_REGISTER_TEST_ENABLE_256 SIMDLIB_HAS_AVX2 +#endif + +namespace +{ + +using namespace SimdLib::Tests::LogicalShuffle; + +/** + * @brief Invokes one Register logical shuffle by expanding a selector array. + * @tparam register_t Register specialization under test. + * @tparam selectors Logical source-lane selectors. + * @tparam positions Output lane positions. + * @param value Source Register. + * @return Register returned by the logical shuffle. + */ +template +[[nodiscard]] register_t invoke_register_shuffle(register_t value, std::index_sequence) noexcept +{ + return value.template shuffle(); +} + +/** + * @brief Invokes one Register byte shuffle by expanding a selector array. + * @tparam register_t Register specialization under test. + * @tparam selectors Source-byte selectors. + * @tparam positions Output byte positions. + * @param value Source Register. + * @return Register returned by the byte shuffle. + */ +template +[[nodiscard]] register_t invoke_register_byte_shuffle(register_t value, std::index_sequence) noexcept +{ + return value.template shuffle_bytes(); +} + +/** + * @brief Reports whether one Register exposes a complete logical selector sequence. + * @tparam register_t Register specialization under test. + * @tparam selectors Logical source-lane selectors. + * @tparam positions Output lane positions. + * @return True when the selector-pack member participates in overload resolution. + */ +template +[[nodiscard]] consteval bool register_accepts_shuffle_impl(std::index_sequence) noexcept +{ + return SimdLib::IRegister::Shuffle; +} + +/** + * @brief Reports whether one Register exposes a complete byte selector sequence. + * @tparam register_t Register specialization under test. + * @tparam selectors Source-byte selectors. + * @tparam positions Output byte positions. + * @return True when the selector-pack member participates in overload resolution. + */ +template +[[nodiscard]] consteval bool register_accepts_byte_shuffle_impl(std::index_sequence) noexcept +{ + return SimdLib::IRegister::ShuffleBytes; +} + +/** + * @brief Reports whether one Register exposes its complete identity logical shuffle. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return True when the desired logical-shuffle interface is available. + */ +template [[nodiscard]] consteval bool register_accepts_identity_shuffle() noexcept +{ + using register_t = SimdLib::Register; + constexpr auto selectors = identity_selectors(); + return register_accepts_shuffle_impl(std::make_index_sequence{}); +} + +/** + * @brief Reports whether one Register exposes its complete identity byte shuffle. + * @tparam element_t Logical lane type retained by the result. + * @tparam bits Register width in bits. + * @return True when one selector is accepted for every byte. + */ +template [[nodiscard]] consteval bool register_accepts_identity_byte_shuffle() noexcept +{ + using register_t = SimdLib::Register; + constexpr auto selectors = identity_selectors(); + return register_accepts_byte_shuffle_impl(std::make_index_sequence{}); +} + +/** + * @brief Compares one Register shuffle result against the independent scalar oracle. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @tparam selectors Logical source-lane selectors. + */ +template void require_register_shuffle() noexcept +{ + using register_t = SimdLib::Register; + constexpr auto source = distinct_lanes(); + const auto actual = + invoke_register_shuffle(register_t::from_array(source), std::make_index_sequence{}).to_array(); + constexpr auto expected = logical_shuffle_oracle(source); + REQUIRE(same_object_representations(actual, expected)); +} + +/** + * @brief Compares one Register byte shuffle against an independent byte-array oracle. + * @tparam element_t Logical lane type retained by the result. + * @tparam bits Register width in bits. + * @tparam selectors Source-byte selectors. + */ +template void require_register_byte_shuffle() noexcept +{ + using register_t = SimdLib::Register; + constexpr auto source = distinct_lanes(); + const auto actual_register = + invoke_register_byte_shuffle(register_t::from_array(source), std::make_index_sequence{}); + const auto actual = std::bit_cast>(actual_register.to_array()); + constexpr auto source_bytes = std::bit_cast>(source); + constexpr auto expected = logical_shuffle_oracle(source_bytes); + REQUIRE(actual == expected); +} + +/** + * @brief Exercises every required logical selector pattern for one Register specialization. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + */ +template void require_register_shuffle_suite() noexcept +{ + require_register_shuffle()>(); + require_register_shuffle()>(); + require_register_shuffle()>(); + require_register_shuffle()>(); + require_register_shuffle()>(); + require_register_shuffle()>(); + require_register_shuffle()>(); + if constexpr (bits == 256) + { + require_register_shuffle()>(); + require_register_shuffle()>(); + require_register_shuffle()>(); + require_register_shuffle()>(); + } +} + +/** @brief Exercises representative local and cross-half byte selector patterns for one Register shape. */ +template void require_register_byte_shuffle_suite() noexcept +{ + require_register_byte_shuffle()>(); + require_register_byte_shuffle()>(); + require_register_byte_shuffle()>(); + if constexpr (bits == 256) + { + require_register_byte_shuffle()>(); + require_register_byte_shuffle()>(); + require_register_byte_shuffle()>(); + } +} + +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_byte_shuffle()); +static_assert(register_accepts_identity_byte_shuffle()); + +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_shuffle()); +static_assert(register_accepts_identity_byte_shuffle()); +static_assert(register_accepts_identity_byte_shuffle()); +static_assert(register_accepts_identity_shuffle()); +#endif + +TEST_CASE("Register logical shuffle matches an independent object-representation oracle", "[simdlib][register][logical-shuffle]") +{ + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); + require_register_shuffle_suite(); +#endif +} + +TEST_CASE("Register byte shuffle preserves the element type while selecting complete-register bytes", "[simdlib][register][logical-shuffle]") +{ + require_register_byte_shuffle_suite(); + require_register_byte_shuffle_suite(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_register_byte_shuffle_suite(); + require_register_byte_shuffle_suite(); +#endif +} + +} // namespace diff --git a/tests/LogicalShuffleTestSupport.h b/tests/LogicalShuffleTestSupport.h new file mode 100644 index 0000000..2260948 --- /dev/null +++ b/tests/LogicalShuffleTestSupport.h @@ -0,0 +1,313 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace SimdLib::Tests::LogicalShuffle +{ + +/** @brief Maps an object size to an unsigned integer with the same representation size. */ +template struct unsigned_bits; + +/** @brief Maps one-byte objects to an unsigned representation type. */ +template <> struct unsigned_bits<1> +{ + using type = std::uint8_t; +}; + +/** @brief Maps two-byte objects to an unsigned representation type. */ +template <> struct unsigned_bits<2> +{ + using type = std::uint16_t; +}; + +/** @brief Maps four-byte objects to an unsigned representation type. */ +template <> struct unsigned_bits<4> +{ + using type = std::uint32_t; +}; + +/** @brief Maps eight-byte objects to an unsigned representation type. */ +template <> struct unsigned_bits<8> +{ + using type = std::uint64_t; +}; + +/** @brief Unsigned integer type that preserves one element's complete object representation. */ +template using object_bits_t = typename unsigned_bits::type; + +/** + * @brief Creates one deterministic integer lane with nonuniform bytes. + * @tparam element_t Integral lane type. + * @param lane Logical lane index. + * @return Element whose object representation is unique within every supported register width. + */ +template [[nodiscard]] constexpr element_t distinct_integer_lane(const std::size_t lane) noexcept +{ + using bits_t = object_bits_t; + bits_t bits{}; + if constexpr (sizeof(element_t) == 1) + bits = static_cast(0xA5u + lane * 0x3Du); + else if constexpr (sizeof(element_t) == 2) + bits = static_cast(0xA55Au + lane * 0x1F3Du); + else if constexpr (sizeof(element_t) == 4) + bits = static_cast(0xA55AC33Cu + lane * 0x01020409u); + else + bits = static_cast(UINT64_C(0xA55AC33CF00F9669) + lane * UINT64_C(0x0102040810204081)); + return std::bit_cast(bits); +} + +/** + * @brief Creates one floating lane with a deliberately observable object representation. + * @tparam element_t Floating-point lane type. + * @param lane Logical lane index. + * @return Element selected from finite, signed-zero, infinity, subnormal, and NaN bit patterns. + */ +template [[nodiscard]] constexpr element_t distinct_floating_lane(const std::size_t lane) noexcept +{ + if constexpr (sizeof(element_t) == 4) + { + constexpr std::array patterns{0x00000000u, 0x80000000u, 0x7FC00001u, 0xFFC12345u, 0x3F800001u, 0xBF000003u, 0x00800005u, 0x7F800000u}; + return std::bit_cast(patterns[lane]); + } + else + { + constexpr std::array patterns{UINT64_C(0x8000000000000000), UINT64_C(0x7FF8000000000001), UINT64_C(0x0000000000000000), + UINT64_C(0xFFF8123456789ABC)}; + return std::bit_cast(patterns[lane]); + } +} + +/** + * @brief Creates unique lane representations for one supported SIMD shape. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return Complete source lane array. + */ +template [[nodiscard]] constexpr std::array distinct_lanes() noexcept +{ + std::array result{}; + for (std::size_t lane = 0; lane < result.size(); ++lane) + { + if constexpr (std::is_floating_point_v) + result[lane] = distinct_floating_lane(lane); + else + result[lane] = distinct_integer_lane(lane); + } + return result; +} + +/** + * @brief Builds identity selectors for one logical SIMD shape. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return One selector per output lane. + */ +template [[nodiscard]] consteval auto identity_selectors() noexcept +{ + std::array result{}; + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = lane; + return result; +} + +/** + * @brief Builds a complete reversal inside each 128-bit source group. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return One group-local selector per output lane. + */ +template [[nodiscard]] consteval auto reverse_selectors() noexcept +{ + constexpr std::size_t lanes_per_group = 128 / (sizeof(element_t) * 8); + auto result = identity_selectors(); + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = lane / lanes_per_group * lanes_per_group + lanes_per_group - 1 - lane % lanes_per_group; + return result; +} + +/** + * @brief Builds selectors that broadcast the first lane of every 128-bit group. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return One group-local selector per output lane. + */ +template [[nodiscard]] consteval auto first_lane_selectors() noexcept +{ + constexpr std::size_t lanes_per_group = 128 / (sizeof(element_t) * 8); + auto result = identity_selectors(); + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = lane / lanes_per_group * lanes_per_group; + return result; +} + +/** + * @brief Builds selectors that broadcast the last lane of every 128-bit group. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return One group-local selector per output lane. + */ +template [[nodiscard]] consteval auto last_lane_selectors() noexcept +{ + constexpr std::size_t lanes_per_group = 128 / (sizeof(element_t) * 8); + auto result = identity_selectors(); + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = lane / lanes_per_group * lanes_per_group + lanes_per_group - 1; + return result; +} + +/** + * @brief Builds selectors containing repeated adjacent source lanes. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return One group-local selector per output lane. + */ +template [[nodiscard]] consteval auto repeated_selectors() noexcept +{ + constexpr std::size_t lanes_per_group = 128 / (sizeof(element_t) * 8); + auto result = identity_selectors(); + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = lane / lanes_per_group * lanes_per_group + (lane % lanes_per_group) / 2; + return result; +} + +/** + * @brief Builds selectors that swap every adjacent logical lane pair. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return One group-local selector per output lane. + */ +template [[nodiscard]] consteval auto pair_swap_selectors() noexcept +{ + auto result = identity_selectors(); + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = lane ^ std::size_t{1}; + return result; +} + +/** + * @brief Builds a one-lane left rotation inside each 128-bit group. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return One group-local selector per output lane. + */ +template [[nodiscard]] consteval auto rotation_selectors() noexcept +{ + constexpr std::size_t lanes_per_group = 128 / (sizeof(element_t) * 8); + auto result = identity_selectors(); + for (std::size_t lane = 0; lane < result.size(); ++lane) + result[lane] = lane / lanes_per_group * lanes_per_group + (lane % lanes_per_group + 1) % lanes_per_group; + return result; +} + +/** + * @brief Builds different lower- and upper-group permutations for a 256-bit shape. + * @tparam element_t Logical lane type. + * @return Identity selectors below bit 128 and reversed selectors above bit 128. + */ +template [[nodiscard]] consteval auto distinct_group_selectors() noexcept +{ + constexpr std::size_t lanes_per_group = 128 / (sizeof(element_t) * 8); + auto result = identity_selectors(); + for (std::size_t lane = lanes_per_group; lane < result.size(); ++lane) + result[lane] = lanes_per_group + lanes_per_group - 1 - lane % lanes_per_group; + return result; +} + +/** + * @brief Builds selectors that exchange the lower and upper 128-bit halves. + * @tparam element_t Logical lane type. + * @return One cross-half source selector per output lane. + */ +template [[nodiscard]] consteval auto swap_half_selectors() noexcept +{ + constexpr std::size_t lane_count = 256 / (sizeof(element_t) * 8); + constexpr std::size_t lanes_per_half = lane_count / 2; + auto result = identity_selectors(); + for (std::size_t lane = 0; lane < lane_count; ++lane) + result[lane] = (lane + lanes_per_half) % lane_count; + return result; +} + +/** + * @brief Builds selectors combining local-half and cross-half sources. + * @tparam element_t Logical lane type. + * @return Identity selectors except for exchanged first lanes in each 128-bit half. + */ +template [[nodiscard]] consteval auto mixed_half_selectors() noexcept +{ + constexpr std::size_t lane_count = 256 / (sizeof(element_t) * 8); + constexpr std::size_t lanes_per_half = lane_count / 2; + auto result = identity_selectors(); + result[0] = lanes_per_half; + result[lanes_per_half] = 0; + return result; +} + +/** + * @brief Builds a complete low-to-high reversal across the entire 256-bit register. + * @tparam element_t Logical lane type. + * @return One full-width reverse selector per output lane. + */ +template [[nodiscard]] consteval auto full_reverse_selectors() noexcept +{ + constexpr std::size_t lane_count = 256 / (sizeof(element_t) * 8); + auto result = identity_selectors(); + for (std::size_t lane = 0; lane < lane_count; ++lane) + result[lane] = lane_count - 1 - lane; + return result; +} +/** + * @brief Expands one compile-time selector array into an independent scalar shuffle result. + * @tparam selectors Logical source-lane selector array. + * @tparam element_t Logical lane type. + * @tparam lane_count Number of source and result lanes. + * @tparam positions Output lane sequence. + * @param source Source lane array. + * @return Scalar-oracle result array. + */ +template +[[nodiscard]] constexpr std::array logical_shuffle_oracle_impl(const std::array &source, + std::index_sequence) noexcept +{ + return {source[selectors[positions]]...}; +} + +/** + * @brief Applies a compile-time logical selector sequence without using Api or Register code. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @tparam selectors One source-lane selector per output lane. + * @param source Source lane array. + * @return Independently computed scalar result. + */ +template +[[nodiscard]] constexpr auto logical_shuffle_oracle(const std::array &source) noexcept +{ + constexpr std::size_t lane_count = bits / (sizeof(element_t) * 8); + static_assert(selectors.size() == lane_count); + return logical_shuffle_oracle_impl(source, std::make_index_sequence{}); +} + +/** + * @brief Compares lane arrays by object representation. + * @tparam element_t Logical lane type. + * @tparam lane_count Number of compared lanes. + * @param lhs Left lane array. + * @param rhs Right lane array. + * @return True only when every corresponding lane contains identical bits. + */ +template +[[nodiscard]] constexpr bool same_object_representations(const std::array &lhs, const std::array &rhs) noexcept +{ + for (std::size_t lane = 0; lane < lane_count; ++lane) + if (std::bit_cast>(lhs[lane]) != std::bit_cast>(rhs[lane])) + return false; + return true; +} + +} // namespace SimdLib::Tests::LogicalShuffle diff --git a/tests/Register.tests.cpp b/tests/Register.tests.cpp new file mode 100644 index 0000000..17d5450 --- /dev/null +++ b/tests/Register.tests.cpp @@ -0,0 +1,320 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef SIMDLIB_REGISTER_TEST_ENABLE_256 +#define SIMDLIB_REGISTER_TEST_ENABLE_256 SIMDLIB_HAS_AVX2 +#endif + +namespace +{ + +/** @brief Creates distinctive, exactly representable values for every lane. */ +template [[nodiscard]] constexpr auto lane_values() noexcept +{ + std::array result{}; + for (std::size_t index = 0; index < result.size(); ++index) + result[index] = static_cast(index + 1); + if constexpr (std::is_integral_v) + { + result.front() = std::numeric_limits::lowest(); + result.back() = std::numeric_limits::max(); + } + else + { + result.front() = static_cast(-3.5); + result.back() = static_cast(7.25); + } + return result; +} + +/** @brief Verifies every compile-time-selected lane against its source value. */ +template +void require_all_lanes(const register_t value, const std::array &expected) +{ + if constexpr (index < register_t::lane_count) + { + REQUIRE(value.template lane() == expected[index]); + require_all_lanes(value, expected); + } +} + +/** @brief Constructs a register from an expanded low-to-high lane array. */ +template +[[nodiscard]] constexpr register_t from_lanes(const std::array &values, + std::index_sequence) noexcept +{ + return register_t::from_lanes(values[indices]...); +} + +/** @brief Verifies construction, observation, and lane replacement for one register type. */ +template void require_value_contracts() +{ + using register_type = SimdLib::Register; + const auto values = lane_values(); + const std::array zeros{}; + + REQUIRE(register_type{}.to_array() == zeros); + REQUIRE(register_type::zero().to_array() == zeros); + REQUIRE(register_type::broadcast(static_cast(7)).to_array() == + [] + { + std::array result{}; + result.fill(static_cast(7)); + return result; + }()); + REQUIRE(register_type::from_array(values).to_array() == values); + REQUIRE(from_lanes(values, std::make_index_sequence{}).to_array() == values); + + const register_type wrapped{register_type::api_type::construct(values)}; + REQUIRE(register_type::api_type::to_array(wrapped.native) == values); + require_all_lanes(wrapped, values); + + const auto first_replaced = wrapped.template with_lane<0>(static_cast(41)).to_array(); + const auto last_replaced = wrapped.template with_lane(static_cast(43)).to_array(); + for (std::size_t index = 0; index < values.size(); ++index) + { + REQUIRE(first_replaced[index] == (index == 0 ? static_cast(41) : values[index])); + REQUIRE(last_replaced[index] == (index + 1 == values.size() ? static_cast(43) : values[index])); + } +} + +/** @brief Verifies exact-width aligned, unaligned, and raw-byte transfers with canaries. */ +template void require_transfer_contracts() +{ + using register_type = SimdLib::Register; + const auto values = lane_values(); + + alignas(register_type::byte_count) std::array aligned_source = values; + alignas(register_type::byte_count) std::array aligned_destination{}; + register_type::load_aligned(std::span{aligned_source}) + .store_aligned(std::span{aligned_destination}); + REQUIRE(aligned_destination == values); + + alignas(register_type::byte_count) std::array unaligned_source{}; + alignas(register_type::byte_count) std::array unaligned_destination{}; + unaligned_source.front() = static_cast(91); + unaligned_source.back() = static_cast(93); + unaligned_destination.front() = static_cast(95); + unaligned_destination.back() = static_cast(97); + for (std::size_t index = 0; index < values.size(); ++index) + unaligned_source[index + 1] = values[index]; + const auto loaded = register_type::load(std::span{unaligned_source.data() + 1, register_type::lane_count}); + loaded.store(std::span{unaligned_destination.data() + 1, register_type::lane_count}); + REQUIRE(unaligned_destination.front() == static_cast(95)); + REQUIRE(unaligned_destination.back() == static_cast(97)); + for (std::size_t index = 0; index < values.size(); ++index) + REQUIRE(unaligned_destination[index + 1] == values[index]); + + std::array source_bytes{}; + for (std::size_t index = 0; index < source_bytes.size(); ++index) + source_bytes[index] = static_cast((index * 37U + 11U) & 0xFFU); + std::array destination_bytes{}; + destination_bytes.front() = std::byte{0xA5}; + destination_bytes.back() = std::byte{0x5A}; + register_type::load_bytes(std::span{source_bytes}) + .store_bytes(std::span{destination_bytes.data() + 1, register_type::byte_count}); + REQUIRE(std::to_integer(destination_bytes.front()) == 0xA5U); + REQUIRE(std::to_integer(destination_bytes.back()) == 0x5AU); + for (std::size_t index = 0; index < source_bytes.size(); ++index) + REQUIRE(std::to_integer(destination_bytes[index + 1]) == std::to_integer(source_bytes[index])); +} + +/** @brief Runs all Register value and transfer contracts for one scalar type. */ +template void require_type_contracts() +{ + require_value_contracts(); + require_transfer_contracts(); + if constexpr (SIMDLIB_REGISTER_TEST_ENABLE_256) + { + require_value_contracts(); + require_transfer_contracts(); + } +} + +/** @brief Returns a compact low-bit mask for one RegisterMask geometry. */ +template [[nodiscard]] constexpr typename mask_t::bits_type logical_bits() noexcept +{ + if constexpr (mask_t::lane_count == std::numeric_limits::digits) + return std::numeric_limits::max(); + else + return (typename mask_t::bits_type{1} << mask_t::lane_count) - 1; +} + +/** @brief Verifies canonical predicate bits, Boolean reductions, combination, and selection. */ +template void require_mask_contracts() +{ + using register_type = SimdLib::Register; + using mask_type = typename register_type::mask_type; + using bits_type = typename mask_type::bits_type; + constexpr bits_type all_bits = logical_bits(); + constexpr bits_type alternating_bits = []() constexpr noexcept + { + bits_type result = 0; + for (std::size_t index = 0; index < mask_type::lane_count; index += 2) + result |= bits_type{1} << index; + return result; + }(); + + std::array left{}; + std::array right{}; + for (std::size_t index = 0; index < left.size(); ++index) + { + left[index] = static_cast((index % 2) == 0 ? 2 : 0); + right[index] = static_cast(1); + } + const register_type lhs = register_type::from_array(left); + const register_type rhs = register_type::from_array(right); + const auto alternating = lhs.compare_greater(rhs); + const auto inverse = lhs.compare_less(rhs); + const auto all_true = lhs.compare_equal(lhs); + const mask_type rewrapped{alternating.native}; + + REQUIRE(mask_type{}.bits() == 0); + REQUIRE(mask_type{}.none()); + REQUIRE_FALSE(mask_type{}.any()); + REQUIRE_FALSE(mask_type{}.all()); + REQUIRE(alternating.bits() == alternating_bits); + REQUIRE(rewrapped.bits() == alternating_bits); + REQUIRE(alternating.any()); + REQUIRE_FALSE(alternating.all()); + REQUIRE(all_true.bits() == all_bits); + REQUIRE(all_true.all()); + REQUIRE((alternating | inverse).bits() == all_bits); + REQUIRE((alternating & inverse).none()); + REQUIRE((alternating ^ inverse).bits() == all_bits); + REQUIRE((~alternating).bits() == (all_bits ^ alternating_bits)); + + auto reassigned = alternating; + reassigned = reassigned & all_true; + REQUIRE(reassigned.bits() == alternating_bits); + reassigned = reassigned | inverse; + REQUIRE(reassigned.all()); + reassigned = reassigned ^ inverse; + REQUIRE(reassigned.bits() == alternating_bits); + + std::array first_left{}; + std::array first_right{}; + first_left.front() = static_cast(1); + first_right.back() = static_cast(1); + const auto first_only = register_type::from_array(first_left).compare_greater(register_type::zero()); + const auto highest_only = register_type::from_array(first_right).compare_greater(register_type::zero()); + REQUIRE(first_only.bits() == bits_type{1}); + REQUIRE(highest_only.bits() == (bits_type{1} << (register_type::lane_count - 1))); + REQUIRE((first_only | highest_only).bits() == (bits_type{1} | (bits_type{1} << (register_type::lane_count - 1)))); + REQUIRE(((first_only | highest_only).bits() & ~all_bits) == 0); + + const auto selected = + alternating.select(register_type::broadcast(static_cast(11)), register_type::broadcast(static_cast(22))).to_array(); + for (std::size_t index = 0; index < selected.size(); ++index) + REQUIRE(selected[index] == static_cast((index % 2) == 0 ? 11 : 22)); + + REQUIRE(lhs.compare_greater_equal(rhs).bits() == alternating_bits); + REQUIRE(lhs.compare_less_equal(rhs).bits() == (all_bits ^ alternating_bits)); + REQUIRE((lhs == lhs)); + REQUIRE_FALSE(lhs != lhs); + REQUIRE_FALSE(lhs == rhs); + REQUIRE(lhs != rhs); + + const auto native_lanes = register_type::api_type::to_array(alternating.native); + for (std::size_t lane = 0; lane < native_lanes.size(); ++lane) + { + const auto bytes = std::bit_cast>(native_lanes[lane]); + for (const auto byte : bytes) + REQUIRE(byte == ((lane % 2) == 0 ? 0xFFU : 0x00U)); + } +} + +/** @brief Verifies signed or unsigned high-bit ordering for one integer geometry. */ +template + requires std::is_integral_v +void require_integer_ordering() +{ + using register_type = SimdLib::Register; + const auto low = register_type::broadcast(std::numeric_limits::lowest()); + const auto high = register_type::broadcast(std::numeric_limits::max()); + REQUIRE(high.compare_greater(low).all()); + REQUIRE(low.compare_less(high).all()); +} + +/** @brief Verifies ordered floating comparison behavior for NaNs and signed zero. */ +template + requires std::is_floating_point_v +void require_floating_comparison_edges() +{ + using register_type = SimdLib::Register; + const auto nan = register_type::broadcast(std::numeric_limits::quiet_NaN()); + const auto one = register_type::broadcast(static_cast(1)); + REQUIRE(nan.compare_equal(nan).none()); + REQUIRE(nan.compare_greater(one).none()); + REQUIRE(nan.compare_greater_equal(one).none()); + REQUIRE(nan.compare_less(one).none()); + REQUIRE(nan.compare_less_equal(one).none()); + REQUIRE(nan != nan); + const auto positive_zero = register_type::broadcast(static_cast(0.0)); + const auto negative_zero = register_type::broadcast(static_cast(-0.0)); + REQUIRE(positive_zero.compare_equal(negative_zero).all()); + REQUIRE(positive_zero == negative_zero); +} + +/** @brief Runs all mask and comparison contracts for one scalar type. */ +template void require_mask_type_contracts() +{ + require_mask_contracts(); + if constexpr (std::is_integral_v) + { + require_integer_ordering(); + } + else + { + require_floating_comparison_edges(); + } + if constexpr (SIMDLIB_REGISTER_TEST_ENABLE_256) + { + require_mask_contracts(); + if constexpr (std::is_integral_v) + require_integer_ordering(); + else + require_floating_comparison_edges(); + } +} + +TEST_CASE("Register construction and exact-width transfers preserve every lane and surrounding canaries", "[simdlib][register][transfer]") +{ + require_type_contracts(); + require_type_contracts(); + require_type_contracts(); + require_type_contracts(); + require_type_contracts(); + require_type_contracts(); + require_type_contracts(); + require_type_contracts(); + require_type_contracts(); + require_type_contracts(); +} + +TEST_CASE("RegisterMask comparisons, reductions, combinations, and selection preserve lane semantics", "[simdlib][register][mask][comparison]") +{ + require_mask_type_contracts(); + require_mask_type_contracts(); + require_mask_type_contracts(); + require_mask_type_contracts(); + require_mask_type_contracts(); + require_mask_type_contracts(); + require_mask_type_contracts(); + require_mask_type_contracts(); + require_mask_type_contracts(); + require_mask_type_contracts(); +} + +} // namespace diff --git a/tests/RegisterBasicOperations.tests.cpp b/tests/RegisterBasicOperations.tests.cpp new file mode 100644 index 0000000..b246b36 --- /dev/null +++ b/tests/RegisterBasicOperations.tests.cpp @@ -0,0 +1,625 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef SIMDLIB_REGISTER_TEST_ENABLE_256 +#define SIMDLIB_REGISTER_TEST_ENABLE_256 SIMDLIB_HAS_AVX2 +#endif + +namespace +{ + +/** @brief Selects an unsigned integer capable of holding one element's complete bit pattern. */ +template > struct bit_integer; + +/** @brief Selects the corresponding unsigned representation for an integral element. */ +template struct bit_integer +{ + using type = std::make_unsigned_t; +}; + +/** @brief Selects a 32-bit representation for a floating-point element. */ +template <> struct bit_integer +{ + using type = std::uint32_t; +}; + +/** @brief Selects a 64-bit representation for a double-precision element. */ +template <> struct bit_integer +{ + using type = std::uint64_t; +}; + +/** @brief Unsigned integer type that preserves one element's complete bit pattern. */ +template using bit_integer_t = typename bit_integer::type; + +/** @brief Returns the object representation of one scalar value. */ +template [[nodiscard]] constexpr bit_integer_t scalar_bits(element_t value) noexcept +{ + return std::bit_cast>(value); +} + +/** @brief Constructs one scalar value from its complete object representation. */ +template [[nodiscard]] constexpr element_t scalar_from_bits(bit_integer_t value) noexcept +{ + return std::bit_cast(value); +} + +/** @brief Requires exact per-lane object-representation equality. */ +template +void require_bitwise_equal(const std::array &actual, const std::array &expected) +{ + for (std::size_t index = 0; index < count; ++index) + REQUIRE(scalar_bits(actual[index]) == scalar_bits(expected[index])); +} + +/** @brief Computes a wrapping scalar sum using unsigned representation arithmetic. */ +template [[nodiscard]] constexpr element_t wrapping_add(element_t lhs, element_t rhs) noexcept +{ + using bits_type = bit_integer_t; + return scalar_from_bits(static_cast(scalar_bits(lhs) + scalar_bits(rhs))); +} + +/** @brief Computes a wrapping scalar difference using unsigned representation arithmetic. */ +template [[nodiscard]] constexpr element_t wrapping_subtract(element_t lhs, element_t rhs) noexcept +{ + using bits_type = bit_integer_t; + return scalar_from_bits(static_cast(scalar_bits(lhs) - scalar_bits(rhs))); +} + +/** @brief Computes a wrapping scalar product using unsigned representation arithmetic. */ +template [[nodiscard]] constexpr element_t wrapping_multiply(element_t lhs, element_t rhs) noexcept +{ + using bits_type = bit_integer_t; + return scalar_from_bits(static_cast(scalar_bits(lhs) * scalar_bits(rhs))); +} + +/** @brief Computes a wrapping scalar negation using unsigned representation arithmetic. */ +template [[nodiscard]] constexpr element_t wrapping_negate(element_t value) noexcept +{ + using bits_type = bit_integer_t; + return scalar_from_bits(static_cast(bits_type{} - scalar_bits(value))); +} + +/** @brief Verifies integral arithmetic against both the Api path and independent scalar oracles. */ +template + requires std::is_integral_v +void require_integral_arithmetic() +{ + using register_type = SimdLib::Register; + using api_type = typename register_type::api_type; + std::array left{}; + std::array right{}; + for (std::size_t index = 0; index < left.size(); ++index) + { + if constexpr (std::is_signed_v) + { + switch (index % 6) + { + case 0: + left[index] = std::numeric_limits::lowest(); + right[index] = element_t{1}; + break; + case 1: + left[index] = std::numeric_limits::max(); + right[index] = element_t{-1}; + break; + case 2: + left[index] = element_t{-17}; + right[index] = element_t{2}; + break; + case 3: + left[index] = element_t{17}; + right[index] = element_t{-3}; + break; + case 4: + left[index] = element_t{-1}; + right[index] = element_t{7}; + break; + default: + left[index] = element_t{}; + right[index] = element_t{5}; + break; + } + } + else + { + using unsigned_type = bit_integer_t; + constexpr auto high_bit = static_cast(unsigned_type{1} << (std::numeric_limits::digits - 1)); + switch (index % 6) + { + case 0: + left[index] = std::numeric_limits::max(); + right[index] = element_t{1}; + break; + case 1: + left[index] = static_cast(high_bit); + right[index] = element_t{2}; + break; + case 2: + left[index] = static_cast(high_bit | unsigned_type{7}); + right[index] = element_t{3}; + break; + case 3: + left[index] = element_t{17}; + right[index] = element_t{5}; + break; + case 4: + left[index] = element_t{1}; + right[index] = element_t{7}; + break; + default: + left[index] = element_t{}; + right[index] = element_t{11}; + break; + } + } + } + + std::array sums{}; + std::array differences{}; + std::array products{}; + std::array quotients{}; + std::array remainders{}; + std::array negations{}; + for (std::size_t index = 0; index < left.size(); ++index) + { + sums[index] = wrapping_add(left[index], right[index]); + differences[index] = wrapping_subtract(left[index], right[index]); + products[index] = wrapping_multiply(left[index], right[index]); + quotients[index] = static_cast(left[index] / right[index]); + remainders[index] = static_cast(left[index] % right[index]); + negations[index] = wrapping_negate(left[index]); + } + + const register_type lhs = register_type::from_array(left); + const register_type rhs = register_type::from_array(right); + REQUIRE((lhs + rhs).to_array() == sums); + REQUIRE((lhs - rhs).to_array() == differences); + REQUIRE((lhs * rhs).to_array() == products); + REQUIRE((lhs / rhs).to_array() == quotients); + REQUIRE((lhs % rhs).to_array() == remainders); + REQUIRE((-lhs).to_array() == negations); + REQUIRE((lhs + rhs).to_array() == api_type::to_array(api_type::add(lhs.native, rhs.native))); + REQUIRE((lhs - rhs).to_array() == api_type::to_array(api_type::subtract(lhs.native, rhs.native))); + REQUIRE((lhs * rhs).to_array() == api_type::to_array(api_type::multiply(lhs.native, rhs.native))); + REQUIRE((lhs / rhs).to_array() == api_type::to_array(api_type::divide(lhs.native, rhs.native))); + REQUIRE((lhs % rhs).to_array() == api_type::to_array(api_type::modulus(lhs.native, rhs.native))); + REQUIRE((-lhs).to_array() == api_type::to_array(api_type::negate(lhs.native))); + + auto reassigned = lhs; + reassigned = reassigned + rhs; + REQUIRE(reassigned.to_array() == sums); + reassigned = lhs; + reassigned = reassigned - rhs; + REQUIRE(reassigned.to_array() == differences); + reassigned = lhs; + reassigned = reassigned * rhs; + REQUIRE(reassigned.to_array() == products); + reassigned = lhs; + reassigned = reassigned / rhs; + REQUIRE(reassigned.to_array() == quotients); + reassigned = lhs; + reassigned = reassigned % rhs; + REQUIRE(reassigned.to_array() == remainders); +} + +/** @brief Reports scalar floating equality while preserving NaN and signed-zero distinctions. */ +template [[nodiscard]] bool equivalent_floating(element_t actual, element_t expected) noexcept +{ + if (std::isnan(expected)) + return std::isnan(actual); + if (actual == element_t{} && expected == element_t{}) + return std::signbit(actual) == std::signbit(expected); + return actual == expected; +} + +/** @brief Requires floating arrays to match scalar-oracle values lane by lane. */ +template +void require_floating_equal(const std::array &actual, const std::array &expected) +{ + for (std::size_t index = 0; index < count; ++index) + REQUIRE(equivalent_floating(actual[index], expected[index])); +} + +/** @brief Verifies floating arithmetic, reassignment, infinities, NaNs, and signed zeros. */ +template + requires std::is_floating_point_v +void require_floating_arithmetic() +{ + using register_type = SimdLib::Register; + using api_type = typename register_type::api_type; + std::array left{}; + std::array right{}; + for (std::size_t index = 0; index < left.size(); ++index) + { + switch (index % 8) + { + case 0: + left[index] = element_t{0.0}; + right[index] = element_t{2.0}; + break; + case 1: + left[index] = element_t{-0.0}; + right[index] = element_t{-2.0}; + break; + case 2: + left[index] = std::numeric_limits::infinity(); + right[index] = element_t{2.0}; + break; + case 3: + left[index] = -std::numeric_limits::infinity(); + right[index] = element_t{2.0}; + break; + case 4: + left[index] = std::numeric_limits::quiet_NaN(); + right[index] = element_t{1.0}; + break; + case 5: + left[index] = std::numeric_limits::max() / element_t{2.0}; + right[index] = element_t{2.0}; + break; + case 6: + left[index] = element_t{-3.5}; + right[index] = element_t{-0.5}; + break; + default: + left[index] = element_t{7.25}; + right[index] = element_t{4.0}; + break; + } + } + + std::array sums{}; + std::array differences{}; + std::array products{}; + std::array quotients{}; + std::array negations{}; + for (std::size_t index = 0; index < left.size(); ++index) + { + sums[index] = left[index] + right[index]; + differences[index] = left[index] - right[index]; + products[index] = left[index] * right[index]; + quotients[index] = left[index] / right[index]; + negations[index] = element_t{} - left[index]; + } + + const register_type lhs = register_type::from_array(left); + const register_type rhs = register_type::from_array(right); + require_floating_equal((lhs + rhs).to_array(), sums); + require_floating_equal((lhs - rhs).to_array(), differences); + require_floating_equal((lhs * rhs).to_array(), products); + require_floating_equal((lhs / rhs).to_array(), quotients); + require_floating_equal((-lhs).to_array(), negations); + require_floating_equal((lhs + rhs).to_array(), api_type::to_array(api_type::add(lhs.native, rhs.native))); + require_floating_equal((lhs - rhs).to_array(), api_type::to_array(api_type::subtract(lhs.native, rhs.native))); + require_floating_equal((lhs * rhs).to_array(), api_type::to_array(api_type::multiply(lhs.native, rhs.native))); + require_floating_equal((lhs / rhs).to_array(), api_type::to_array(api_type::divide(lhs.native, rhs.native))); + require_floating_equal((-lhs).to_array(), api_type::to_array(api_type::negate(lhs.native))); + + auto reassigned = lhs; + reassigned = reassigned + rhs; + require_floating_equal(reassigned.to_array(), sums); + reassigned = lhs; + reassigned = reassigned - rhs; + require_floating_equal(reassigned.to_array(), differences); + reassigned = lhs; + reassigned = reassigned * rhs; + require_floating_equal(reassigned.to_array(), products); + reassigned = lhs; + reassigned = reassigned / rhs; + require_floating_equal(reassigned.to_array(), quotients); +} + +/** @brief Verifies bitwise operations and both scalar mask granularities for one geometry. */ +template void require_bitwise_operations() +{ + using register_type = SimdLib::Register; + using api_type = typename register_type::api_type; + using bits_type = bit_integer_t; + constexpr int element_bits = std::numeric_limits::digits; + std::array left{}; + std::array right{}; + std::array intersection{}; + std::array union_values{}; + std::array exclusive{}; + std::array complement{}; + std::array andnot_values{}; + std::uint32_t expected_movemask = 0; + std::uint32_t expected_lane_bits = 0; + for (std::size_t index = 0; index < left.size(); ++index) + { + const auto high_bit = static_cast(bits_type{1} << (element_bits - 1)); + const auto left_bits = static_cast((index % 2 == 0 ? high_bit : bits_type{}) | static_cast(index * 37U + 0x15U)); + const auto right_bits = static_cast((index % 3 == 0 ? high_bit : bits_type{}) | static_cast(index * 19U + 0x2AU)); + left[index] = scalar_from_bits(left_bits); + right[index] = scalar_from_bits(right_bits); + intersection[index] = scalar_from_bits(static_cast(left_bits & right_bits)); + union_values[index] = scalar_from_bits(static_cast(left_bits | right_bits)); + exclusive[index] = scalar_from_bits(static_cast(left_bits ^ right_bits)); + complement[index] = scalar_from_bits(static_cast(~left_bits)); + andnot_values[index] = scalar_from_bits(static_cast((~left_bits) & right_bits)); + if ((left_bits & high_bit) != 0) + expected_lane_bits |= std::uint32_t{1} << index; + const auto bytes = std::bit_cast>(left[index]); + for (std::size_t byte = 0; byte < bytes.size(); ++byte) + { + if ((bytes[byte] & 0x80U) != 0) + expected_movemask |= std::uint32_t{1} << (index * sizeof(element_t) + byte); + } + } + + const register_type lhs = register_type::from_array(left); + const register_type rhs = register_type::from_array(right); + require_bitwise_equal((lhs & rhs).to_array(), intersection); + require_bitwise_equal((lhs | rhs).to_array(), union_values); + require_bitwise_equal((lhs ^ rhs).to_array(), exclusive); + require_bitwise_equal((~lhs).to_array(), complement); + require_bitwise_equal(lhs.andnot(rhs).to_array(), andnot_values); + require_bitwise_equal((lhs & rhs).to_array(), api_type::to_array(api_type::bitwise_and(lhs.native, rhs.native))); + require_bitwise_equal((lhs | rhs).to_array(), api_type::to_array(api_type::bitwise_or(lhs.native, rhs.native))); + require_bitwise_equal((lhs ^ rhs).to_array(), api_type::to_array(api_type::bitwise_xor(lhs.native, rhs.native))); + require_bitwise_equal((~lhs).to_array(), api_type::to_array(api_type::bitwise_not(lhs.native))); + require_bitwise_equal(lhs.andnot(rhs).to_array(), api_type::to_array(api_type::bitwise_andnot(lhs.native, rhs.native))); + REQUIRE(lhs.movemask() == expected_movemask); + REQUIRE(lhs.lane_sign_bits() == expected_lane_bits); + REQUIRE(lhs.movemask() == api_type::movemask(lhs.native)); + REQUIRE(lhs.lane_sign_bits() == api_type::movemask_slim(lhs.native)); + + auto reassigned = lhs; + reassigned = reassigned & rhs; + require_bitwise_equal(reassigned.to_array(), intersection); + reassigned = lhs; + reassigned = reassigned | rhs; + require_bitwise_equal(reassigned.to_array(), union_values); + reassigned = lhs; + reassigned = reassigned ^ rhs; + require_bitwise_equal(reassigned.to_array(), exclusive); +} + +/** @brief Computes one scalar per-lane logical left shift with backend boundary semantics. */ +template [[nodiscard]] constexpr element_t logical_left(element_t value, int count) noexcept +{ + using bits_type = bit_integer_t; + constexpr int width = std::numeric_limits::digits; + if (count >= width) + return element_t{}; + return scalar_from_bits(static_cast(scalar_bits(value) << count)); +} + +/** @brief Computes one scalar per-lane logical right shift with backend boundary semantics. */ +template [[nodiscard]] constexpr element_t logical_right(element_t value, int count) noexcept +{ + using bits_type = bit_integer_t; + constexpr int width = std::numeric_limits::digits; + if (count >= width) + return element_t{}; + return scalar_from_bits(static_cast(scalar_bits(value) >> count)); +} + +/** @brief Computes one scalar arithmetic right shift without relying on signed C++ shift behavior. */ +template [[nodiscard]] constexpr element_t arithmetic_right(element_t value, int count) noexcept +{ + using bits_type = bit_integer_t; + constexpr int width = std::numeric_limits::digits; + if (count >= width) + count = width - 1; + const bits_type input = scalar_bits(value); + bits_type result = static_cast(input >> count); + const bits_type sign = static_cast(bits_type{1} << (width - 1)); + if (count > 0 && (input & sign) != 0) + result = static_cast(result | static_cast(~bits_type{}) << (width - count)); + return scalar_from_bits(result); +} + +/** @brief Verifies all per-lane shift boundaries and reassignment spellings for one integral geometry. */ +template + requires std::is_integral_v +void require_lane_shifts() +{ + using register_type = SimdLib::Register; + using api_type = typename register_type::api_type; + using bits_type = bit_integer_t; + constexpr int width = std::numeric_limits::digits; + std::array source{}; + for (std::size_t index = 0; index < source.size(); ++index) + { + const auto high = static_cast(bits_type{1} << (width - 1)); + source[index] = scalar_from_bits(static_cast(high | static_cast(index * 17U + 3U))); + } + const register_type value = register_type::from_array(source); + for (const int count : std::array{0, width - 1, width, width + 1}) + { + std::array expected_left{}; + std::array expected_logical{}; + std::array expected_operator_right{}; + for (std::size_t index = 0; index < source.size(); ++index) + { + expected_left[index] = logical_left(source[index], count); + expected_logical[index] = logical_right(source[index], count); + if constexpr (std::is_signed_v) + expected_operator_right[index] = arithmetic_right(source[index], count); + else + expected_operator_right[index] = expected_logical[index]; + } + REQUIRE((value << count).to_array() == expected_left); + REQUIRE(value.logical_shift_right(count).to_array() == expected_logical); + REQUIRE((value >> count).to_array() == expected_operator_right); + REQUIRE((value << count).to_array() == api_type::to_array(api_type::shift_left(value.native, count))); + REQUIRE(value.logical_shift_right(count).to_array() == api_type::to_array(api_type::shift_right(value.native, count))); + if constexpr (std::is_signed_v) + REQUIRE((value >> count).to_array() == api_type::to_array(api_type::shift_right_arithmetic(value.native, count))); + + auto reassigned = value; + reassigned = reassigned << count; + REQUIRE(reassigned.to_array() == expected_left); + reassigned = value; + reassigned = reassigned >> count; + REQUIRE(reassigned.to_array() == expected_operator_right); + } +} + +/** @brief Computes a complete-register left shift for two low-to-high 64-bit words. */ +[[nodiscard]] constexpr std::array whole_left(std::array value, int count) noexcept +{ + if (count <= 0) + return value; + if (count >= 128) + return {}; + if (count >= 64) + return {0, value[0] << (count - 64)}; + return {value[0] << count, static_cast((value[1] << count) | (value[0] >> (64 - count)))}; +} + +/** @brief Computes a complete-register right shift for two low-to-high 64-bit words. */ +[[nodiscard]] constexpr std::array whole_right(std::array value, int count) noexcept +{ + if (count <= 0) + return value; + if (count >= 128) + return {}; + if (count >= 64) + return {value[1] >> (count - 64), 0}; + return {static_cast((value[0] >> count) | (value[1] << (64 - count))), value[1] >> count}; +} + +/** @brief Verifies byte and whole-register shift boundaries for the supported 128-bit shape. */ +void require_complete_register_shifts() +{ + using byte_register = SimdLib::Register; + std::array bytes{}; + for (std::size_t index = 0; index < bytes.size(); ++index) + bytes[index] = static_cast(index + 1); + const byte_register byte_value = byte_register::from_array(bytes); + constexpr std::array counts{std::numeric_limits::lowest(), -17, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + std::numeric_limits::max()}; + for (const int count : counts) + { + std::array left{}; + std::array right{}; + if (count <= 0) + { + left = bytes; + right = bytes; + } + else if (count < 16) + { + for (std::size_t index = static_cast(count); index < bytes.size(); ++index) + left[index] = bytes[index - static_cast(count)]; + for (std::size_t index = 0; index + static_cast(count) < bytes.size(); ++index) + right[index] = bytes[index + static_cast(count)]; + } + REQUIRE(byte_value.shift_bytes_left_slow(count).to_array() == left); + REQUIRE(byte_value.shift_bytes_right_slow(count).to_array() == right); + } + + using word_register = SimdLib::Register; + constexpr std::array words{0x0123456789ABCDEFULL, 0xFEDCBA9876543210ULL}; + const word_register word_value = word_register::from_array(words); + constexpr std::array bit_counts{std::numeric_limits::lowest(), -1, 0, 1, 63, 64, 65, 127, 128, 129, std::numeric_limits::max()}; + for (const int count : bit_counts) + { + REQUIRE(word_value.shift_bits_left_slow(count).to_array() == whole_left(words, count)); + REQUIRE(word_value.shift_bits_right_slow(count).to_array() == whole_right(words, count)); + } + REQUIRE(word_value.template shift_bits_left<0>().to_array() == whole_left(words, 0)); + REQUIRE(word_value.template shift_bits_left<1>().to_array() == whole_left(words, 1)); + REQUIRE(word_value.template shift_bits_left<63>().to_array() == whole_left(words, 63)); + REQUIRE(word_value.template shift_bits_left<64>().to_array() == whole_left(words, 64)); + REQUIRE(word_value.template shift_bits_left<65>().to_array() == whole_left(words, 65)); + REQUIRE(word_value.template shift_bits_left<127>().to_array() == whole_left(words, 127)); + REQUIRE(word_value.template shift_bits_left<128>().to_array() == whole_left(words, 128)); + REQUIRE(word_value.template shift_bits_left<129>().to_array() == whole_left(words, 129)); + REQUIRE(word_value.template shift_bits_right<0>().to_array() == whole_right(words, 0)); + REQUIRE(word_value.template shift_bits_right<1>().to_array() == whole_right(words, 1)); + REQUIRE(word_value.template shift_bits_right<63>().to_array() == whole_right(words, 63)); + REQUIRE(word_value.template shift_bits_right<64>().to_array() == whole_right(words, 64)); + REQUIRE(word_value.template shift_bits_right<65>().to_array() == whole_right(words, 65)); + REQUIRE(word_value.template shift_bits_right<127>().to_array() == whole_right(words, 127)); + REQUIRE(word_value.template shift_bits_right<128>().to_array() == whole_right(words, 128)); + REQUIRE(word_value.template shift_bits_right<129>().to_array() == whole_right(words, 129)); +} + +/** @brief Runs arithmetic coverage at both supported register widths. */ +template void require_arithmetic_type() +{ + if constexpr (std::is_integral_v) + { + require_integral_arithmetic(); + if constexpr (SIMDLIB_REGISTER_TEST_ENABLE_256) + require_integral_arithmetic(); + } + else + { + require_floating_arithmetic(); + if constexpr (SIMDLIB_REGISTER_TEST_ENABLE_256) + require_floating_arithmetic(); + } +} + +/** @brief Runs bitwise coverage at both supported register widths. */ +template void require_bitwise_type() +{ + require_bitwise_operations(); + if constexpr (SIMDLIB_REGISTER_TEST_ENABLE_256) + require_bitwise_operations(); +} + +/** @brief Runs per-lane shift coverage at both supported register widths. */ +template void require_shift_type() +{ + require_lane_shifts(); + if constexpr (SIMDLIB_REGISTER_TEST_ENABLE_256) + require_lane_shifts(); +} + +TEST_CASE("Register arithmetic matches Api and independent scalar edge-case oracles", "[simdlib][register][arithmetic]") +{ + require_arithmetic_type(); + require_arithmetic_type(); + require_arithmetic_type(); + require_arithmetic_type(); + require_arithmetic_type(); + require_arithmetic_type(); + require_arithmetic_type(); + require_arithmetic_type(); + require_arithmetic_type(); + require_arithmetic_type(); +} + +TEST_CASE("Register bitwise operations and sign masks preserve exact bits", "[simdlib][register][bitwise][movemask]") +{ + require_bitwise_type(); + require_bitwise_type(); + require_bitwise_type(); + require_bitwise_type(); + require_bitwise_type(); + require_bitwise_type(); + require_bitwise_type(); + require_bitwise_type(); + require_bitwise_type(); + require_bitwise_type(); +} + +TEST_CASE("Register shifts match lane and complete-register boundary contracts", "[simdlib][register][shift]") +{ + require_shift_type(); + require_shift_type(); + require_shift_type(); + require_shift_type(); + require_shift_type(); + require_shift_type(); + require_shift_type(); + require_shift_type(); + require_complete_register_shifts(); +} + +} // namespace diff --git a/tests/RegisterOperationMatrix.tests.cpp b/tests/RegisterOperationMatrix.tests.cpp new file mode 100644 index 0000000..86e28a3 --- /dev/null +++ b/tests/RegisterOperationMatrix.tests.cpp @@ -0,0 +1,186 @@ +#include +#include + +#include +#include +#include + +#ifndef SIMDLIB_REGISTER_TEST_ENABLE_256 +#define SIMDLIB_REGISTER_TEST_ENABLE_256 SIMDLIB_HAS_AVX2 +#endif + +namespace +{ + +/** @brief Reports whether a Register exposes a complete identity logical shuffle. */ +template [[nodiscard]] consteval bool has_identity_shuffle(std::index_sequence) noexcept +{ + return SimdLib::IRegister::Shuffle; +} + +/** @brief Reports whether an Api exposes a complete identity logical shuffle. */ +template [[nodiscard]] consteval bool has_identity_api_shuffle(std::index_sequence) noexcept +{ + return SimdLib::IApi::Shuffle; +} + +/** @brief Reports whether a Register exposes a complete identity byte shuffle. */ +template [[nodiscard]] consteval bool has_identity_byte_shuffle(std::index_sequence) noexcept +{ + return SimdLib::IRegister::ShuffleBytes; +} + +/** @brief Audits every public Register and RegisterMask declaration for one supported element/width cell. */ +template [[nodiscard]] consteval bool has_complete_surface() noexcept +{ + using api_t = SimdLib::Api; + using register_t = SimdLib::Register; + using mask_t = typename register_t::mask_type; + + constexpr bool integral = std::is_integral_v; + constexpr bool signed_integral = integral && std::is_signed_v; + constexpr bool byte_and_bit_shifts = integral && bits == 128; + + constexpr bool register_core = + SimdLib::IRegister::Type && SimdLib::IRegister::Zero && SimdLib::IRegister::Broadcast && + SimdLib::IRegister::FromArray && SimdLib::IRegister::Load && SimdLib::IRegister::LoadAligned && + SimdLib::IRegister::LoadBytes && SimdLib::IRegister::Store && SimdLib::IRegister::StoreAligned && + SimdLib::IRegister::StoreBytes && SimdLib::IRegister::ToArray && SimdLib::IRegister::Lane && + SimdLib::IRegister::WithLane && !SimdLib::IRegister::Lane && + !SimdLib::IRegister::WithLane; + + constexpr bool arithmetic = + SimdLib::IRegister::Add == SimdLib::IApi::Add && SimdLib::IRegister::Subtract == SimdLib::IApi::Subtract && + SimdLib::IRegister::Multiply == SimdLib::IApi::Multiply && SimdLib::IRegister::Divide == SimdLib::IApi::Divide && + SimdLib::IRegister::Modulus == SimdLib::IApi::Modulus && SimdLib::IRegister::Negate == SimdLib::IApi::Negate; + + constexpr bool specialized = + SimdLib::IRegister::Min == SimdLib::IApi::Min && SimdLib::IRegister::Max == SimdLib::IApi::Max && + SimdLib::IRegister::Absolute == SimdLib::IApi::Absolute && SimdLib::IRegister::Sqrt == SimdLib::IApi::Sqrt && + SimdLib::IRegister::Average == SimdLib::IApi::Average && + SimdLib::IRegister::MultiplyAdd == SimdLib::IApi::MultiplyAdd && + SimdLib::IRegister::Magnitude == SimdLib::IApi::Magnitude && + SimdLib::IRegister::MagnitudeChecked == SimdLib::IApi::MagnitudeChecked && + SimdLib::IRegister::Normalize == SimdLib::IApi::Normalize && + SimdLib::IRegister::HorizontalAdd == SimdLib::IApi::HorizontalAdd && + SimdLib::IRegister::HorizontalSubtract == SimdLib::IApi::HorizontalSubtract && + SimdLib::IRegister::MultiplyAddAdjacent == SimdLib::IApi::MultiplyAddAdjacent && + SimdLib::IRegister::MultiplyAddUnsignedSignedBytes == SimdLib::IApi::ByteMultiplyAdd && + SimdLib::IRegister::SumAbsoluteByteDifferences == SimdLib::IApi::Sad && + SimdLib::IRegister::MultiSumAbsoluteByteDifferences == SimdLib::IApi::MultiSad && + SimdLib::IRegister::MinPosition == SimdLib::IApi::MinPosition && + SimdLib::IRegister::MaxPosition == SimdLib::IApi::MaxPosition && + SimdLib::IRegister::AddSaturated == SimdLib::IApi::AddSaturated && + SimdLib::IRegister::SubtractSaturated == SimdLib::IApi::SubtractSaturated && + SimdLib::IRegister::HorizontalAddSaturated == SimdLib::IApi::HorizontalAddSaturated && + SimdLib::IRegister::HorizontalSubtractSaturated == SimdLib::IApi::HorizontalSubtractSaturated && + SimdLib::IRegister::AddSubtract == SimdLib::IApi::AddSubtract && + SimdLib::IRegister::DotProduct == SimdLib::IApi::DotProduct && !SimdLib::IRegister::DotProduct && + !SimdLib::IRegister::DotProduct; + + constexpr bool bitwise_and_comparison = + SimdLib::IRegister::BitwiseAnd && SimdLib::IRegister::BitwiseOr && SimdLib::IRegister::BitwiseXor && + SimdLib::IRegister::BitwiseNot && SimdLib::IRegister::BitwiseAndNot && SimdLib::IRegister::Movemask && + SimdLib::IRegister::LaneSignBits && SimdLib::IRegister::CompareEqual && SimdLib::IRegister::CompareGreater && + SimdLib::IRegister::CompareGreaterEqual && SimdLib::IRegister::CompareLess && + SimdLib::IRegister::CompareLessEqual && SimdLib::IRegister::Equal && SimdLib::IRegister::NotEqual; + + constexpr bool register_mask = SimdLib::IRegisterMask::Type && SimdLib::IRegisterMask::Any && SimdLib::IRegisterMask::All && + SimdLib::IRegisterMask::None && SimdLib::IRegisterMask::Bits && SimdLib::IRegisterMask::Select && + SimdLib::IRegisterMask::BitwiseAnd && SimdLib::IRegisterMask::BitwiseOr && + SimdLib::IRegisterMask::BitwiseXor && SimdLib::IRegisterMask::BitwiseNot; + + constexpr bool shifts = + SimdLib::IRegister::ShiftLeft == SimdLib::IApi::ShiftLeft && + SimdLib::IRegister::LogicalShiftRight == SimdLib::IApi::ShiftRight && + SimdLib::IRegister::ShiftRight == (signed_integral ? SimdLib::IApi::ArithmeticShiftRight : SimdLib::IApi::ShiftRight) && + SimdLib::IRegister::ShiftBytesLeftSlow == byte_and_bit_shifts && + SimdLib::IRegister::ShiftBytesRightSlow == byte_and_bit_shifts && SimdLib::IRegister::ShiftBytesLeft == integral && + SimdLib::IRegister::ShiftBytesRight == integral && !SimdLib::IRegister::ShiftBytesLeft && + !SimdLib::IRegister::ShiftBytesRight && SimdLib::IRegister::ShiftBitsLeftSlow == byte_and_bit_shifts && + SimdLib::IRegister::ShiftBitsRightSlow == byte_and_bit_shifts && SimdLib::IRegister::ShiftBitsLeft == byte_and_bit_shifts && + SimdLib::IRegister::ShiftBitsRight == byte_and_bit_shifts && !SimdLib::IRegister::ShiftBitsLeft && + !SimdLib::IRegister::ShiftBitsRight; + + constexpr bool lower_half = SimdLib::IRegister::LowerHalf == (bits == 256 && SimdLib::IApi::LowerHalf); + constexpr bool unpack_low = SimdLib::IRegister::UnpackLow == SimdLib::IApi::UnpackLow; + constexpr bool unpack_high = SimdLib::IRegister::UnpackHigh == SimdLib::IApi::UnpackHigh; + constexpr bool logical_shuffle = has_identity_shuffle(std::make_index_sequence{}) == + has_identity_api_shuffle(std::make_index_sequence{}); + constexpr bool byte_shuffle = has_identity_byte_shuffle(std::make_index_sequence{}) == + has_identity_api_shuffle>(std::make_index_sequence{}); + constexpr bool shuffle_low = SimdLib::IRegister::ShuffleLow == SimdLib::IApi::ShuffleLow; + constexpr bool shuffle_high = SimdLib::IRegister::ShuffleHigh == SimdLib::IApi::ShuffleHigh; + constexpr bool blend = SimdLib::IRegister::Blend == SimdLib::IApi::Blend; + + static_assert(register_core); + static_assert(arithmetic); + static_assert(specialized); + static_assert(bitwise_and_comparison); + static_assert(register_mask); + static_assert(shifts); + static_assert(lower_half); + static_assert(unpack_low); + static_assert(unpack_high); + static_assert(logical_shuffle); + static_assert(byte_shuffle); + static_assert(shuffle_low); + static_assert(shuffle_high); + static_assert(blend); + return true; +} + +/** @brief Audits all ten supported element types for one register width. */ +template [[nodiscard]] consteval bool has_complete_surface_for_all_elements() noexcept +{ + return has_complete_surface() && has_complete_surface() && has_complete_surface() && + has_complete_surface() && has_complete_surface() && has_complete_surface() && + has_complete_surface() && has_complete_surface() && has_complete_surface() && + has_complete_surface(); +} + +static_assert(has_complete_surface_for_all_elements<128>()); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +static_assert(has_complete_surface_for_all_elements<256>()); +#endif + +/** @brief Audits full-width bit casts, numeric conversions, and widening destinations for one source cell. */ +template [[nodiscard]] consteval bool has_complete_conversion_surface() noexcept +{ + using api_t = SimdLib::Api; + using register_t = SimdLib::Register; + + const auto target_matches = []() consteval noexcept + { + constexpr bool common = SimdLib::IRegister::BitCast && + SimdLib::IRegister::Convert == SimdLib::IApi::Convert && + SimdLib::IRegister::WidenLow == SimdLib::IApi::Widen>; + if constexpr (SimdLib::is_api_available_v<256, target_t>) + return common && SimdLib::IRegister::WidenLow == SimdLib::IApi::Widen>; + else + return common && !SimdLib::IRegister::WidenLow; + }; + + return target_matches.template operator()() && target_matches.template operator()() && + target_matches.template operator()() && target_matches.template operator()() && + target_matches.template operator()() && target_matches.template operator()() && + target_matches.template operator()() && target_matches.template operator()() && + target_matches.template operator()() && target_matches.template operator()(); +} + +/** @brief Audits conversion destinations for all ten source element types at one register width. */ +template [[nodiscard]] consteval bool has_complete_conversion_surface_for_all_elements() noexcept +{ + return has_complete_conversion_surface() && has_complete_conversion_surface() && + has_complete_conversion_surface() && has_complete_conversion_surface() && + has_complete_conversion_surface() && has_complete_conversion_surface() && + has_complete_conversion_surface() && has_complete_conversion_surface() && + has_complete_conversion_surface() && has_complete_conversion_surface(); +} + +static_assert(has_complete_conversion_surface_for_all_elements<128>()); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +static_assert(has_complete_conversion_surface_for_all_elements<256>()); +#endif + +} // namespace diff --git a/tests/RegisterPreconditionFailure.tests.cpp b/tests/RegisterPreconditionFailure.tests.cpp new file mode 100644 index 0000000..fef7041 --- /dev/null +++ b/tests/RegisterPreconditionFailure.tests.cpp @@ -0,0 +1,78 @@ +#include + +#include +#include + +namespace +{ +/** @brief Unique marker emitted only by the Register precondition-failure harness. */ +inline constexpr char expected_register_precondition_failure_marker[] = "SIMDLIB_REGISTER_PRECONDITION_FAILURE_EXPECTED_61B4C2"; + +/** @brief Diagnostic process exit code used after an expected precondition failure. */ +inline constexpr int register_precondition_failure_exit_code = 74; + +/** + * @brief Terminates the isolated test process after proving a Register precondition fired. + * @param message Diagnostic supplied by the failed public precondition. + */ +[[noreturn]] void fail_register_precondition(const char *message) noexcept +{ + (void)message; + std::fputs(expected_register_precondition_failure_marker, stderr); + std::fputc('\n', stderr); + std::fflush(stderr); + std::exit(register_precondition_failure_exit_code); +} +} // namespace + +#define SIMDLIB_PRECONDITION(condition, message) \ + do \ + { \ + if (!(condition)) \ + fail_register_precondition(message); \ + } while (false) + +#include + +#undef SIMDLIB_PRECONDITION + +#include +#include +#include + +TEST_CASE("Register left shift rejects a negative per-lane count", "[simdlib][register][preconditions]") +{ + using register_type = SimdLib::Register; + (void)(register_type::broadcast(1U) << -1); + FAIL("Register left shift accepted a negative count"); +} + +TEST_CASE("Register logical right shift rejects a negative per-lane count", "[simdlib][register][preconditions]") +{ + using register_type = SimdLib::Register; + (void)register_type::broadcast(-1).logical_shift_right(-1); + FAIL("Register logical right shift accepted a negative count"); +} + +TEST_CASE("Register arithmetic right shift rejects a negative per-lane count", "[simdlib][register][preconditions]") +{ + using register_type = SimdLib::Register; + (void)(register_type::broadcast(-1) >> -1); + FAIL("Register arithmetic right shift accepted a negative count"); +} + +TEST_CASE("Register aligned load rejects a misaligned source", "[simdlib][register][preconditions]") +{ + using register_type = SimdLib::Register; + alignas(register_type::byte_count) std::array source{}; + (void)register_type::load_aligned(std::span{source.data() + 1, register_type::lane_count}); + FAIL("Register aligned load accepted a misaligned source"); +} + +TEST_CASE("Register aligned store rejects a misaligned destination", "[simdlib][register][preconditions]") +{ + using register_type = SimdLib::Register; + alignas(register_type::byte_count) std::array destination{}; + register_type::zero().store_aligned(std::span{destination.data() + 1, register_type::lane_count}); + FAIL("Register aligned store accepted a misaligned destination"); +} diff --git a/tests/RegisterRearrangementConversion.tests.cpp b/tests/RegisterRearrangementConversion.tests.cpp new file mode 100644 index 0000000..bafefe9 --- /dev/null +++ b/tests/RegisterRearrangementConversion.tests.cpp @@ -0,0 +1,298 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef SIMDLIB_REGISTER_TEST_ENABLE_256 +#define SIMDLIB_REGISTER_TEST_ENABLE_256 SIMDLIB_HAS_AVX2 +#endif + +namespace +{ + +/** @brief Builds distinctive logical lanes for one Register specialization. */ +template [[nodiscard]] constexpr std::array make_distinct_lanes() noexcept +{ + using element_t = typename register_t::element_type; + std::array result{}; + for (std::size_t lane = 0; lane < result.size(); ++lane) + { + if constexpr (std::is_floating_point_v) + result[lane] = static_cast(lane * 3 + 1) / static_cast(2); + else if constexpr (std::is_signed_v) + result[lane] = static_cast(lane % 2 == 0 ? static_cast(lane + 1) : -static_cast(lane + 1)); + else + result[lane] = static_cast(lane * 7 + 3); + } + return result; +} + +/** @brief Reports whether a Register accepts its complete logical identity selector list. */ +template [[nodiscard]] consteval bool has_complete_shuffle() noexcept +{ + return [](std::index_sequence) consteval + { return SimdLib::IRegister::Shuffle; }(std::make_index_sequence{}); +} + +/** @brief Verifies low/high unpack lane order independently of the Api implementation. */ +template void require_unpack_contract() +{ + using register_t = SimdLib::Register; + constexpr std::size_t lanes_per_group = 128 / (sizeof(element_t) * 8); + constexpr std::size_t lanes_per_half = lanes_per_group / 2; + const auto left = make_distinct_lanes(); + auto right = make_distinct_lanes(); + for (std::size_t lane = 0; lane < right.size(); ++lane) + right[lane] = static_cast(right[lane] + static_cast(37)); + + std::array expected_low{}; + std::array expected_high{}; + for (std::size_t group = 0; group < register_t::lane_count; group += lanes_per_group) + { + for (std::size_t lane = 0; lane < lanes_per_half; ++lane) + { + expected_low[group + lane * 2] = left[group + lane]; + expected_low[group + lane * 2 + 1] = right[group + lane]; + expected_high[group + lane * 2] = left[group + lanes_per_half + lane]; + expected_high[group + lane * 2 + 1] = right[group + lanes_per_half + lane]; + } + } + + const auto lhs = register_t::from_array(left); + const auto rhs = register_t::from_array(right); + REQUIRE(lhs.unpack_low(rhs).to_array() == expected_low); + REQUIRE(lhs.unpack_high(rhs).to_array() == expected_high); +} + +/** @brief Verifies intrinsic-compatible immediate blend selection for one Register type. */ +template void require_blend_contract() +{ + using register_t = SimdLib::Register; + const auto left = make_distinct_lanes(); + auto right = make_distinct_lanes(); + for (std::size_t lane = 0; lane < right.size(); ++lane) + right[lane] = static_cast(right[lane] + static_cast(53)); + std::array expected{}; + for (std::size_t lane = 0; lane < expected.size(); ++lane) + expected[lane] = (static_cast(immediate) & (1u << (lane % 8))) != 0 ? right[lane] : left[lane]; + + const auto actual = register_t::from_array(left).template blend(register_t::from_array(right)); + REQUIRE(actual.to_array() == expected); +} + +/** @brief Verifies the explicitly consumed source prefix for one widening shape. */ +template void require_widen_low_contract() +{ + using source_register = SimdLib::Register; + using target_register = SimdLib::Register; + auto source = make_distinct_lanes(); + source.front() = std::numeric_limits::min(); + source.back() = std::numeric_limits::max(); + std::array expected{}; + for (std::size_t lane = 0; lane < expected.size(); ++lane) + expected[lane] = static_cast(source[lane]); + + const auto widened = source_register::from_array(source).template widen_low(); + REQUIRE(widened.to_array() == expected); +} + +/** @brief Verifies all supported widening destinations for one signedness family. */ +template void require_widening_family() +{ + require_widen_low_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_widen_low_contract(); +#endif + require_widen_low_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_widen_low_contract(); +#endif + require_widen_low_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_widen_low_contract(); +#endif + require_widen_low_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_widen_low_contract(); +#endif + require_widen_low_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_widen_low_contract(); +#endif + require_widen_low_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_widen_low_contract(); +#endif +} + +using I8x128 = SimdLib::Register; +using U8x128 = SimdLib::Register; +using I16x128 = SimdLib::Register; +using U16x128 = SimdLib::Register; +using I32x128 = SimdLib::Register; +using U32x128 = SimdLib::Register; +using I64x128 = SimdLib::Register; +using U64x128 = SimdLib::Register; +using F32x128 = SimdLib::Register; +using F64x128 = SimdLib::Register; +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +using U8x256 = SimdLib::Register; +#endif + +static_assert(!SimdLib::IRegister::LowerHalf); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +static_assert(SimdLib::IRegister::LowerHalf>); +#endif +static_assert(SimdLib::IRegister::UnpackLow && SimdLib::IRegister::UnpackHigh); +static_assert(has_complete_shuffle()); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +static_assert(has_complete_shuffle()); +#endif +static_assert(has_complete_shuffle()); +static_assert(SimdLib::IRegister::ShuffleLow && SimdLib::IRegister::ShuffleHigh); +static_assert(!SimdLib::IRegister::ShuffleLow); +static_assert(SimdLib::IRegister::Blend && SimdLib::IRegister::Blend && SimdLib::IRegister::Blend && + SimdLib::IRegister::Blend); +static_assert(!SimdLib::IRegister::Blend && !SimdLib::IRegister::Blend); +static_assert(SimdLib::IRegister::BitCast && SimdLib::IRegister::BitCast); +static_assert(SimdLib::IRegister::Convert && SimdLib::IRegister::Convert && SimdLib::IRegister::Convert); +static_assert(!SimdLib::IRegister::Convert && !SimdLib::IRegister::Convert); +static_assert(SimdLib::IRegister::WidenLow); +static_assert(!SimdLib::IRegister::WidenLow); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +static_assert(SimdLib::IRegister::WidenLow && SimdLib::IRegister::WidenLow); +static_assert(!SimdLib::IRegister::WidenLow, std::int16_t, 256>); +#endif + +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +TEST_CASE("Register lower-half preserves the complete low 128-bit lane sequence", "[simdlib][register][rearrangement]") +{ + using register_t = SimdLib::Register; + const auto lanes = make_distinct_lanes(); + const auto actual = register_t::from_array(lanes).lower_half().to_array(); + REQUIRE(actual == std::array{lanes[0], lanes[1], lanes[2], lanes[3]}); +} +#endif + +TEST_CASE("Register unpack methods preserve intrinsic 128-bit grouping and lane order", "[simdlib][register][rearrangement]") +{ + require_unpack_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_unpack_contract(); + require_unpack_contract(); +#endif + require_unpack_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_unpack_contract(); + require_unpack_contract(); +#endif +} + +TEST_CASE("Register logical byte shuffle uses complete lane-local selector lists", "[simdlib][register][rearrangement]") +{ + using register128_t = SimdLib::Register; + const auto source128 = make_distinct_lanes(); + const auto reversed128 = register128_t::from_array(source128).template shuffle<15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0>().to_array(); + for (std::size_t lane = 0; lane < 16; ++lane) + REQUIRE(reversed128[lane] == source128[15 - lane]); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + using register256_t = SimdLib::Register; + const auto source256 = make_distinct_lanes(); + const auto reversed256 = + register256_t::from_array(source256) + .template shuffle<15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16>() + .to_array(); + for (std::size_t lane = 0; lane < 16; ++lane) + { + REQUIRE(reversed256[lane] == source256[15 - lane]); + REQUIRE(reversed256[16 + lane] == source256[31 - lane]); + } +#endif +} + +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +TEST_CASE("Register 16-bit half shuffles preserve the unselected half in every 128-bit group", "[simdlib][register][rearrangement]") +{ + using register_t = SimdLib::Register; + const auto source = make_distinct_lanes(); + const auto low = register_t::from_array(source).template shuffle_low<0x1B>().to_array(); + const auto high = register_t::from_array(source).template shuffle_high<0x1B>().to_array(); + for (std::size_t group = 0; group < source.size(); group += 8) + { + for (std::size_t lane = 0; lane < 4; ++lane) + { + REQUIRE(low[group + lane] == source[group + 3 - lane]); + REQUIRE(low[group + 4 + lane] == source[group + 4 + lane]); + REQUIRE(high[group + lane] == source[group + lane]); + REQUIRE(high[group + 4 + lane] == source[group + 7 - lane]); + } + } +} +#endif + +TEST_CASE("Register immediate blend retains operation-specific mask-bit behavior", "[simdlib][register][rearrangement]") +{ + require_blend_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_blend_contract(); +#endif + require_blend_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_blend_contract(); +#endif + require_blend_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_blend_contract(); +#endif + require_blend_contract(); +#if SIMDLIB_REGISTER_TEST_ENABLE_256 + require_blend_contract(); +#endif +} + +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +TEST_CASE("Register bit-cast preserves floating edge-value object representations", "[simdlib][register][conversion]") +{ + using bits_t = SimdLib::Register; + constexpr std::array patterns{0x00000000u, 0x80000000u, 0x3F800000u, 0xBF800000u, 0x7F800000u, 0xFF800000u, 0x7FC12345u, 0xFFC54321u}; + const auto floating = bits_t::from_array(patterns).template bit_cast(); + REQUIRE(floating.template bit_cast().to_array() == patterns); + REQUIRE(std::bit_cast(floating.template lane<6>()) == patterns[6]); +} +#endif + +TEST_CASE("Register numeric conversion is distinct from bit reinterpretation", "[simdlib][register][conversion]") +{ + using signed_t = SimdLib::Register; + using unsigned_t = SimdLib::Register; + using float_register = SimdLib::Register; + const auto signed_values = + signed_t::from_lanes(std::numeric_limits::min(), -16'777'217, 16'777'217, std::numeric_limits::max()); + const auto unsigned_values = unsigned_t::from_lanes(0u, 16'777'217u, 0x80000000u, 0xFFFFFFFFu); + const auto converted_signed = signed_values.template convert().to_array(); + const auto converted_unsigned = unsigned_values.template convert().to_array(); + for (std::size_t lane = 0; lane < 4; ++lane) + { + REQUIRE(converted_signed[lane] == static_cast(signed_values.to_array()[lane])); + REQUIRE(converted_unsigned[lane] == static_cast(unsigned_values.to_array()[lane])); + } + + const auto rounded = float_register::from_lanes(-2.5F, -1.5F, 2.5F, 3.5F).template convert().to_array(); + REQUIRE(rounded == std::array{-2, -2, 2, 4}); + REQUIRE(signed_values.template bit_cast().to_array() != converted_signed); +} + +TEST_CASE("Register widening consumes exactly the documented low source lanes", "[simdlib][register][conversion]") +{ + require_widening_family(); + require_widening_family(); +} + +} // namespace diff --git a/tests/RegisterSpecializedOperations.tests.cpp b/tests/RegisterSpecializedOperations.tests.cpp new file mode 100644 index 0000000..cadcfd3 --- /dev/null +++ b/tests/RegisterSpecializedOperations.tests.cpp @@ -0,0 +1,948 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef SIMDLIB_REGISTER_TEST_ENABLE_256 +#define SIMDLIB_REGISTER_TEST_ENABLE_256 SIMDLIB_HAS_AVX2 +#endif +#if SIMDLIB_REGISTER_TEST_ENABLE_256 +#define SIMDLIB_REGISTER_IF_256(...) __VA_ARGS__ +#else +#define SIMDLIB_REGISTER_IF_256(...) +#endif + +namespace +{ + +/** @brief Reports whether one constrained promoted-result alias is available. */ +template +concept has_multiply_add_adjacent_alias = requires { typename SimdLib::multiply_add_adjacent_result_t; }; +template +concept has_byte_multiply_add_alias = requires { typename SimdLib::byte_multiply_add_result_t; }; +template +concept has_sad_alias = requires { typename SimdLib::sad_result_t; }; +template +concept has_multi_sad_alias = requires { typename SimdLib::multi_sad_result_t; }; + +/** @brief Mirrors the proposal's adjacent multiply-add lane promotion mapping. */ +template +using expected_adjacent_element_t = std::conditional_t< + (sizeof(element_t) >= sizeof(std::int64_t)), element_t, + std::conditional_t, + std::conditional_t>, + std::conditional_t>>>; + +/** @brief Verifies availability parity and exact promoted result mappings for one source shape. */ +template consteval bool validate_specialized_surface() +{ + using register_t = SimdLib::Register; + using api_t = SimdLib::Api; + using other_element_t = std::conditional_t, std::uint8_t, std::int8_t>; + + static_assert(SimdLib::IRegister::Add == SimdLib::IApi::Add); + static_assert(SimdLib::IRegister::Subtract == SimdLib::IApi::Subtract); + static_assert(SimdLib::IRegister::Multiply == SimdLib::IApi::Multiply); + static_assert(SimdLib::IRegister::Divide == SimdLib::IApi::Divide); + static_assert(SimdLib::IRegister::Modulus == SimdLib::IApi::Modulus); + static_assert(SimdLib::IRegister::Negate == SimdLib::IApi::Negate); + static_assert(SimdLib::IRegister::Min == SimdLib::IApi::Min); + static_assert(SimdLib::IRegister::Max == SimdLib::IApi::Max); + static_assert(SimdLib::IRegister::Absolute == SimdLib::IApi::Absolute); + static_assert(SimdLib::IRegister::Sqrt == SimdLib::IApi::Sqrt); + static_assert(SimdLib::IRegister::Average == SimdLib::IApi::Average); + static_assert(SimdLib::IRegister::MultiplyAdd == SimdLib::IApi::MultiplyAdd); + static_assert(SimdLib::IRegister::Magnitude == SimdLib::IApi::Magnitude); + static_assert(SimdLib::IRegister::MagnitudeChecked == SimdLib::IApi::MagnitudeChecked); + static_assert(SimdLib::IRegister::Normalize == SimdLib::IApi::Normalize); + static_assert(SimdLib::IRegister::HorizontalAdd == SimdLib::IApi::HorizontalAdd); + static_assert(SimdLib::IRegister::HorizontalSubtract == SimdLib::IApi::HorizontalSubtract); + static_assert(SimdLib::IRegister::MinPosition == SimdLib::IApi::MinPosition); + static_assert(SimdLib::IRegister::MaxPosition == SimdLib::IApi::MaxPosition); + static_assert(SimdLib::IRegister::AddSaturated == SimdLib::IApi::AddSaturated); + static_assert(SimdLib::IRegister::SubtractSaturated == SimdLib::IApi::SubtractSaturated); + static_assert(SimdLib::IRegister::HorizontalAddSaturated == SimdLib::IApi::HorizontalAddSaturated); + static_assert(SimdLib::IRegister::HorizontalSubtractSaturated == SimdLib::IApi::HorizontalSubtractSaturated); + static_assert(SimdLib::IRegister::AddSubtract == SimdLib::IApi::AddSubtract); + static_assert(SimdLib::IRegister::DotProduct == SimdLib::IApi::DotProduct); + static_assert(SimdLib::IRegister::DotProduct == SimdLib::IApi::DotProduct); + static_assert(SimdLib::IRegister::DotProduct == SimdLib::IApi::DotProduct); + static_assert(SimdLib::IRegister::MultiSumAbsoluteByteDifferences == SimdLib::IApi::MultiSad); + static_assert(SimdLib::IRegister::MultiSumAbsoluteByteDifferences == SimdLib::IApi::MultiSad); + static_assert(!SimdLib::IRegister::DotProduct); + static_assert(!SimdLib::IRegister::DotProduct); + static_assert(!SimdLib::IRegister::MultiSumAbsoluteByteDifferences); + static_assert(!SimdLib::IRegister::MultiSumAbsoluteByteDifferences); + static_assert(!SimdLib::IRegister::MultiplyAddAdjacent); + static_assert(!SimdLib::IRegister::MultiplyAddUnsignedSignedBytes); + static_assert(!SimdLib::IRegister::SumAbsoluteByteDifferences); + static_assert(!SimdLib::IRegister::MultiSumAbsoluteByteDifferences); + + static_assert(has_multiply_add_adjacent_alias == (std::is_integral_v && SimdLib::IApi::MultiplyAddAdjacent)); + static_assert(has_byte_multiply_add_alias == (std::is_integral_v && SimdLib::IApi::ByteMultiplyAdd)); + static_assert(has_sad_alias == (std::is_integral_v && SimdLib::IApi::Sad)); + static_assert(has_multi_sad_alias == (std::is_integral_v && SimdLib::IApi::MultiSad)); + static_assert(SimdLib::IRegister::MultiplyAddAdjacent == SimdLib::IApi::MultiplyAddAdjacent); + static_assert(SimdLib::IRegister::MultiplyAddUnsignedSignedBytes == SimdLib::IApi::ByteMultiplyAdd); + static_assert(SimdLib::IRegister::SumAbsoluteByteDifferences == SimdLib::IApi::Sad); + static_assert(SimdLib::IRegister::MultiSumAbsoluteByteDifferences == SimdLib::IApi::MultiSad); + + if constexpr (has_multiply_add_adjacent_alias) + { + using result_t = SimdLib::multiply_add_adjacent_result_t; + static_assert(std::same_as, bits>>); + static_assert(std::same_as().multiply_add_adjacent(std::declval())), result_t>); + } + if constexpr (has_byte_multiply_add_alias) + { + using result_t = SimdLib::byte_multiply_add_result_t; + static_assert(std::same_as>); + static_assert(std::same_as().multiply_add_unsigned_signed_bytes(std::declval())), result_t>); + } + if constexpr (has_sad_alias) + { + using result_t = SimdLib::sad_result_t; + static_assert(std::same_as>); + static_assert(std::same_as().sum_absolute_byte_differences(std::declval())), result_t>); + } + if constexpr (has_multi_sad_alias) + { + using result_t = SimdLib::multi_sad_result_t; + static_assert(std::same_as>); + static_assert(std::same_as().template multi_sum_absolute_byte_differences<0>(std::declval())), result_t>); + static_assert( + std::same_as().template multi_sum_absolute_byte_differences<255>(std::declval())), result_t>); + } + return true; +} + +#define SIMDLIB_VALIDATE_SPECIALIZED_TYPE(type) \ + static_assert(validate_specialized_surface()); \ + SIMDLIB_REGISTER_IF_256(static_assert(validate_specialized_surface());) +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(std::int8_t); +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(std::uint8_t); +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(std::int16_t); +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(std::uint16_t); +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(std::int32_t); +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(std::uint32_t); +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(std::int64_t); +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(std::uint64_t); +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(float); +SIMDLIB_VALIDATE_SPECIALIZED_TYPE(double); +#undef SIMDLIB_VALIDATE_SPECIALIZED_TYPE + +/** @brief Returns the exact nearest integer square root of a bounded unsigned square sum. */ +constexpr std::uint64_t rounded_integer_sqrt(const std::uint64_t total, const std::uint64_t maximum) noexcept +{ + std::uint64_t low = 0; + std::uint64_t high = maximum; + while (low < high) + { + const std::uint64_t middle = low + (high - low + 1) / 2; + if (middle <= total / middle) + low = middle; + else + high = middle - 1; + } + return low < maximum && total > low * low + low ? low + 1 : low; +} + +/** @brief Converts one signed or unsigned integer lane to its exact unsigned magnitude. */ +template constexpr std::uint64_t unsigned_lane_magnitude(const element_t value) noexcept +{ + using unsigned_t = std::make_unsigned_t; + const unsigned_t bits = static_cast(value); + if constexpr (std::is_signed_v) + return value < 0 ? static_cast(static_cast(unsigned_t{0} - bits)) : static_cast(bits); + else + return static_cast(bits); +} + +/** @brief Compares checked Register magnitudes with an independent threshold-clamped scalar oracle. */ +template +void require_checked_magnitude_oracle(const std::array::lane_count> &input) +{ + using register_t = SimdLib::Register; + using unsigned_t = std::make_unsigned_t; + constexpr std::size_t groupLanes = 128 / (sizeof(element_t) * 8); + constexpr std::uint64_t maximum = static_cast(std::numeric_limits::max()); + constexpr std::uint64_t threshold = maximum * maximum + maximum + 1; + constexpr element_t overflowMask = std::bit_cast(static_cast(~unsigned_t{0})); + const auto actual = register_t::from_array(input).magnitude_checked().to_array(); + std::array diagnosticInput{}; + std::transform(input.begin(), input.end(), diagnosticInput.begin(), [](const element_t value) { return static_cast(value); }); + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * groupLanes; + std::uint64_t total = 0; + bool overflow = false; + for (std::size_t lane = 0; lane < groupLanes; ++lane) + { + const std::uint64_t magnitude = unsigned_lane_magnitude(input[base + lane]); + const std::uint64_t square = magnitude * magnitude; + if (square >= threshold - total) + { + overflow = true; + break; + } + total += square; + } + const std::uint64_t expectedMagnitude = overflow ? maximum : rounded_integer_sqrt(total, maximum); + CAPTURE(sizeof(element_t), bits, group, total, overflow); + CAPTURE(diagnosticInput); + REQUIRE(actual[base] == static_cast(expectedMagnitude)); + REQUIRE(actual[base + 1] == (overflow ? overflowMask : element_t{0})); + } +} + +/** @brief Verifies integer roots plus sparse fast and checked magnitude contracts for one Register shape. */ +template void require_integer_roots_and_magnitude() +{ + using register_t = SimdLib::Register; + using unsigned_t = std::make_unsigned_t; + constexpr std::size_t groupLanes = 128 / (sizeof(element_t) * 8); + constexpr element_t maximum = std::numeric_limits::max(); + constexpr element_t overflowMask = std::bit_cast(static_cast(~unsigned_t{0})); + std::array roots{}; + std::array squares{}; + std::array safeInput{}; + std::array boundaryInput{}; + std::array nearBoundaryInput{}; + std::array overflowInput{}; + std::array roundingDownInput{}; + std::array roundingUpInput{}; + for (std::size_t index = 0; index < roots.size(); ++index) + { + roots[index] = static_cast(index % 10); + squares[index] = static_cast(roots[index] * roots[index]); + overflowInput[index] = maximum; + } + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * groupLanes; + safeInput[base] = element_t{3}; + safeInput[base + 1] = element_t{4}; + boundaryInput[base] = maximum; + nearBoundaryInput[base] = maximum; + nearBoundaryInput[base + 1] = element_t{1}; + roundingDownInput[base] = element_t{1}; + roundingDownInput[base + 1] = element_t{1}; + roundingUpInput[base] = element_t{2}; + roundingUpInput[base + 1] = element_t{2}; + } + + REQUIRE(register_t::from_array(squares).sqrt().to_array() == roots); + const auto fast = register_t::from_array(safeInput).magnitude().to_array(); + const auto checkedSafe = register_t::from_array(safeInput).magnitude_checked().to_array(); + const auto checkedBoundary = register_t::from_array(boundaryInput).magnitude_checked().to_array(); + const auto checkedOverflow = register_t::from_array(overflowInput).magnitude_checked().to_array(); + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * groupLanes; + REQUIRE(fast[base] == element_t{5}); + REQUIRE(checkedSafe[base] == element_t{5}); + REQUIRE(checkedSafe[base + 1] == element_t{0}); + REQUIRE(checkedBoundary[base] == maximum); + REQUIRE(checkedBoundary[base + 1] == element_t{0}); + REQUIRE(checkedOverflow[base] == maximum); + REQUIRE(checkedOverflow[base + 1] == overflowMask); + } + + if constexpr (sizeof(element_t) <= 4) + { + require_checked_magnitude_oracle(safeInput); + require_checked_magnitude_oracle(boundaryInput); + require_checked_magnitude_oracle(nearBoundaryInput); + require_checked_magnitude_oracle(overflowInput); + require_checked_magnitude_oracle(roundingDownInput); + require_checked_magnitude_oracle(roundingUpInput); + for (std::uint64_t caseIndex = 0; caseIndex < 8; ++caseIndex) + { + std::array generated{}; + std::uint64_t state = 0x9E37'79B9'7F4A'7C15ULL ^ (caseIndex * 0xD1B5'4A32'D192'ED03ULL); + for (std::size_t lane = 0; lane < generated.size(); ++lane) + { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + unsigned_t laneBits = static_cast(state * 0x2545'F491'4F6C'DD1DULL); + if ((caseIndex & 1U) == 0) + { + const unsigned_t safeMaximum = static_cast(static_cast(maximum) / groupLanes); + laneBits = static_cast(laneBits % (safeMaximum + unsigned_t{1})); + if constexpr (std::is_signed_v) + if ((lane & 1U) != 0) + laneBits = unsigned_t{0} - laneBits; + } + generated[lane] = std::bit_cast(laneBits); + } + require_checked_magnitude_oracle(generated); + } + } + + if constexpr (std::is_signed_v) + { + std::array minimumInput{}; + for (std::size_t group = 0; group < bits / 128; ++group) + minimumInput[group * groupLanes] = std::numeric_limits::min(); + const auto checkedMinimum = register_t::from_array(minimumInput).magnitude_checked().to_array(); + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * groupLanes; + REQUIRE(checkedMinimum[base] == maximum); + REQUIRE(checkedMinimum[base + 1] == overflowMask); + } + if constexpr (sizeof(element_t) <= 4) + require_checked_magnitude_oracle(minimumInput); + } +} +/** @brief Returns one source value represented modulo the adjacent-result lane width. */ +template constexpr std::make_unsigned_t adjacent_operand_bits(const element_t value) noexcept +{ + return static_cast>(static_cast(value)); +} + +/** @brief Verifies promoted adjacent multiply-add lane order, signedness, padding, and modular overflow. */ +template void require_adjacent_multiply_add_contract() +{ + using source_register = SimdLib::Register; + using result_register = SimdLib::multiply_add_adjacent_result_t; + using result_t = typename result_register::element_type; + using unsigned_result_t = std::make_unsigned_t; + constexpr std::size_t sourceGroupLanes = 128 / (sizeof(element_t) * 8); + constexpr std::size_t resultGroupLanes = 128 / (sizeof(result_t) * 8); + std::array lhs{}; + std::array rhs{}; + std::array expectedBits{}; + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t sourceBase = group * sourceGroupLanes; + const element_t overflowValue = [] + { + if constexpr (std::is_signed_v) + return std::numeric_limits::lowest(); + else + return std::numeric_limits::max(); + }(); + std::fill_n(lhs.begin() + static_cast(sourceBase), sourceGroupLanes, overflowValue); + std::fill_n(rhs.begin() + static_cast(sourceBase), sourceGroupLanes, overflowValue); + if constexpr (std::is_signed_v && sourceGroupLanes >= 4) + { + lhs[sourceBase + 2] = element_t{-3}; + lhs[sourceBase + 3] = element_t{4}; + rhs[sourceBase + 2] = element_t{5}; + rhs[sourceBase + 3] = element_t{-6}; + } + for (std::size_t pair = 0; pair < sourceGroupLanes / 2; ++pair) + { + const std::size_t sourceIndex = sourceBase + pair * 2; + const std::size_t resultIndex = group * resultGroupLanes + pair; + const std::uint64_t lowProduct = static_cast(adjacent_operand_bits(lhs[sourceIndex])) * + static_cast(adjacent_operand_bits(rhs[sourceIndex])); + const std::uint64_t highProduct = static_cast(adjacent_operand_bits(lhs[sourceIndex + 1])) * + static_cast(adjacent_operand_bits(rhs[sourceIndex + 1])); + expectedBits[resultIndex] = static_cast(lowProduct + highProduct); + } + } + const auto actual = source_register::from_array(lhs).multiply_add_adjacent(source_register::from_array(rhs)).to_array(); + for (std::size_t index = 0; index < actual.size(); ++index) + REQUIRE(std::bit_cast(actual[index]) == expectedBits[index]); +} + +/** @brief Computes an independent MPSADBW oracle for one immediate and Register width. */ +template +std::array multi_sad_oracle(const std::array &lhs, const std::array &rhs) +{ + std::array expected{}; + for (std::size_t group = 0; group < bits / 128; ++group) + { + const unsigned control = (static_cast(imm8) >> (group * 3)) & 0x7U; + const std::size_t groupBase = group * 16; + const std::size_t lhsBase = groupBase + ((control >> 2) & 0x1U) * 4; + const std::size_t rhsBase = groupBase + (control & 0x3U) * 4; + for (std::size_t output = 0; output < 8; ++output) + { + for (std::size_t offset = 0; offset < 4; ++offset) + { + expected[group * 8 + output] += + static_cast(std::abs(static_cast(lhs[lhsBase + output + offset]) - static_cast(rhs[rhsBase + offset]))); + } + } + } + return expected; +} + +/** @brief Verifies one MPSADBW immediate against the independent byte-window oracle. */ +template void require_multi_sad_immediate() +{ + using bytes = SimdLib::Register; + std::array lhs{}; + std::array rhs{}; + for (std::size_t index = 0; index < lhs.size(); ++index) + { + lhs[index] = static_cast((index * 17 + 3) % 251); + rhs[index] = static_cast((index * 29 + 11) % 253); + } + const auto actual = bytes::from_array(lhs).template multi_sum_absolute_byte_differences(bytes::from_array(rhs)).to_array(); + REQUIRE(actual == multi_sad_oracle(lhs, rhs)); +} + +/** @brief Computes the intrinsic-selected dot-product result independently for one immediate. */ +template +std::array::lane_count> dot_product_oracle( + const std::array::lane_count> &lhs, + const std::array::lane_count> &rhs) +{ + using register_t = SimdLib::Register; + constexpr std::size_t groupLanes = 128 / (sizeof(element_t) * 8); + std::array expected{}; + for (std::size_t group = 0; group < bits / 128; ++group) + { + element_t total{}; + for (std::size_t lane = 0; lane < groupLanes; ++lane) + { + if ((imm8 & (1 << (lane + 4))) != 0) + total += lhs[group * groupLanes + lane] * rhs[group * groupLanes + lane]; + } + for (std::size_t lane = 0; lane < groupLanes; ++lane) + { + if ((imm8 & (1 << lane)) != 0) + expected[group * groupLanes + lane] = total; + } + } + return expected; +} + +/** @brief Verifies one dot-product immediate against an independent selection-and-reduction oracle. */ +template +void require_dot_product_immediate(const std::array::lane_count> &lhs, + const std::array::lane_count> &rhs) +{ + using register_t = SimdLib::Register; + const auto actual = register_t::from_array(lhs).template dot_product(register_t::from_array(rhs)).to_array(); + REQUIRE(actual == dot_product_oracle(lhs, rhs)); +} +/** @brief Verifies extrema and absolute-value lane semantics for one supported source shape. */ +template void require_extrema_and_absolute_contract() +{ + using register_t = SimdLib::Register; + std::array lhs{}; + std::array rhs{}; + std::array minima{}; + std::array maxima{}; + std::array absolutes{}; + for (std::size_t index = 0; index < lhs.size(); ++index) + { + if constexpr (std::is_unsigned_v) + lhs[index] = static_cast(index * 3 + 1); + else + lhs[index] = static_cast((index % 2 == 0 ? -1 : 1) * static_cast(index + 1)); + rhs[index] = static_cast(index + 2); + minima[index] = std::min(lhs[index], rhs[index]); + maxima[index] = std::max(lhs[index], rhs[index]); + if constexpr (std::is_unsigned_v) + absolutes[index] = lhs[index]; + else + absolutes[index] = static_cast(std::abs(lhs[index])); + } + const register_t left = register_t::from_array(lhs); + const register_t right = register_t::from_array(rhs); + REQUIRE(left.min(right).to_array() == minima); + REQUIRE(left.max(right).to_array() == maxima); + REQUIRE(left.absolute().to_array() == absolutes); + if constexpr (std::is_integral_v && std::is_signed_v) + { + for (const auto value : register_t::broadcast(std::numeric_limits::lowest()).absolute().to_array()) + REQUIRE(value == std::numeric_limits::lowest()); + } +} + +/** @brief Verifies rounded unsigned average semantics for one supported lane type. */ +template void require_average_contract() +{ + using register_t = SimdLib::Register; + std::array lhs{}; + std::array rhs{}; + std::array expected{}; + for (std::size_t index = 0; index < lhs.size(); ++index) + { + lhs[index] = static_cast(index + 1); + rhs[index] = static_cast(index + 4); + expected[index] = static_cast((static_cast(lhs[index]) + static_cast(rhs[index]) + 1U) / 2U); + } + REQUIRE(register_t::from_array(lhs).average(register_t::from_array(rhs)).to_array() == expected); +} + +/** @brief Verifies lane order and 128-bit grouping for one supported horizontal arithmetic type. */ +template void require_horizontal_contract() +{ + using register_t = SimdLib::Register; + constexpr std::size_t groupLanes = 128 / (sizeof(element_t) * 8); + std::array lhs{}; + std::array rhs{}; + std::array expectedAdd{}; + std::array expectedSubtract{}; + for (std::size_t index = 0; index < lhs.size(); ++index) + { + lhs[index] = static_cast(index + 2); + rhs[index] = static_cast(index + 20); + } + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * groupLanes; + const std::size_t half = groupLanes / 2; + for (std::size_t pair = 0; pair < half; ++pair) + { + expectedAdd[base + pair] = static_cast(lhs[base + pair * 2] + lhs[base + pair * 2 + 1]); + expectedSubtract[base + pair] = static_cast(lhs[base + pair * 2] - lhs[base + pair * 2 + 1]); + expectedAdd[base + half + pair] = static_cast(rhs[base + pair * 2] + rhs[base + pair * 2 + 1]); + expectedSubtract[base + half + pair] = static_cast(rhs[base + pair * 2] - rhs[base + pair * 2 + 1]); + } + } + const register_t left = register_t::from_array(lhs); + const register_t right = register_t::from_array(rhs); + REQUIRE(left.horizontal_add(right).to_array() == expectedAdd); + REQUIRE(left.horizontal_subtract(right).to_array() == expectedSubtract); +} +/** @brief Verifies extrema, absolute value, square root, average, and multiply-add behavior. */ +template void require_lane_specialized_arithmetic() +{ + using integers = SimdLib::Register; + std::array lhsValues{}; + std::array rhsValues{}; + for (std::size_t index = 0; index < lhsValues.size(); ++index) + { + lhsValues[index] = static_cast((index % 2 == 0 ? -1 : 1) * static_cast(index + 2)); + rhsValues[index] = static_cast(5 - static_cast(index)); + } + const integers lhs = integers::from_array(lhsValues); + const integers rhs = integers::from_array(rhsValues); + std::array minima{}; + std::array maxima{}; + std::array absolutes{}; + for (std::size_t index = 0; index < lhsValues.size(); ++index) + { + minima[index] = std::min(lhsValues[index], rhsValues[index]); + maxima[index] = std::max(lhsValues[index], rhsValues[index]); + absolutes[index] = + lhsValues[index] == std::numeric_limits::lowest() ? lhsValues[index] : static_cast(std::abs(lhsValues[index])); + } + REQUIRE(lhs.min(rhs).to_array() == minima); + REQUIRE(lhs.max(rhs).to_array() == maxima); + REQUIRE(lhs.absolute().to_array() == absolutes); + + using bytes = SimdLib::Register; + const auto averaged = bytes::broadcast(2).average(bytes::broadcast(7)).to_array(); + for (const auto value : averaged) + REQUIRE(value == 5); + + using floats = SimdLib::Register; + std::array squareValues{}; + std::array rootValues{}; + for (std::size_t index = 0; index < squareValues.size(); ++index) + { + rootValues[index] = static_cast(index + 1); + squareValues[index] = rootValues[index] * rootValues[index]; + } + REQUIRE(floats::from_array(squareValues).sqrt().to_array() == rootValues); + const auto multiplyAdded = floats::broadcast(2.0F).multiply_add(floats::broadcast(3.0F), floats::broadcast(4.0F)).to_array(); + for (const auto value : multiplyAdded) + REQUIRE(value == 10.0F); +} + +/** @brief Verifies 128-bit grouping for magnitude, normalization, and horizontal operations. */ +template void require_grouped_operations() +{ + using floats = SimdLib::Register; + std::array values{}; + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * 4; + values[base] = group == 0 ? 3.0F : 5.0F; + values[base + 1] = group == 0 ? 4.0F : 12.0F; + } + const auto magnitude = floats::from_array(values).magnitude().to_array(); + const auto normalized = floats::from_array(values).normalize().to_array(); + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * 4; + const float expectedMagnitude = group == 0 ? 5.0F : 13.0F; + for (std::size_t offset = 0; offset < 4; ++offset) + REQUIRE(magnitude[base + offset] == expectedMagnitude); + REQUIRE(std::abs(normalized[base] - values[base] / expectedMagnitude) < 0.0001F); + REQUIRE(std::abs(normalized[base + 1] - values[base + 1] / expectedMagnitude) < 0.0001F); + } + + using integers = SimdLib::Register; + std::array lhs{}; + std::array rhs{}; + std::array expectedAdd{}; + std::array expectedSubtract{}; + for (std::size_t index = 0; index < lhs.size(); ++index) + { + lhs[index] = static_cast(index + 1); + rhs[index] = static_cast(20 + index); + } + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * 4; + expectedAdd[base] = lhs[base] + lhs[base + 1]; + expectedAdd[base + 1] = lhs[base + 2] + lhs[base + 3]; + expectedAdd[base + 2] = rhs[base] + rhs[base + 1]; + expectedAdd[base + 3] = rhs[base + 2] + rhs[base + 3]; + expectedSubtract[base] = lhs[base] - lhs[base + 1]; + expectedSubtract[base + 1] = lhs[base + 2] - lhs[base + 3]; + expectedSubtract[base + 2] = rhs[base] - rhs[base + 1]; + expectedSubtract[base + 3] = rhs[base + 2] - rhs[base + 3]; + } + const integers left = integers::from_array(lhs); + const integers right = integers::from_array(rhs); + REQUIRE(left.horizontal_add(right).to_array() == expectedAdd); + REQUIRE(left.horizontal_subtract(right).to_array() == expectedSubtract); +} + +/** @brief Verifies first-tie positions and unique highest-lane extrema for one integral shape. */ +template void require_position_contract() +{ + using register_t = SimdLib::Register; + CAPTURE(bits, sizeof(element_t), std::is_signed_v); + constexpr std::size_t minimumTiePosition = register_t::lane_count > 2 ? 1 : 0; + constexpr std::size_t maximumTiePosition = register_t::lane_count > 2 ? 2 : 0; + std::array values{}; + values.fill(element_t{5}); + values[minimumTiePosition] = element_t{1}; + values.back() = element_t{1}; + REQUIRE(register_t::from_array(values).min_position() == minimumTiePosition); + values.fill(element_t{5}); + values[maximumTiePosition] = element_t{9}; + values.back() = element_t{9}; + REQUIRE(register_t::from_array(values).max_position() == maximumTiePosition); + values.fill(element_t{5}); + values.back() = element_t{1}; + REQUIRE(register_t::from_array(values).min_position() == register_t::lane_count - 1); + values.fill(element_t{5}); + values.back() = element_t{9}; + REQUIRE(register_t::from_array(values).max_position() == register_t::lane_count - 1); +} + +/** @brief Verifies lane saturation and signed horizontal saturation. */ +template void require_saturation_contract() +{ + using signed_bytes = SimdLib::Register; + using unsigned_bytes = SimdLib::Register; + using signed_words = SimdLib::Register; + using unsigned_words = SimdLib::Register; + for (const auto value : signed_bytes::broadcast(120).add_saturated(signed_bytes::broadcast(20)).to_array()) + REQUIRE(value == std::numeric_limits::max()); + for (const auto value : unsigned_bytes::broadcast(3).subtract_saturated(unsigned_bytes::broadcast(9)).to_array()) + REQUIRE(value == 0); + for (const auto value : signed_words::broadcast(-30'000).subtract_saturated(signed_words::broadcast(10'000)).to_array()) + REQUIRE(value == std::numeric_limits::lowest()); + for (const auto value : unsigned_words::broadcast(65'000).add_saturated(unsigned_words::broadcast(1'000)).to_array()) + REQUIRE(value == std::numeric_limits::max()); + + std::array left{}; + std::array right{}; + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * 8; + left[base] = 30'000; + left[base + 1] = 10'000; + left[base + 2] = -30'000; + left[base + 3] = -10'000; + right[base] = 30'000; + right[base + 1] = -10'000; + } + const auto added = signed_words::from_array(left).horizontal_add_saturated(signed_words::from_array(right)).to_array(); + const auto subtracted = signed_words::from_array(left).horizontal_subtract_saturated(signed_words::from_array(right)).to_array(); + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * 8; + REQUIRE(added[base] == std::numeric_limits::max()); + REQUIRE(added[base + 1] == std::numeric_limits::lowest()); + REQUIRE(subtracted[base] == 20'000); + REQUIRE(subtracted[base + 1] == -20'000); + } +} + +/** @brief Returns the independently computed unsigned 16-bit saturated sum. */ +[[nodiscard]] constexpr std::uint16_t saturated_add_u16(std::uint16_t lhs, std::uint16_t rhs) noexcept +{ + const auto sum = static_cast(lhs) + static_cast(rhs); + return static_cast(std::min(sum, static_cast(std::numeric_limits::max()))); +} + +/** @brief Returns the independently computed unsigned 16-bit saturated difference. */ +[[nodiscard]] constexpr std::uint16_t saturated_subtract_u16(std::uint16_t lhs, std::uint16_t rhs) noexcept +{ + return lhs < rhs ? std::uint16_t{0} : static_cast(lhs - rhs); +} + +/** @brief Verifies every unsigned horizontal saturation lane against independent scalar edge-case oracles. */ +template void require_unsigned_horizontal_saturation_contract() +{ + using register_t = SimdLib::Register; + using pair_t = std::array; + constexpr std::array pairCases{ + pair_t{0, 0}, pair_t{0, 1}, pair_t{1, 0}, pair_t{1, 1}, pair_t{1, 2}, pair_t{2, 1}, + pair_t{32'767, 32'768}, pair_t{32'768, 32'767}, pair_t{32'768, 32'768}, pair_t{65'535, 0}, pair_t{0, 65'535}, pair_t{65'535, 1}, + pair_t{1, 65'535}, pair_t{65'535, 65'535}, pair_t{40'000, 25'535}, pair_t{40'000, 25'536}, pair_t{12'345, 54'321}, pair_t{54'321, 12'345}, + }; + + for (std::size_t rotation = 0; rotation < pairCases.size(); ++rotation) + { + std::array lhs{}; + std::array rhs{}; + std::array expectedAdd{}; + std::array expectedSubtract{}; + std::size_t caseIndex = rotation; + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * 8; + for (std::size_t pair = 0; pair < 4; ++pair) + { + const auto &values = pairCases[caseIndex++ % pairCases.size()]; + lhs[base + pair * 2] = values[0]; + lhs[base + pair * 2 + 1] = values[1]; + expectedAdd[base + pair] = saturated_add_u16(values[0], values[1]); + expectedSubtract[base + pair] = saturated_subtract_u16(values[0], values[1]); + } + for (std::size_t pair = 0; pair < 4; ++pair) + { + const auto &values = pairCases[caseIndex++ % pairCases.size()]; + rhs[base + pair * 2] = values[0]; + rhs[base + pair * 2 + 1] = values[1]; + expectedAdd[base + 4 + pair] = saturated_add_u16(values[0], values[1]); + expectedSubtract[base + 4 + pair] = saturated_subtract_u16(values[0], values[1]); + } + } + + CAPTURE(bits, rotation); + const auto lhsRegister = register_t::from_array(lhs); + const auto rhsRegister = register_t::from_array(rhs); + REQUIRE(lhsRegister.horizontal_add_saturated(rhsRegister).to_array() == expectedAdd); + REQUIRE(lhsRegister.horizontal_subtract_saturated(rhsRegister).to_array() == expectedSubtract); + } +} + +/** @brief Verifies promoted multiply-add and byte-difference result grouping. */ +template void require_promoted_results() +{ + using words = SimdLib::Register; + using dwords = SimdLib::multiply_add_adjacent_result_t; + std::array lhs{}; + std::array rhs{}; + std::array expected{}; + for (std::size_t index = 0; index < lhs.size(); ++index) + { + lhs[index] = static_cast(index + 1); + rhs[index] = static_cast((index % 3) + 2); + } + for (std::size_t index = 0; index < expected.size(); ++index) + expected[index] = static_cast(lhs[index * 2]) * rhs[index * 2] + static_cast(lhs[index * 2 + 1]) * rhs[index * 2 + 1]; + REQUIRE(words::from_array(lhs).multiply_add_adjacent(words::from_array(rhs)).to_array() == expected); + + using bytes = SimdLib::Register; + std::array unsignedBytes{}; + std::array signedBytes{}; + std::array::lane_count> maddExpected{}; + for (std::size_t index = 0; index < unsignedBytes.size(); ++index) + { + unsignedBytes[index] = static_cast((index % 5) + 1); + signedBytes[index] = static_cast(static_cast((index % 2 == 0) ? -3 : 4)); + } + for (std::size_t index = 0; index < maddExpected.size(); ++index) + maddExpected[index] = static_cast(static_cast(unsignedBytes[index * 2]) * static_cast(signedBytes[index * 2]) + + static_cast(unsignedBytes[index * 2 + 1]) * static_cast(signedBytes[index * 2 + 1])); + const bytes byteLhs = bytes::from_array(unsignedBytes); + const bytes byteRhs = bytes::from_array(signedBytes); + REQUIRE(byteLhs.multiply_add_unsigned_signed_bytes(byteRhs).to_array() == maddExpected); + + std::array saturatedLhs{}; + std::array saturatedRhs{}; + std::array::lane_count> saturatedExpected{}; + for (std::size_t index = 0; index < saturatedExpected.size(); ++index) + { + saturatedLhs[index * 2] = std::numeric_limits::max(); + saturatedLhs[index * 2 + 1] = std::numeric_limits::max(); + const std::int8_t signedFactor = index % 2 == 0 ? std::numeric_limits::max() : std::numeric_limits::lowest(); + saturatedRhs[index * 2] = static_cast(signedFactor); + saturatedRhs[index * 2 + 1] = static_cast(signedFactor); + saturatedExpected[index] = index % 2 == 0 ? std::numeric_limits::max() : std::numeric_limits::lowest(); + } + REQUIRE(bytes::from_array(saturatedLhs).multiply_add_unsigned_signed_bytes(bytes::from_array(saturatedRhs)).to_array() == saturatedExpected); + + std::array::lane_count> sadExpected{}; + for (std::size_t block = 0; block < sadExpected.size(); ++block) + for (std::size_t offset = 0; offset < 8; ++offset) + sadExpected[block] += + static_cast(std::abs(static_cast(unsignedBytes[block * 8 + offset]) - static_cast(signedBytes[block * 8 + offset]))); + REQUIRE(byteLhs.sum_absolute_byte_differences(byteRhs).to_array() == sadExpected); + + require_multi_sad_immediate<0, bits>(); + require_multi_sad_immediate<0x1B, bits>(); + require_multi_sad_immediate<0x3F, bits>(); + require_multi_sad_immediate<255, bits>(); +} + +/** @brief Verifies alternating floating arithmetic and immediate-controlled dot-product output lanes. */ +template void require_floating_specialized_operations() +{ + using register_t = SimdLib::Register; + constexpr std::size_t groupLanes = 128 / (sizeof(element_t) * 8); + std::array lhs{}; + std::array rhs{}; + for (std::size_t index = 0; index < rhs.size(); ++index) + { + lhs[index] = static_cast(index % 4 + 1); + rhs[index] = static_cast(index % 5 + 2); + } + const auto alternating = register_t::broadcast(element_t{10}).add_subtract(register_t::from_array(rhs)).to_array(); + for (std::size_t index = 0; index < alternating.size(); ++index) + REQUIRE(alternating[index] == (index % 2 == 0 ? element_t{10} - rhs[index] : element_t{10} + rhs[index])); + + std::array squares{}; + std::array roots{}; + std::array magnitudeInput{}; + for (std::size_t index = 0; index < squares.size(); ++index) + { + roots[index] = static_cast(index % groupLanes + 1); + squares[index] = roots[index] * roots[index]; + } + for (std::size_t group = 0; group < bits / 128; ++group) + { + magnitudeInput[group * groupLanes] = element_t{3}; + magnitudeInput[group * groupLanes + 1] = element_t{4}; + } + REQUIRE(register_t::from_array(squares).sqrt().to_array() == roots); + const auto magnitudes = register_t::from_array(magnitudeInput).magnitude().to_array(); + const auto normalized = register_t::from_array(magnitudeInput).normalize().to_array(); + for (std::size_t group = 0; group < bits / 128; ++group) + { + const std::size_t base = group * groupLanes; + for (std::size_t lane = 0; lane < groupLanes; ++lane) + REQUIRE(magnitudes[base + lane] == element_t{5}); + REQUIRE(std::abs(normalized[base] - static_cast(0.6)) < static_cast(0.0001)); + REQUIRE(std::abs(normalized[base + 1] - static_cast(0.8)) < static_cast(0.0001)); + } + for (const auto value : + register_t::broadcast(element_t{2}).multiply_add(register_t::broadcast(element_t{3}), register_t::broadcast(element_t{4})).to_array()) + REQUIRE(value == element_t{10}); + require_dot_product_immediate<0, element_t, bits>(lhs, rhs); + require_dot_product_immediate<0x11, element_t, bits>(lhs, rhs); + require_dot_product_immediate<0xD3, element_t, bits>(lhs, rhs); + require_dot_product_immediate<255, element_t, bits>(lhs, rhs); +} + +TEST_CASE("Register specialized lane arithmetic follows scalar semantics", "[simdlib][register][specialized][arithmetic]") +{ + require_lane_specialized_arithmetic<128>(); + require_grouped_operations<128>(); + SIMDLIB_REGISTER_IF_256(require_lane_specialized_arithmetic<256>();) + SIMDLIB_REGISTER_IF_256(require_grouped_operations<256>();) +#define SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(type) \ + require_extrema_and_absolute_contract(); \ + SIMDLIB_REGISTER_IF_256(require_extrema_and_absolute_contract();) + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(std::int8_t); + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(std::uint8_t); + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(std::int16_t); + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(std::uint16_t); + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(std::int32_t); + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(std::uint32_t); + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(std::int64_t); + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(std::uint64_t); + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(float); + SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE(double); +#undef SIMDLIB_REQUIRE_EXTREMA_AND_ABSOLUTE + require_average_contract(); + require_average_contract(); + SIMDLIB_REGISTER_IF_256(require_average_contract();) + SIMDLIB_REGISTER_IF_256(require_average_contract();) +#define SIMDLIB_REQUIRE_HORIZONTAL(type) \ + require_horizontal_contract(); \ + SIMDLIB_REGISTER_IF_256(require_horizontal_contract();) + SIMDLIB_REQUIRE_HORIZONTAL(std::int16_t); + SIMDLIB_REQUIRE_HORIZONTAL(std::uint16_t); + SIMDLIB_REQUIRE_HORIZONTAL(std::int32_t); + SIMDLIB_REQUIRE_HORIZONTAL(std::uint32_t); + SIMDLIB_REQUIRE_HORIZONTAL(float); + SIMDLIB_REQUIRE_HORIZONTAL(double); +#undef SIMDLIB_REQUIRE_HORIZONTAL +#define SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE(type) \ + require_integer_roots_and_magnitude(); \ + SIMDLIB_REGISTER_IF_256(require_integer_roots_and_magnitude();) + SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE(std::int8_t); + SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE(std::uint8_t); + SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE(std::int16_t); + SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE(std::uint16_t); + SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE(std::int32_t); + SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE(std::uint32_t); + SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE(std::int64_t); + SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE(std::uint64_t); +#undef SIMDLIB_REQUIRE_INTEGER_ROOTS_AND_MAGNITUDE +} + +TEST_CASE("Register positions cover first ties and the highest lane", "[simdlib][register][specialized][position]") +{ +#define SIMDLIB_REQUIRE_POSITIONS(type) \ + require_position_contract(); \ + SIMDLIB_REGISTER_IF_256(require_position_contract();) + SIMDLIB_REQUIRE_POSITIONS(std::int8_t); + SIMDLIB_REQUIRE_POSITIONS(std::uint8_t); + SIMDLIB_REQUIRE_POSITIONS(std::int16_t); + SIMDLIB_REQUIRE_POSITIONS(std::uint16_t); + SIMDLIB_REQUIRE_POSITIONS(std::int32_t); + SIMDLIB_REQUIRE_POSITIONS(std::uint32_t); + SIMDLIB_REQUIRE_POSITIONS(std::int64_t); + SIMDLIB_REQUIRE_POSITIONS(std::uint64_t); +#undef SIMDLIB_REQUIRE_POSITIONS +} + +TEST_CASE("Register saturation preserves lane and 128-bit grouping semantics", "[simdlib][register][specialized][saturation]") +{ + require_saturation_contract<128>(); + require_unsigned_horizontal_saturation_contract<128>(); + SIMDLIB_REGISTER_IF_256(require_saturation_contract<256>();) + SIMDLIB_REGISTER_IF_256(require_unsigned_horizontal_saturation_contract<256>();) +} + +TEST_CASE("Register promoted results preserve lane order and signedness", "[simdlib][register][specialized][promoted]") +{ + require_promoted_results<128>(); + SIMDLIB_REGISTER_IF_256(require_promoted_results<256>();) +#define SIMDLIB_REQUIRE_ADJACENT_CONTRACT(type) \ + require_adjacent_multiply_add_contract(); \ + SIMDLIB_REGISTER_IF_256(require_adjacent_multiply_add_contract();) + SIMDLIB_REQUIRE_ADJACENT_CONTRACT(std::int8_t); + SIMDLIB_REQUIRE_ADJACENT_CONTRACT(std::uint8_t); + SIMDLIB_REQUIRE_ADJACENT_CONTRACT(std::int16_t); + SIMDLIB_REQUIRE_ADJACENT_CONTRACT(std::uint16_t); + SIMDLIB_REQUIRE_ADJACENT_CONTRACT(std::int32_t); + SIMDLIB_REQUIRE_ADJACENT_CONTRACT(std::uint32_t); + SIMDLIB_REQUIRE_ADJACENT_CONTRACT(std::int64_t); + SIMDLIB_REQUIRE_ADJACENT_CONTRACT(std::uint64_t); +#undef SIMDLIB_REQUIRE_ADJACENT_CONTRACT +} + +TEST_CASE("Register floating specialized operations preserve immediate output behavior", "[simdlib][register][specialized][floating]") +{ + require_floating_specialized_operations(); + require_floating_specialized_operations(); + SIMDLIB_REGISTER_IF_256(require_floating_specialized_operations();) + SIMDLIB_REGISTER_IF_256(require_floating_specialized_operations();) +} + +} // namespace + +#undef SIMDLIB_REGISTER_IF_256 diff --git a/tests/SimdAlgo.tests.cpp b/tests/SimdAlgo.tests.cpp index 11897ca..61d0f91 100644 --- a/tests/SimdAlgo.tests.cpp +++ b/tests/SimdAlgo.tests.cpp @@ -15,8 +15,7 @@ namespace * @tparam ReadWidth Source element width in bits. * @tparam Count Static source element count. */ -template -void require_compare_tail_contract() +template void require_compare_tail_contract() { using Algo = SimdLib::SimdAlgo; using read_t = typename Algo::read_t; @@ -47,8 +46,7 @@ void require_compare_tail_contract() * @brief Verifies every full-register and tail outcome of AnyEqual. * @tparam ReadWidth Source element width in bits. */ -template -void require_any_equal_outcome_contract() +template void require_any_equal_outcome_contract() { using Algo = SimdLib::SimdAlgo; using read_t = typename Algo::read_t; @@ -63,7 +61,7 @@ void require_any_equal_outcome_contract() full.fill(other); REQUIRE_FALSE(Algo::AnyEqual(std::span{full}, predicate)); REQUIRE(Algo::AnyEqual(std::span{full}, predicate) == - std::ranges::any_of(full, [](const read_t value) { return value == predicate; })); + std::ranges::any_of(full, [](const read_t value) { return value == predicate; })); full.front() = predicate; REQUIRE(Algo::AnyEqual(std::span{full}, predicate)); @@ -80,15 +78,14 @@ void require_any_equal_outcome_contract() tail.back() = predicate; REQUIRE(Algo::AnyEqual(std::span{tail}, predicate)); REQUIRE(Algo::AnyEqual(std::span{tail}, predicate) == - std::ranges::any_of(tail, [](const read_t value) { return value == predicate; })); + std::ranges::any_of(tail, [](const read_t value) { return value == predicate; })); } /** * @brief Verifies every full-register and tail outcome of AllEqual. * @tparam ReadWidth Source element width in bits. */ -template -void require_all_equal_outcome_contract() +template void require_all_equal_outcome_contract() { using Algo = SimdLib::SimdAlgo; using read_t = typename Algo::read_t; @@ -103,7 +100,7 @@ void require_all_equal_outcome_contract() full.fill(predicate); REQUIRE(Algo::AllEqual(std::span{full}, predicate)); REQUIRE(Algo::AllEqual(std::span{full}, predicate) == - std::ranges::all_of(full, [](const read_t value) { return value == predicate; })); + std::ranges::all_of(full, [](const read_t value) { return value == predicate; })); full.front() = other; REQUIRE_FALSE(Algo::AllEqual(std::span{full}, predicate)); @@ -120,15 +117,14 @@ void require_all_equal_outcome_contract() tail.back() = other; REQUIRE_FALSE(Algo::AllEqual(std::span{tail}, predicate)); REQUIRE(Algo::AllEqual(std::span{tail}, predicate) == - std::ranges::all_of(tail, [](const read_t value) { return value == predicate; })); + std::ranges::all_of(tail, [](const read_t value) { return value == predicate; })); } /** * @brief Verifies empty, single-element, multi-element, exact-register, and tail static extents. * @tparam ReadWidth Source element width in bits. */ -template -void require_search_extent_contract() +template void require_search_extent_contract() { using Algo = SimdLib::SimdAlgo; using read_t = typename Algo::read_t; diff --git a/tests/SimdResample.tests.cpp b/tests/SimdResample.tests.cpp index e46789d..f369f09 100644 --- a/tests/SimdResample.tests.cpp +++ b/tests/SimdResample.tests.cpp @@ -52,13 +52,13 @@ void expand_reference(const std::span src, const std::span data, std::mt19937& random) +void fill_random(const std::span data, std::mt19937 &random) { std::uniform_int_distribution distribution(0, 255); - for (auto& value : data) + for (auto &value : data) value = static_cast(distribution(random)); } -} +} // namespace TEST_CASE("SimdResample preserves reduce bit ordering", "[simdlib][resample][ordering]") { diff --git a/tests/SimdVector.tests.cpp b/tests/SimdVector.tests.cpp index c4e4899..09c8e9f 100644 --- a/tests/SimdVector.tests.cpp +++ b/tests/SimdVector.tests.cpp @@ -23,7 +23,7 @@ namespace */ template requires requires { typename Vector::simd; } -void require_lanes(const Vector& value, const std::array& expected) +void require_lanes(const Vector &value, const std::array &expected) { const auto actual = value.toArray(); for (std::size_t index = 0; index < Count; ++index) @@ -42,11 +42,11 @@ void require_lanes(const Vector& value, const std::array& expect */ template requires(!requires { typename Register::simd; }) -void require_lanes(const Register value, const std::array& expected) +void require_lanes(const Register value, const std::array &expected) { require_lanes(SimdLib::SimdVector(Count)>{value}, expected); } -} +} // namespace namespace { @@ -58,20 +58,17 @@ namespace * @param expected Expected scalar dot product. */ template -void require_dot_product(const std::array& lhs, const std::array& rhs, const Element expected) +void require_dot_product(const std::array &lhs, const std::array &rhs, const Element expected) { using Vector = SimdLib::SimdVector(Count)>; const Vector lhs_vector(lhs); const Vector rhs_vector(rhs); REQUIRE(lhs_vector.dot_product(rhs_vector.getRegister()) == expected); } -} +} // namespace -TEST_CASE("SimdVector exposes the complete aliases and storage facade", "[simdlib][vector]") +TEST_CASE("SimdVector exposes the complete storage facade", "[simdlib][vector]") { - static_assert(std::same_as>); - static_assert(std::same_as>); - static_assert(std::same_as>); SimdLib::SimdVector value(4, -7, 11); require_lanes(value, std::array{4, -7, 11}); @@ -160,8 +157,7 @@ TEST_CASE("SimdVector bitwise saturation widening and hash match logical lanes", require_lanes(wide, std::array{-4, 7, 300}); const auto sameHash = std::hash>{}(wide); - const auto otherHash = std::hash>{}( - SimdLib::SimdVector(-4, 7, 301)); + const auto otherHash = std::hash>{}(SimdLib::SimdVector(-4, 7, 301)); REQUIRE(sameHash != otherHash); } @@ -185,23 +181,29 @@ TEST_CASE("SimdVector integer area covers full partial odd and cross-lane extent REQUIRE(SimdLib::SimdVector(std::numeric_limits::max(), 2).area() == -2); } -TEST_CASE("SimdVector integer magnitude preserves per-128-bit-lane results", "[simdlib][vector][partial][magnitude]") +TEST_CASE("SimdVector integer magnitude preserves sparse per-128-bit-group results", "[simdlib][vector][partial][magnitude]") { using Signed = SimdLib::SimdVector; - const Signed signed_value(3, 4, 0, 0, 0, 0, 0, 0, 6); - const auto signed_magnitude = Signed::simd::to_array(signed_value.magnitude()); - for (std::size_t index = 0; index < 8; ++index) - REQUIRE(signed_magnitude[index] == 5); - for (std::size_t index = 8; index < signed_magnitude.size(); ++index) - REQUIRE(signed_magnitude[index] == 6); + const Signed signedValue(3, 4, 0, 0, 0, 0, 0, 0, 6); + const auto signedMagnitude = Signed::simd::to_array(signedValue.magnitude()); + const auto signedChecked = Signed::simd::to_array(signedValue.magnitude_checked()); + REQUIRE(signedMagnitude[0] == 5); + REQUIRE(signedMagnitude[8] == 6); + REQUIRE(signedChecked[0] == 5); + REQUIRE(signedChecked[1] == 0); + REQUIRE(signedChecked[8] == 6); + REQUIRE(signedChecked[9] == 0); using Unsigned = SimdLib::SimdVector; - const Unsigned unsigned_value(6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9); - const auto unsigned_magnitude = Unsigned::simd::to_array(unsigned_value.magnitude()); - for (std::size_t index = 0; index < 16; ++index) - REQUIRE(unsigned_magnitude[index] == 10); - for (std::size_t index = 16; index < unsigned_magnitude.size(); ++index) - REQUIRE(unsigned_magnitude[index] == 9); + const Unsigned unsignedValue(6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9); + const auto unsignedMagnitude = Unsigned::simd::to_array(unsignedValue.magnitude()); + const auto unsignedChecked = Unsigned::simd::to_array(unsignedValue.magnitude_checked()); + REQUIRE(unsignedMagnitude[0] == 10); + REQUIRE(unsignedMagnitude[16] == 9); + REQUIRE(unsignedChecked[0] == 10); + REQUIRE(unsignedChecked[1] == 0); + REQUIRE(unsignedChecked[16] == 9); + REQUIRE(unsignedChecked[17] == 0); } TEST_CASE("SimdVector partial positions ignore inactive zero-filled lanes", "[simdlib][vector][partial][position]") { @@ -219,14 +221,12 @@ TEST_CASE("SimdVector hashes respect floating equality for signed zero", "[simdl const SimdLib::SimdVector positive_zero(0.0f, 2.0f, 0.0f); const SimdLib::SimdVector negative_zero(-0.0f, 2.0f, -0.0f); REQUIRE(positive_zero == negative_zero.getRegister()); - REQUIRE(std::hash>{}(positive_zero) == - std::hash>{}(negative_zero)); + REQUIRE(std::hash>{}(positive_zero) == std::hash>{}(negative_zero)); const SimdLib::SimdVector positive_double_zero(0.0); const SimdLib::SimdVector negative_double_zero(-0.0); REQUIRE(positive_double_zero == negative_double_zero.getRegister()); - REQUIRE(std::hash>{}(positive_double_zero) == - std::hash>{}(negative_double_zero)); + REQUIRE(std::hash>{}(positive_double_zero) == std::hash>{}(negative_double_zero)); } TEST_CASE("SimdVector floating hashes cover nonzero infinities and NaNs", "[simdlib][vector][hash][float]") @@ -250,11 +250,9 @@ TEST_CASE("SimdVector floating hashes cover nonzero infinities and NaNs", "[simd REQUIRE(double_hash(double_value) == double_hash(double_copy)); REQUIRE(double_hash(double_value) != double_hash(double_distinct)); - const FloatVector float_infinities( - std::numeric_limits::infinity(), -std::numeric_limits::infinity(), 1.0f, 2.0f, 3.0f); + const FloatVector float_infinities(std::numeric_limits::infinity(), -std::numeric_limits::infinity(), 1.0f, 2.0f, 3.0f); REQUIRE(float_hash(float_infinities) == float_hash(FloatVector(float_infinities))); - const DoubleVector double_infinities( - std::numeric_limits::infinity(), -std::numeric_limits::infinity(), 1.0); + const DoubleVector double_infinities(std::numeric_limits::infinity(), -std::numeric_limits::infinity(), 1.0); REQUIRE(double_hash(double_infinities) == double_hash(DoubleVector(double_infinities))); const float alternate_float_nan = std::bit_cast(std::uint32_t{0x7FC00001u}); @@ -386,7 +384,7 @@ TEST_CASE("SimdVector documentation examples produce their documented results", REQUIRE(I16x4::simd::to_array(I16x4{30000, -10000, -30000, 10000}.subtract_horizontal_saturated(I16x4{1, 2, 3, 4})) == std::array{32767, -32768, 0, 0, -1, -1, 0, 0}); REQUIRE(SimdLib::Api<128, std::int32_t>::to_array(I16x4{1, 2, 3, 4}.multiply_add_adjacent(I16x4{5, 6, 7, 8})) == std::array{17, 53, 0, 0}); - using U8x16 = SimdLib::uint8x16; + using U8x16 = SimdLib::SimdVector; REQUIRE(SimdLib::Api<128, std::int16_t>::to_array(U8x16{2}.multiply_add_unsigned_signed_bytes(U8x16{3})) == std::array{12, 12, 12, 12, 12, 12, 12, 12}); REQUIRE(SimdLib::Api<128, std::uint64_t>::to_array(U8x16{9}.sum_absolute_byte_differences(U8x16{4})) == std::array{40, 40}); diff --git a/tests/SimdVectorChecks.tests.cpp b/tests/SimdVectorChecks.tests.cpp index 2f0a144..bdad2f3 100644 --- a/tests/SimdVectorChecks.tests.cpp +++ b/tests/SimdVectorChecks.tests.cpp @@ -9,7 +9,7 @@ inline bool every_condition_passed = true; * @param condition Condition evaluated by the public operation. * @param message Operation description supplied to the precondition hook. */ -inline void RecordPrecondition(const bool condition, const char* message) noexcept +inline void RecordPrecondition(const bool condition, const char *message) noexcept { ++invocation_count; every_condition_passed = every_condition_passed && condition; @@ -22,7 +22,7 @@ inline void Reset() noexcept invocation_count = 0; every_condition_passed = true; } -} +} // namespace SimdVectorCheckProbe #define SIMDLIB_PRECONDITION(condition, message) ::SimdVectorCheckProbe::RecordPrecondition((condition), (message)) diff --git a/tests/TestSupport.h b/tests/TestSupport.h index 5f7a202..54b0d34 100644 --- a/tests/TestSupport.h +++ b/tests/TestSupport.h @@ -1,7 +1,7 @@ #pragma once -#include #include "constexpr/ApiConstexprContracts.h" +#include #include @@ -18,275 +18,419 @@ namespace SimdLib::Tests { -template -void require_addition_parity() -{ - using simd = Api; - std::array lhs{}; - std::array rhs{}; - std::array expected{}; - for (std::size_t index = 0; index < simd::element_count; ++index) - { - lhs[index] = static_cast(index + 1); - rhs[index] = static_cast(2); - expected[index] = static_cast(index + 3); - } - REQUIRE(simd::to_array(simd::add(simd::load(lhs), simd::load(rhs))) == expected); -} - -template -void require_supported_addition_matrix() -{ - require_addition_parity(); - require_addition_parity(); - require_addition_parity(); - require_addition_parity(); - require_addition_parity(); - require_addition_parity(); - require_addition_parity(); - require_addition_parity(); - require_addition_parity(); - require_addition_parity(); +template void require_addition_parity() +{ + using simd = Api; + std::array lhs{}; + std::array rhs{}; + std::array expected{}; + for (std::size_t index = 0; index < simd::element_count; ++index) + { + lhs[index] = static_cast(index + 1); + rhs[index] = static_cast(2); + expected[index] = static_cast(index + 3); + } + REQUIRE(simd::to_array(simd::add(simd::load(lhs), simd::load(rhs))) == expected); } -template -void require_transfer_contracts() -{ - using simd = Api; - alignas(Width / 8) std::array aligned{}; - for (std::size_t index = 0; index < aligned.size(); ++index) - aligned[index] = static_cast(index + 1); - - const auto aligned_register = simd::load_aligned(aligned); - alignas(Width / 8) std::array aligned_output{}; - simd::store_aligned(aligned_register, aligned_output); - REQUIRE(aligned_output == aligned); - - alignas(64) std::array offset_storage{}; - std::copy(aligned.begin(), aligned.end(), offset_storage.begin() + 1); - const std::span unaligned_input{offset_storage.data() + 1, simd::element_count}; - const auto unaligned_register = simd::load_unaligned(unaligned_input); - - alignas(64) std::array offset_output{}; - std::span unaligned_output{offset_output.data() + 1, simd::element_count}; - simd::store_unaligned(unaligned_register, unaligned_output); - REQUIRE(std::equal(aligned.begin(), aligned.end(), unaligned_output.begin())); - - std::array bytes{}; - simd::store(unaligned_register, std::span{bytes}); - REQUIRE(bytes.size() == simd::byte_count); - - std::array oversized_bytes{}; - simd::store(unaligned_register, std::span{oversized_bytes}); - std::array recovered{}; - std::memcpy(recovered.data(), oversized_bytes.data(), simd::byte_count); - REQUIRE(recovered == aligned); -} - -template -void require_supported_transfer_matrix() -{ - require_transfer_contracts(); - require_transfer_contracts(); - require_transfer_contracts(); - require_transfer_contracts(); - require_transfer_contracts(); - require_transfer_contracts(); - require_transfer_contracts(); - require_transfer_contracts(); - require_transfer_contracts(); - require_transfer_contracts(); +template void require_supported_addition_matrix() +{ + require_addition_parity(); + require_addition_parity(); + require_addition_parity(); + require_addition_parity(); + require_addition_parity(); + require_addition_parity(); + require_addition_parity(); + require_addition_parity(); + require_addition_parity(); + require_addition_parity(); } -template -void require_partial_transfer_contracts() +/** + * @brief Verifies runtime-selected extraction from every lane of one register specialization. + * @tparam Width Register width in bits. + * @tparam Element Scalar lane type. + */ +template void require_runtime_extraction_contract() +{ + using simd = Api; + std::array expected{}; + for (std::size_t index = 0; index < expected.size(); ++index) + { + if constexpr (std::is_floating_point_v) + expected[index] = static_cast(index) + static_cast(0.25); + else if constexpr (std::is_signed_v) + expected[index] = static_cast(static_cast(index) - 8); + else + expected[index] = static_cast(index * 7 + 3); + } + + const auto value = simd::construct(expected); + for (std::size_t index = 0; index < expected.size(); ++index) + { + const volatile int runtime_index = static_cast(index); + REQUIRE(simd::extract_slow(value, runtime_index) == expected[index]); + } +} + +/** @brief Verifies runtime-selected extraction for every lane of every supported 128-bit element type. */ +inline void require_runtime_extraction_matrix_128() +{ + require_runtime_extraction_contract<128, std::int8_t>(); + require_runtime_extraction_contract<128, std::uint8_t>(); + require_runtime_extraction_contract<128, std::int16_t>(); + require_runtime_extraction_contract<128, std::uint16_t>(); + require_runtime_extraction_contract<128, std::int32_t>(); + require_runtime_extraction_contract<128, std::uint32_t>(); + require_runtime_extraction_contract<128, std::int64_t>(); + require_runtime_extraction_contract<128, std::uint64_t>(); + require_runtime_extraction_contract<128, float>(); + require_runtime_extraction_contract<128, double>(); +} + +/** + * @brief Verifies runtime-selected insertion into every lane of one register specialization. + * @tparam Width Register width in bits. + * @tparam Element Scalar lane type. + */ +template void require_runtime_insertion_contract() +{ + using simd = Api; + std::array source{}; + for (std::size_t index = 0; index < source.size(); ++index) + { + if constexpr (std::is_floating_point_v) + source[index] = static_cast(index) + static_cast(0.25); + else if constexpr (std::is_signed_v) + source[index] = static_cast(static_cast(index) - 8); + else + source[index] = static_cast(index * 7 + 3); + } + + const auto value = simd::construct(source); + for (std::size_t index = 0; index < source.size(); ++index) + { + const Element replacement = [&]() constexpr + { + if constexpr (std::is_floating_point_v) + return static_cast(-static_cast(index) - 0.75); + else if constexpr (std::is_signed_v) + return static_cast(-static_cast(index) - 11); + else + return static_cast(std::numeric_limits::max() - static_cast(index)); + }(); + auto expected = source; + expected[index] = replacement; + const volatile int runtime_index = static_cast(index); + REQUIRE(simd::to_array(simd::insert_slow(value, replacement, runtime_index)) == expected); + } +} + +/** @brief Verifies runtime-selected insertion for every lane of every supported 128-bit element type. */ +inline void require_runtime_insertion_matrix_128() { - using simd = Api; - alignas(64) std::array storage{}; - for (std::size_t index = 0; index < simd::element_count; ++index) - storage[index + 1] = static_cast(index + 1); + require_runtime_insertion_contract<128, std::int8_t>(); + require_runtime_insertion_contract<128, std::uint8_t>(); + require_runtime_insertion_contract<128, std::int16_t>(); + require_runtime_insertion_contract<128, std::uint16_t>(); + require_runtime_insertion_contract<128, std::int32_t>(); + require_runtime_insertion_contract<128, std::uint32_t>(); + require_runtime_insertion_contract<128, std::int64_t>(); + require_runtime_insertion_contract<128, std::uint64_t>(); + require_runtime_insertion_contract<128, float>(); + require_runtime_insertion_contract<128, double>(); +} - const std::span unaligned{storage.data() + 1, simd::element_count}; - const auto none = simd::template load_partial<0>(unaligned); - REQUIRE(simd::to_array(none) == std::array{}); +#if SIMDLIB_HAS_AVX2 +/** @brief Verifies runtime-selected extraction for every lane of every supported 256-bit element type. */ +inline void require_runtime_extraction_matrix_256() +{ + require_runtime_extraction_contract<256, std::int8_t>(); + require_runtime_extraction_contract<256, std::uint8_t>(); + require_runtime_extraction_contract<256, std::int16_t>(); + require_runtime_extraction_contract<256, std::uint16_t>(); + require_runtime_extraction_contract<256, std::int32_t>(); + require_runtime_extraction_contract<256, std::uint32_t>(); + require_runtime_extraction_contract<256, std::int64_t>(); + require_runtime_extraction_contract<256, std::uint64_t>(); + require_runtime_extraction_contract<256, float>(); + require_runtime_extraction_contract<256, double>(); +} - const auto one = simd::template load_partial<1>(unaligned); - auto expected_one = std::array{}; - expected_one[0] = storage[1]; - REQUIRE(simd::to_array(one) == expected_one); +/** @brief Verifies runtime-selected insertion for every lane of every supported 256-bit element type. */ +inline void require_runtime_insertion_matrix_256() +{ + require_runtime_insertion_contract<256, std::int8_t>(); + require_runtime_insertion_contract<256, std::uint8_t>(); + require_runtime_insertion_contract<256, std::int16_t>(); + require_runtime_insertion_contract<256, std::uint16_t>(); + require_runtime_insertion_contract<256, std::int32_t>(); + require_runtime_insertion_contract<256, std::uint32_t>(); + require_runtime_insertion_contract<256, std::int64_t>(); + require_runtime_insertion_contract<256, std::uint64_t>(); + require_runtime_insertion_contract<256, float>(); + require_runtime_insertion_contract<256, double>(); +} +#endif - const auto almost_full = simd::template load_partial(unaligned); - auto expected_almost_full = std::array{}; - std::copy_n(storage.begin() + 1, simd::element_count - 1, expected_almost_full.begin()); - REQUIRE(simd::to_array(almost_full) == expected_almost_full); +template void require_transfer_contracts() +{ + using simd = Api; + alignas(Width / 8) std::array aligned{}; + for (std::size_t index = 0; index < aligned.size(); ++index) + aligned[index] = static_cast(index + 1); + + const auto aligned_register = simd::load_aligned(aligned); + alignas(Width / 8) std::array aligned_output{}; + simd::store_aligned(aligned_register, aligned_output); + REQUIRE(aligned_output == aligned); + + alignas(64) std::array offset_storage{}; + std::copy(aligned.begin(), aligned.end(), offset_storage.begin() + 1); + const std::span unaligned_input{offset_storage.data() + 1, simd::element_count}; + const auto unaligned_register = simd::load_unaligned(unaligned_input); + + alignas(64) std::array offset_output{}; + std::span unaligned_output{offset_output.data() + 1, simd::element_count}; + simd::store_unaligned(unaligned_register, unaligned_output); + REQUIRE(std::equal(aligned.begin(), aligned.end(), unaligned_output.begin())); + + std::array bytes{}; + simd::store(unaligned_register, std::span{bytes}); + REQUIRE(bytes.size() == simd::byte_count); + const auto byte_loaded = simd::load(std::span{bytes}); + std::array exact_bytes{}; + simd::store(byte_loaded, std::span{exact_bytes}); + for (std::size_t index = 0; index < bytes.size(); ++index) + REQUIRE(std::to_integer(exact_bytes[index]) == std::to_integer(bytes[index])); + REQUIRE(simd::to_array(byte_loaded) == aligned); + + std::array oversized_bytes{}; + simd::store(unaligned_register, std::span{oversized_bytes}); + std::array recovered{}; + std::memcpy(recovered.data(), oversized_bytes.data(), simd::byte_count); + REQUIRE(recovered == aligned); +} - const auto full = simd::template load_partial(unaligned); - std::array expected_full{}; - std::copy_n(storage.begin() + 1, simd::element_count, expected_full.begin()); - REQUIRE(simd::to_array(full) == expected_full); +template void require_supported_transfer_matrix() +{ + require_transfer_contracts(); + require_transfer_contracts(); + require_transfer_contracts(); + require_transfer_contracts(); + require_transfer_contracts(); + require_transfer_contracts(); + require_transfer_contracts(); + require_transfer_contracts(); + require_transfer_contracts(); + require_transfer_contracts(); } -template -void require_supported_partial_transfer_matrix() +template void require_partial_transfer_contracts() { - require_partial_transfer_contracts(); - require_partial_transfer_contracts(); - require_partial_transfer_contracts(); - require_partial_transfer_contracts(); - require_partial_transfer_contracts(); + using simd = Api; + alignas(64) std::array storage{}; + for (std::size_t index = 0; index < simd::element_count; ++index) + storage[index + 1] = static_cast(index + 1); + + const std::span unaligned{storage.data() + 1, simd::element_count}; + const auto none = simd::template load_partial<0>(unaligned); + REQUIRE(simd::to_array(none) == std::array{}); + + const auto one = simd::template load_partial<1>(unaligned); + auto expected_one = std::array{}; + expected_one[0] = storage[1]; + REQUIRE(simd::to_array(one) == expected_one); + + const auto almost_full = simd::template load_partial(unaligned); + auto expected_almost_full = std::array{}; + std::copy_n(storage.begin() + 1, simd::element_count - 1, expected_almost_full.begin()); + REQUIRE(simd::to_array(almost_full) == expected_almost_full); + + const auto full = simd::template load_partial(unaligned); + std::array expected_full{}; + std::copy_n(storage.begin() + 1, simd::element_count, expected_full.begin()); + REQUIRE(simd::to_array(full) == expected_full); +} + +template void require_supported_partial_transfer_matrix() +{ + require_partial_transfer_contracts(); + require_partial_transfer_contracts(); + require_partial_transfer_contracts(); + require_partial_transfer_contracts(); + require_partial_transfer_contracts(); } template requires std::is_arithmetic_v void require_comparison_contract() { - using simd = Api; - std::array lhs{}; - std::array rhs{}; - for (std::size_t index = 0; index < simd::element_count; ++index) - { - switch (index % 4) - { - case 0: - lhs[index] = Element{0}; - rhs[index] = Element{0}; - break; - case 1: - lhs[index] = Element{1}; - rhs[index] = Element{2}; - break; - case 2: - lhs[index] = Element{3}; - rhs[index] = Element{2}; - break; - default: - lhs[index] = std::numeric_limits::max(); - rhs[index] = std::numeric_limits::lowest(); - break; - } - } - - typename simd::mask_t eq = 0; - typename simd::mask_t gt = 0; - typename simd::mask_t ge = 0; - typename simd::mask_t lt = 0; - typename simd::mask_t le = 0; - constexpr typename simd::mask_t lane_mask = - static_cast((typename simd::mask_t{1} << sizeof(Element)) - 1); - for (std::size_t index = 0; index < simd::element_count; ++index) - { - const auto mask = static_cast(lane_mask << (index * sizeof(Element))); - if (lhs[index] == rhs[index]) - eq |= mask; - if (lhs[index] > rhs[index]) - gt |= mask; - if (lhs[index] >= rhs[index]) - ge |= mask; - if (lhs[index] < rhs[index]) - lt |= mask; - if (lhs[index] <= rhs[index]) - le |= mask; - } - - const auto left = simd::construct(lhs); - const auto right = simd::construct(rhs); - REQUIRE(simd::cmp_eq(left, right) == eq); - REQUIRE(simd::cmp_eq_mask(left, right) == eq); - REQUIRE(simd::cmp_gt(left, right) == gt); - REQUIRE(simd::cmp_ge(left, right) == ge); - REQUIRE(simd::cmp_lt(left, right) == lt); - REQUIRE(simd::cmp_le(left, right) == le); -} - -template -void require_supported_comparison_matrix() -{ - require_comparison_contract(); - require_comparison_contract(); - require_comparison_contract(); - require_comparison_contract(); - require_comparison_contract(); - require_comparison_contract(); - require_comparison_contract(); - require_comparison_contract(); -} - -template -void require_transform_pack_mask_contract() -{ - using simd = Api; - using write_t = typename simd::template packed_element_t<1>; - constexpr std::size_t output_count = simd::template packed_element_count<1, Count>; - - std::array input{}; - for (std::size_t index = 0; index < Count; ++index) - input[index] = index % 3 == 1 ? Element{0} : static_cast(index + 1); - - std::array guarded{}; - guarded.fill(static_cast(0xA5)); - std::span output{guarded.data() + 1, output_count}; - const auto predicate = simd::set1(Element{0}); - simd::template transform_pack<1>( - std::span{input}, output, - [&predicate](const typename simd::vector_t value) noexcept - { return simd::movemask_slim(simd::cmpeq(value, predicate)); }); - - std::array expected{}; - for (std::size_t index = 0; index < Count; ++index) - if (input[index] == 0) - expected[index / 8] |= static_cast(write_t{1} << (index % 8)); - - REQUIRE(std::equal(output.begin(), output.end(), expected.begin())); - REQUIRE(guarded.front() == static_cast(0xA5)); - REQUIRE(guarded.back() == static_cast(0xA5)); -} - -template -void require_transform_pack_width_contract() -{ - static_assert(ResultBitWidth > 0 && ResultBitWidth <= 64); - using simd = Api; - using write_t = typename simd::template packed_element_t; - constexpr std::size_t output_count = simd::template packed_element_count; - constexpr std::size_t write_element_width = std::numeric_limits::digits; - constexpr std::uint64_t result_mask = ResultBitWidth == 64 - ? std::numeric_limits::max() - : (std::uint64_t{1} << ResultBitWidth) - 1; - - std::array input{}; - for (std::size_t index = 0; index < Count; ++index) - input[index] = static_cast(index * 5 + 3); - - std::array guarded{}; - guarded.fill(static_cast(0xA5)); - std::span output{guarded.data() + 1, output_count}; - simd::template transform_pack( - std::span{input}, output, - [](const typename simd::vector_t value) noexcept - { - const auto lanes = simd::to_array(value); - std::uint64_t packed = 0; - for (std::size_t lane = 0; lane < lanes.size(); ++lane) - packed |= (static_cast(lanes[lane]) & result_mask) << (lane * ResultBitWidth); - return packed; - }); - - std::array expected{}; - for (std::size_t index = 0; index < Count; ++index) - { - const std::uint64_t result = static_cast(input[index]) & result_mask; - for (std::size_t bit = 0; bit < ResultBitWidth; ++bit) - { - const std::size_t output_bit = index * ResultBitWidth + bit; - if ((result & (std::uint64_t{1} << bit)) != 0) - expected[output_bit / write_element_width] |= - static_cast(write_t{1} << (output_bit % write_element_width)); - } - } - - REQUIRE(std::equal(output.begin(), output.end(), expected.begin())); - REQUIRE(guarded.front() == static_cast(0xA5)); - REQUIRE(guarded.back() == static_cast(0xA5)); + using simd = Api; + std::array lhs{}; + std::array rhs{}; + for (std::size_t index = 0; index < simd::element_count; ++index) + { + switch (index % 4) + { + case 0: + lhs[index] = Element{0}; + rhs[index] = Element{0}; + break; + case 1: + lhs[index] = Element{1}; + rhs[index] = Element{2}; + break; + case 2: + lhs[index] = Element{3}; + rhs[index] = Element{2}; + break; + default: + lhs[index] = std::numeric_limits::max(); + rhs[index] = std::numeric_limits::lowest(); + break; + } + } + + typename simd::mask_t eq = 0; + typename simd::mask_t gt = 0; + typename simd::mask_t ge = 0; + typename simd::mask_t lt = 0; + typename simd::mask_t le = 0; + typename simd::mask_t eqSlim = 0; + typename simd::mask_t gtSlim = 0; + typename simd::mask_t geSlim = 0; + typename simd::mask_t ltSlim = 0; + typename simd::mask_t leSlim = 0; + std::array selected{}; + constexpr typename simd::mask_t lane_mask = static_cast((typename simd::mask_t{1} << sizeof(Element)) - 1); + for (std::size_t index = 0; index < simd::element_count; ++index) + { + const auto mask = static_cast(lane_mask << (index * sizeof(Element))); + if (lhs[index] == rhs[index]) + { + eq |= mask; + eqSlim |= typename simd::mask_t{1} << index; + } + if (lhs[index] > rhs[index]) + { + gt |= mask; + gtSlim |= typename simd::mask_t{1} << index; + } + if (lhs[index] >= rhs[index]) + { + ge |= mask; + geSlim |= typename simd::mask_t{1} << index; + } + if (lhs[index] < rhs[index]) + { + lt |= mask; + ltSlim |= typename simd::mask_t{1} << index; + } + if (lhs[index] <= rhs[index]) + { + le |= mask; + leSlim |= typename simd::mask_t{1} << index; + } + selected[index] = lhs[index] == rhs[index] ? lhs[index] : rhs[index]; + } + + const auto left = simd::construct(lhs); + const auto right = simd::construct(rhs); + REQUIRE(simd::cmp_eq_mask(left, right) == eq); + REQUIRE(simd::cmp_gt_mask(left, right) == gt); + REQUIRE(simd::cmp_ge_mask(left, right) == ge); + REQUIRE(simd::cmp_lt_mask(left, right) == lt); + REQUIRE(simd::cmp_le_mask(left, right) == le); + REQUIRE(simd::cmp_eq_slim(left, right) == eqSlim); + REQUIRE(simd::cmp_gt_slim(left, right) == gtSlim); + REQUIRE(simd::cmp_ge_slim(left, right) == geSlim); + REQUIRE(simd::cmp_lt_slim(left, right) == ltSlim); + REQUIRE(simd::cmp_le_slim(left, right) == leSlim); + REQUIRE(simd::to_array(simd::select(simd::compare_equal(left, right), left, right)) == selected); +} + +template void require_supported_comparison_matrix() +{ + require_comparison_contract(); + require_comparison_contract(); + require_comparison_contract(); + require_comparison_contract(); + require_comparison_contract(); + require_comparison_contract(); + require_comparison_contract(); + require_comparison_contract(); +} + +template void require_transform_pack_mask_contract() +{ + using simd = Api; + using write_t = typename simd::template packed_element_t<1>; + constexpr std::size_t output_count = simd::template packed_element_count<1, Count>; + + std::array input{}; + for (std::size_t index = 0; index < Count; ++index) + input[index] = index % 3 == 1 ? Element{0} : static_cast(index + 1); + + std::array guarded{}; + guarded.fill(static_cast(0xA5)); + std::span output{guarded.data() + 1, output_count}; + const auto predicate = simd::set1(Element{0}); + simd::template transform_pack<1>(std::span{input}, output, + [&predicate](const typename simd::vector_t value) noexcept { return simd::movemask_slim(simd::cmpeq(value, predicate)); }); + + std::array expected{}; + for (std::size_t index = 0; index < Count; ++index) + if (input[index] == 0) + expected[index / 8] |= static_cast(write_t{1} << (index % 8)); + + REQUIRE(std::equal(output.begin(), output.end(), expected.begin())); + REQUIRE(guarded.front() == static_cast(0xA5)); + REQUIRE(guarded.back() == static_cast(0xA5)); +} + +template void require_transform_pack_width_contract() +{ + static_assert(ResultBitWidth > 0 && ResultBitWidth <= 64); + using simd = Api; + using write_t = typename simd::template packed_element_t; + constexpr std::size_t output_count = simd::template packed_element_count; + constexpr std::size_t write_element_width = std::numeric_limits::digits; + constexpr std::uint64_t result_mask = ResultBitWidth == 64 ? std::numeric_limits::max() : (std::uint64_t{1} << ResultBitWidth) - 1; + + std::array input{}; + for (std::size_t index = 0; index < Count; ++index) + input[index] = static_cast(index * 5 + 3); + + std::array guarded{}; + guarded.fill(static_cast(0xA5)); + std::span output{guarded.data() + 1, output_count}; + simd::template transform_pack(std::span{input}, output, + [](const typename simd::vector_t value) noexcept + { + const auto lanes = simd::to_array(value); + std::uint64_t packed = 0; + for (std::size_t lane = 0; lane < lanes.size(); ++lane) + packed |= (static_cast(lanes[lane]) & result_mask) << (lane * ResultBitWidth); + return packed; + }); + + std::array expected{}; + for (std::size_t index = 0; index < Count; ++index) + { + const std::uint64_t result = static_cast(input[index]) & result_mask; + for (std::size_t bit = 0; bit < ResultBitWidth; ++bit) + { + const std::size_t output_bit = index * ResultBitWidth + bit; + if ((result & (std::uint64_t{1} << bit)) != 0) + expected[output_bit / write_element_width] |= static_cast(write_t{1} << (output_bit % write_element_width)); + } + } + + REQUIRE(std::equal(output.begin(), output.end(), expected.begin())); + REQUIRE(guarded.front() == static_cast(0xA5)); + REQUIRE(guarded.back() == static_cast(0xA5)); } /** @@ -296,61 +440,56 @@ void require_transform_pack_width_contract() * This is intentionally separate from the general width matrix: it documents the no-shift-by-64 * boundary and requires the accumulator flush that writes a complete native word. */ -template -void require_transform_pack_full_native_word_contract() +template void require_transform_pack_full_native_word_contract() { - using simd = Api; - constexpr std::size_t resultBitWidth = 64 / simd::element_count; - require_transform_pack_width_contract(); + using simd = Api; + constexpr std::size_t resultBitWidth = 64 / simd::element_count; + require_transform_pack_width_contract(); } /** * @brief Adds a fixed scalar amount to every lane of a 32-bit SIMD register. * @tparam Width SIMD register width in bits. */ -template -struct Add17Transform +template struct Add17Transform { - using simd = Api; + using simd = Api; - /** @brief Applies the transform to one register. */ - [[nodiscard]] typename simd::vector_t operator()(const typename simd::vector_t value) const noexcept - { - return simd::add(value, simd::set1(17)); - } + /** @brief Applies the transform to one register. */ + [[nodiscard]] typename simd::vector_t operator()(const typename simd::vector_t value) const noexcept + { + return simd::add(value, simd::set1(17)); + } }; /** * @brief Subtracts a fixed scalar amount from every lane of a 32-bit SIMD register. * @tparam Width SIMD register width in bits. */ -template -struct Subtract13Transform +template struct Subtract13Transform { - using simd = Api; + using simd = Api; - /** @brief Applies the transform to one register. */ - [[nodiscard]] typename simd::vector_t operator()(const typename simd::vector_t value) const noexcept - { - return simd::subtract(value, simd::set1(13)); - } + /** @brief Applies the transform to one register. */ + [[nodiscard]] typename simd::vector_t operator()(const typename simd::vector_t value) const noexcept + { + return simd::subtract(value, simd::set1(13)); + } }; /** * @brief Subtracts corresponding lanes of two 32-bit SIMD registers. * @tparam Width SIMD register width in bits. */ -template -struct SubtractTransform +template struct SubtractTransform { - using simd = Api; + using simd = Api; - /** @brief Applies the transform to two registers. */ - [[nodiscard]] typename simd::vector_t operator()( - const typename simd::vector_t lhs, const typename simd::vector_t rhs) const noexcept - { - return simd::subtract(lhs, rhs); - } + /** @brief Applies the transform to two registers. */ + [[nodiscard]] typename simd::vector_t operator()(const typename simd::vector_t lhs, const typename simd::vector_t rhs) const noexcept + { + return simd::subtract(lhs, rhs); + } }; /** @@ -358,142 +497,134 @@ struct SubtractTransform * @tparam Width SIMD register width in bits. * @tparam Count Number of logical elements in each source and destination span. */ -template -void require_transform_overload_case() -{ - using simd = Api; - constexpr std::uint32_t guard = 0xDEADBEEFU; - std::array unaryStorage{}; - std::array leftStorage{}; - std::array rightStorage{}; - std::array outputStorage{}; - unaryStorage.fill(guard); - leftStorage.fill(guard); - rightStorage.fill(guard); - outputStorage.fill(guard); - - auto unary = std::span(unaryStorage).subspan(1, Count); - auto left = std::span(leftStorage).subspan(1, Count); - auto right = std::span(rightStorage).subspan(1, Count); - auto output = std::span(outputStorage).subspan(1, Count); - for (std::size_t index = 0; index < Count; ++index) - { - unary[index] = static_cast(index * 7 + 5); - left[index] = static_cast(index * 7 + 50); - right[index] = static_cast(index + 3); - } - - simd::transform(unary, Add17Transform{}); - - for (std::size_t index = 0; index < Count; ++index) - REQUIRE(unary[index] == static_cast(index * 7 + 22)); - REQUIRE(unaryStorage.front() == guard); - REQUIRE(unaryStorage.back() == guard); - - simd::transform(std::span(left), output, Subtract13Transform{}); - - for (std::size_t index = 0; index < Count; ++index) - REQUIRE(output[index] == static_cast(index * 7 + 37)); - REQUIRE(outputStorage.front() == guard); - REQUIRE(outputStorage.back() == guard); - - std::fill(output.begin(), output.end(), guard); - simd::transform(std::span(left), std::span(right), output, SubtractTransform{}); - - for (std::size_t index = 0; index < Count; ++index) - REQUIRE(output[index] == static_cast(index * 6 + 47)); - REQUIRE(outputStorage.front() == guard); - REQUIRE(outputStorage.back() == guard); +template void require_transform_overload_case() +{ + using simd = Api; + constexpr std::uint32_t guard = 0xDEADBEEFU; + constexpr std::size_t storageCount = Count + 2 > simd::element_count + 1 ? Count + 2 : simd::element_count + 1; + std::array unaryStorage{}; + std::array leftStorage{}; + std::array rightStorage{}; + std::array outputStorage{}; + unaryStorage.fill(guard); + leftStorage.fill(guard); + rightStorage.fill(guard); + outputStorage.fill(guard); + + auto unary = std::span(unaryStorage).subspan(1, Count); + auto left = std::span(leftStorage).subspan(1, Count); + auto right = std::span(rightStorage).subspan(1, Count); + auto output = std::span(outputStorage).subspan(1, Count); + for (std::size_t index = 0; index < Count; ++index) + { + unary[index] = static_cast(index * 7 + 5); + left[index] = static_cast(index * 7 + 50); + right[index] = static_cast(index + 3); + } + + simd::transform(unary, Add17Transform{}); + + for (std::size_t index = 0; index < Count; ++index) + REQUIRE(unary[index] == static_cast(index * 7 + 22)); + REQUIRE(unaryStorage.front() == guard); + REQUIRE(unaryStorage[Count + 1] == guard); + + simd::transform(std::span(left), output, Subtract13Transform{}); + + for (std::size_t index = 0; index < Count; ++index) + REQUIRE(output[index] == static_cast(index * 7 + 37)); + REQUIRE(outputStorage.front() == guard); + REQUIRE(outputStorage[Count + 1] == guard); + + std::fill(output.begin(), output.end(), guard); + simd::transform(std::span(left), std::span(right), output, SubtractTransform{}); + + for (std::size_t index = 0; index < Count; ++index) + REQUIRE(output[index] == static_cast(index * 6 + 47)); + REQUIRE(outputStorage.front() == guard); + REQUIRE(outputStorage[Count + 1] == guard); } /** * @brief Verifies public transform overloads across empty, tail, full-register, and multi-register extents. * @tparam Width SIMD register width in bits. */ -template -void require_transform_overload_contract() +template void require_transform_overload_contract() { - constexpr std::size_t laneCount = Api::element_count; - require_transform_overload_case(); - require_transform_overload_case(); - require_transform_overload_case(); - require_transform_overload_case(); - require_transform_overload_case(); + constexpr std::size_t laneCount = Api::element_count; + require_transform_overload_case(); + require_transform_overload_case(); + require_transform_overload_case(); + require_transform_overload_case(); + require_transform_overload_case(); } -template -constexpr auto movemask_test_bytes() +template constexpr auto movemask_test_bytes() { - std::array bytes{}; - for (std::size_t index = 0; index < bytes.size(); ++index) - bytes[index] = static_cast((index * 19u) | (index % 3u == 1u ? 0u : 0x80u)); - return bytes; + std::array bytes{}; + for (std::size_t index = 0; index < bytes.size(); ++index) + bytes[index] = static_cast((index * 19u) | (index % 3u == 1u ? 0u : 0x80u)); + return bytes; } -template -constexpr auto movemask_test_values() +template constexpr auto movemask_test_values() { - using simd = Api; - constexpr auto bytes = movemask_test_bytes(); - static_assert(sizeof(bytes) == sizeof(std::array)); - return std::bit_cast>(bytes); + using simd = Api; + constexpr auto bytes = movemask_test_bytes(); + static_assert(sizeof(bytes) == sizeof(std::array)); + return std::bit_cast>(bytes); } -template -constexpr auto expected_byte_movemask() +template constexpr auto expected_byte_movemask() { - using simd = Api; - constexpr auto bytes = movemask_test_bytes(); - typename simd::mask_t result = 0; - for (std::size_t index = 0; index < bytes.size(); ++index) - result |= static_cast((bytes[index] >> 7) & 1u) << index; - return result; + using simd = Api; + constexpr auto bytes = movemask_test_bytes(); + typename simd::mask_t result = 0; + for (std::size_t index = 0; index < bytes.size(); ++index) + result |= static_cast((bytes[index] >> 7) & 1u) << index; + return result; } -template -constexpr auto expected_slim_movemask() +template constexpr auto expected_slim_movemask() { - using simd = Api; - constexpr auto bytes = movemask_test_bytes(); - typename simd::mask_t result = 0; - for (std::size_t index = 0; index < simd::element_count; ++index) - { - const std::size_t sign_byte = (index + 1) * sizeof(Element) - 1; - result |= static_cast((bytes[sign_byte] >> 7) & 1u) << index; - } - return result; + using simd = Api; + constexpr auto bytes = movemask_test_bytes(); + typename simd::mask_t result = 0; + for (std::size_t index = 0; index < simd::element_count; ++index) + { + const std::size_t sign_byte = (index + 1) * sizeof(Element) - 1; + result |= static_cast((bytes[sign_byte] >> 7) & 1u) << index; + } + return result; } -template -void require_movemask_contract() +template void require_movemask_contract() { - using simd = Api; - const auto value = simd::construct(movemask_test_values()); - REQUIRE(simd::movemask(value) == expected_byte_movemask()); - REQUIRE(simd::movemask_slim(value) == expected_slim_movemask()); + using simd = Api; + const auto value = simd::construct(movemask_test_values()); + REQUIRE(simd::movemask(value) == expected_byte_movemask()); + REQUIRE(simd::movemask_slim(value) == expected_slim_movemask()); } -template -void require_supported_movemask_matrix() +template void require_supported_movemask_matrix() { - require_movemask_contract(); - require_movemask_contract(); - require_movemask_contract(); - require_movemask_contract(); - require_movemask_contract(); - require_movemask_contract(); - require_movemask_contract(); - require_movemask_contract(); - require_movemask_contract(); - require_movemask_contract(); + require_movemask_contract(); + require_movemask_contract(); + require_movemask_contract(); + require_movemask_contract(); + require_movemask_contract(); + require_movemask_contract(); + require_movemask_contract(); + require_movemask_contract(); + require_movemask_contract(); + require_movemask_contract(); } /** * @brief Compares constant evaluation with optimized runtime dispatch using volatile-derived inputs. * @tparam Width SIMD register width in bits. */ -template -void require_constexpr_runtime_parity() +template void require_constexpr_runtime_parity() { using simd = Api; constexpr auto lhsConstant = Constexpr::lane_values(); @@ -526,8 +657,7 @@ void require_constexpr_runtime_parity() * @tparam Width The Api register width. * @tparam Element The signed or unsigned integer lane type. */ -template -void require_extrema_position_contract() +template void require_extrema_position_contract() { using simd = Api; std::array values{}; @@ -573,8 +703,7 @@ void require_extrema_position_contract() * * @tparam Width The Api register width. */ -template -void require_integer_extrema_position_matrix() +template void require_integer_extrema_position_matrix() { require_extrema_position_contract(); require_extrema_position_contract(); @@ -590,8 +719,7 @@ void require_integer_extrema_position_matrix() * * @tparam Width The Api register width. */ -template -void require_64bit_arithmetic_contract() +template void require_64bit_arithmetic_contract() { using signed_simd = Api; const auto signed_value = signed_simd::set1(-9); @@ -616,14 +744,70 @@ void require_64bit_arithmetic_contract() REQUIRE(unsigned_simd::to_array(unsigned_simd::max(unsigned_value, unsigned_divisor))[0] == 0x8000'0000'0000'0003ULL); } +/** + * @brief Verifies scalar remainder semantics for every lane of one integer specialization. + * @tparam Width Native register width in bits. + * @tparam Element Signed or unsigned integer lane type. + */ +template void require_integer_remainder_contract() +{ + using simd = Api; + std::array lhs{}; + std::array rhs{}; + std::array expected{}; + for (std::size_t index = 0; index < simd::element_count; ++index) + { + if constexpr (std::is_signed_v) + { + const auto magnitude = static_cast(17 + index * 3); + const auto divisor = static_cast(2 + index % 5); + lhs[index] = index % 2 == 0 ? static_cast(-magnitude) : magnitude; + rhs[index] = index % 3 == 0 ? static_cast(-divisor) : divisor; + } + else + { + lhs[index] = static_cast(20 + index * 7); + rhs[index] = static_cast(2 + index % 5); + } + } + + if constexpr (std::is_signed_v) + { + lhs.back() = std::numeric_limits::lowest(); + rhs.back() = static_cast(3); + } + else + { + lhs.back() = std::numeric_limits::max(); + rhs.back() = static_cast(7); + } + + for (std::size_t index = 0; index < simd::element_count; ++index) + expected[index] = static_cast(lhs[index] % rhs[index]); + + REQUIRE(simd::to_array(simd::modulus(simd::construct(lhs), simd::construct(rhs))) == expected); +} + +/** @brief Verifies scalar remainder semantics for every integer element type at one register width. */ +template void require_integer_remainder_matrix() +{ + require_integer_remainder_contract(); + require_integer_remainder_contract(); + require_integer_remainder_contract(); + require_integer_remainder_contract(); + require_integer_remainder_contract(); + require_integer_remainder_contract(); + require_integer_remainder_contract(); + require_integer_remainder_contract(); +} + /** * @brief Verifies arithmetic, bitwise, lane-access, and shift behavior for one integer Api specialization. * * @tparam Width The Api register width. * @tparam Element The signed or unsigned integer lane type. */ -template -void require_integer_operation_contract() +template void require_integer_operation_contract() { using simd = Api; using unsigned_t = std::make_unsigned_t; @@ -683,9 +867,9 @@ void require_integer_operation_contract() if constexpr (std::is_signed_v) REQUIRE(simd::to_array(simd::shift_right_arithmetic(absolute_source, 1)) == simd::to_array(simd::set1(-4))); - REQUIRE(simd::get_element(left, 0) == lhs[0]); + REQUIRE(simd::extract_slow(left, 0) == lhs[0]); const auto replacement = static_cast(42); - const auto replaced = simd::set_element(left, static_cast(simd::element_count - 1), replacement); + const auto replaced = simd::insert_slow(left, replacement, static_cast(simd::element_count - 1)); auto expected_replaced = lhs; expected_replaced.back() = replacement; REQUIRE(simd::to_array(replaced) == expected_replaced); @@ -696,8 +880,7 @@ void require_integer_operation_contract() * * @tparam Width The Api register width. */ -template -void require_integer_operation_matrix() +template void require_integer_operation_matrix() { require_integer_operation_contract(); require_integer_operation_contract(); @@ -715,8 +898,7 @@ void require_integer_operation_matrix() * @tparam Width The Api register width. * @tparam Element The floating-point lane type. */ -template -void require_floating_operation_contract() +template void require_floating_operation_contract() { using simd = Api; using bits_t = std::conditional_t; @@ -757,8 +939,8 @@ void require_floating_operation_contract() REQUIRE(simd::to_array(simd::max(left, right)) == maximum); REQUIRE(simd::to_array(simd::absolute(left)) == absolute); REQUIRE(simd::to_array(simd::negate(left)) == negated); - REQUIRE(simd::get_element(left, 0) == lhs[0]); - const auto replaced = simd::set_element(left, static_cast(simd::element_count - 1), static_cast(-9.25)); + REQUIRE(simd::extract_slow(left, 0) == lhs[0]); + const auto replaced = simd::insert_slow(left, static_cast(-9.25), static_cast(simd::element_count - 1)); auto expected_replaced = lhs; expected_replaced.back() = static_cast(-9.25); REQUIRE(simd::to_array(replaced) == expected_replaced); @@ -794,8 +976,7 @@ void require_floating_operation_contract() * * @tparam Width The Api register width. */ -template -void require_floating_operation_matrix() +template void require_floating_operation_matrix() { require_floating_operation_contract(); require_floating_operation_contract(); @@ -808,13 +989,11 @@ void require_floating_operation_matrix() * * @tparam Width The Api register width. */ -template -void require_unsigned_32bit_contract() +template void require_unsigned_32bit_contract() { using integers = Api; using floats = Api; - constexpr std::array numerators{ - 0, 1, 7, 0x7FFF'FFFFU, 0x8000'0000U, 0xFFFF'FFFFU, 4'000'000'001U, 10}; + constexpr std::array numerators{0, 1, 7, 0x7FFF'FFFFU, 0x8000'0000U, 0xFFFF'FFFFU, 4'000'000'001U, 10}; constexpr std::array divisors{1, 1, 3, 7, 2, 65'535, 3, 4}; std::array lhs{}; std::array rhs{}; @@ -841,8 +1020,7 @@ void require_unsigned_32bit_contract() * * @tparam Width The Api register width. */ -template -void require_uint64_multiply_add_adjacent_contract() +template void require_uint64_multiply_add_adjacent_contract() { using simd = Api; std::array lhs{}; @@ -865,8 +1043,7 @@ void require_uint64_multiply_add_adjacent_contract() * * @tparam Width The Api register width. */ -template -void require_signed_32bit_conversion_contract() +template void require_signed_32bit_conversion_contract() { using integers = Api; using floats = Api; @@ -891,8 +1068,7 @@ void require_signed_32bit_conversion_contract() * * @tparam Width The Api register width. */ -template -void require_transform_pack_type_matrix() +template void require_transform_pack_type_matrix() { require_transform_pack_mask_contract::element_count + 3>(); require_transform_pack_mask_contract::element_count + 3>(); @@ -912,8 +1088,7 @@ void require_transform_pack_type_matrix() * @param value Raw register produced by the documented invocation. * @param expected Values shown in the documentation. */ -template -void require_documented_register(const Vector value, const Expected& expected) +template void require_documented_register(const Vector value, const Expected &expected) { const auto actual = Simd::to_array(value); STATIC_REQUIRE(std::tuple_size_v == std::tuple_size_v); diff --git a/tests/UInt128.tests.cpp b/tests/UInt128.tests.cpp index 78e1cb7..cfefa28 100644 --- a/tests/UInt128.tests.cpp +++ b/tests/UInt128.tests.cpp @@ -2,6 +2,9 @@ #ifndef SIMDLIB_EXPECT_CARRY_PATH #define SIMDLIB_EXPECT_CARRY_PATH -1 #endif +#ifndef SIMDLIB_TEST_CONSTEXPR_ASSERTIONS +#define SIMDLIB_TEST_CONSTEXPR_ASSERTIONS 0 +#endif #if SIMDLIB_EXPECT_CARRY_PATH == 1 #if !SIMDLIB_USE_COMPILER_CARRY_INTRINSICS || !SIMDLIB_COMPILER_MSVC || !defined(_M_X64) @@ -36,7 +39,7 @@ struct words128 final std::uint64_t low = 0; std::uint64_t high = 0; - friend constexpr bool operator==(const words128&, const words128&) noexcept = default; + friend constexpr bool operator==(const words128 &, const words128 &) noexcept = default; }; /** @brief Describes one heterogeneous signed-integral comparison contract. */ @@ -82,10 +85,7 @@ struct bit_ceil_case final #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif -[[nodiscard]] uint128_t deprecated_extract( - const uint128_t value, - const std::uint8_t length, - const std::uint8_t start) noexcept +[[nodiscard]] uint128_t deprecated_extract(const uint128_t value, const std::uint8_t length, const std::uint8_t start) noexcept { return value.extract(length, start); } @@ -175,79 +175,125 @@ constexpr bool constexpr_contract() noexcept const uint128_t lhs{0xFEDC'BA98'7654'3210ULL, 0x0123'4567'89AB'CDEFULL}; const uint128_t rhs{0x1111'2222'3333'4444ULL, 0x5555'6666'7777'8888ULL}; - if (words(lhs + rhs) != add_words(words(lhs), words(rhs))) return false; - if (words(lhs - rhs) != subtract_words(words(lhs), words(rhs))) return false; + if (words(lhs + rhs) != add_words(words(lhs), words(rhs))) + return false; + if (words(lhs - rhs) != subtract_words(words(lhs), words(rhs))) + return false; uint128_t compound = lhs; compound += rhs; - if (compound != lhs + rhs) return false; + if (compound != lhs + rhs) + return false; compound -= rhs; - if (compound != lhs) return false; - if ((lhs & rhs) != uint128_t{lhs.low() & rhs.low(), lhs.high() & rhs.high()}) return false; - if ((lhs | rhs) != uint128_t{lhs.low() | rhs.low(), lhs.high() | rhs.high()}) return false; - if ((lhs ^ rhs) != uint128_t{lhs.low() ^ rhs.low(), lhs.high() ^ rhs.high()}) return false; - if ((~lhs) != uint128_t{~lhs.low(), ~lhs.high()}) return false; + if (compound != lhs) + return false; + if ((lhs & rhs) != uint128_t{lhs.low() & rhs.low(), lhs.high() & rhs.high()}) + return false; + if ((lhs | rhs) != uint128_t{lhs.low() | rhs.low(), lhs.high() | rhs.high()}) + return false; + if ((lhs ^ rhs) != uint128_t{lhs.low() ^ rhs.low(), lhs.high() ^ rhs.high()}) + return false; + if ((~lhs) != uint128_t{~lhs.low(), ~lhs.high()}) + return false; compound = lhs; compound &= rhs; compound |= uint128_t{0x10, 0x20}; compound ^= uint128_t{0x01, 0x02}; - if (compound != (((lhs & rhs) | uint128_t{0x10, 0x20}) ^ uint128_t{0x01, 0x02})) return false; + if (compound != (((lhs & rhs) | uint128_t{0x10, 0x20}) ^ uint128_t{0x01, 0x02})) + return false; constexpr std::array shifts{0, 1, 63, 64, 65, 127, 128, 129, 255, 256}; for (const unsigned shift : shifts) { - if (words(lhs << shift) != shift_left_words(words(lhs), shift)) return false; - if (words(lhs >> shift) != shift_right_words(words(lhs), shift)) return false; + if (words(lhs << shift) != shift_left_words(words(lhs), shift)) + return false; + if (words(lhs >> shift) != shift_right_words(words(lhs), shift)) + return false; compound = lhs; compound <<= shift; - if (compound != lhs << shift) return false; + if (compound != lhs << shift) + return false; compound = lhs; compound >>= shift; - if (compound != lhs >> shift) return false; + if (compound != lhs >> shift) + return false; } - if ((lhs << -4) != lhs || (lhs >> -4) != lhs) return false; + if ((lhs << -4) != lhs || (lhs >> -4) != lhs) + return false; compound = uint128_t{std::numeric_limits::max(), 7}; const uint128_t beforeIncrement = compound++; - if (beforeIncrement != uint128_t{std::numeric_limits::max(), 7} || compound != uint128_t{0, 8}) return false; + if (beforeIncrement != uint128_t{std::numeric_limits::max(), 7} || compound != uint128_t{0, 8}) + return false; const uint128_t beforeDecrement = compound--; - if (beforeDecrement != uint128_t{0, 8} || compound != uint128_t{std::numeric_limits::max(), 7}) return false; - if (++compound != uint128_t{0, 8}) return false; - if (--compound != uint128_t{std::numeric_limits::max(), 7}) return false; - if (-uint128_t{1} != std::numeric_limits::max()) return false; - if (lhs.abs_diff(rhs) != (lhs > rhs ? lhs - rhs : rhs - lhs)) return false; - - if (!(lhs < rhs) || !(uint128_t{7} == 7u) || !(uint128_t{7} > -1)) return false; - if (static_cast(lhs) != 0x7654'3210U) return false; - if (!static_cast(lhs) || static_cast(uint128_t{})) return false; - if (lhs.getBlock(0) != lhs.low() || lhs.getBlock(1) != lhs.high()) return false; - - if (popcount(lhs) != std::popcount(lhs.low()) + std::popcount(lhs.high())) return false; - if (countr_zero(uint128_t{}) != 128 || countl_zero(uint128_t{}) != 128) return false; - if (countr_one(std::numeric_limits::max()) != 128) return false; - if (countl_one(std::numeric_limits::max()) != 128) return false; - if (bit_width(uint128_t{0, 1}) != 65) return false; - if (bit_floor(uint128_t{0, 3}) != uint128_t{0, 2}) return false; - if (bit_ceil(uint128_t{0, 3}) != uint128_t{0, 4}) return false; - if (!has_single_bit(uint128_t{0, 8}) || has_single_bit(uint128_t{3})) return false; - - if (uint128_t::create_mask(0) != uint128_t{}) return false; - if (uint128_t::create_mask(64) != uint128_t{~std::uint64_t{0}, 0}) return false; - if (uint128_t::create_mask(128) != std::numeric_limits::max()) return false; - if (uint128_t::create_mask<5>(62) != (uint128_t::create_mask(5) << 62)) return false; - if (Bmi::blsi(lhs) != (lhs & -lhs)) return false; - if (Bmi::blsr(lhs) != (lhs & (lhs - uint128_t{1}))) return false; - if (Bmi::blsmsk(lhs) != (lhs ^ (lhs - uint128_t{1}))) return false; - if (Bmi::bzhi(lhs, 65) != (lhs & uint128_t::create_mask(65))) return false; - if (Bmi::andn(lhs, rhs) != (~lhs & rhs)) return false; - if (Bmi::bextr(lhs, 17, 61) != ((lhs >> 61) & uint128_t::create_mask(17))) return false; - - if (42_u128 != uint128_t{42}) return false; - if (std::hash{}(lhs) != static_cast(lhs.low() ^ lhs.high())) return false; + if (beforeDecrement != uint128_t{0, 8} || compound != uint128_t{std::numeric_limits::max(), 7}) + return false; + if (++compound != uint128_t{0, 8}) + return false; + if (--compound != uint128_t{std::numeric_limits::max(), 7}) + return false; + if (-uint128_t{1} != std::numeric_limits::max()) + return false; + if (lhs.abs_diff(rhs) != (lhs > rhs ? lhs - rhs : rhs - lhs)) + return false; + + if (!(lhs < rhs) || !(uint128_t{7} == 7u) || !(uint128_t{7} > -1)) + return false; + if (static_cast(lhs) != 0x7654'3210U) + return false; + if (!static_cast(lhs) || static_cast(uint128_t{})) + return false; + if (lhs.getBlock(0) != lhs.low() || lhs.getBlock(1) != lhs.high()) + return false; + + if (popcount(lhs) != std::popcount(lhs.low()) + std::popcount(lhs.high())) + return false; + if (countr_zero(uint128_t{}) != 128 || countl_zero(uint128_t{}) != 128) + return false; + if (countr_one(std::numeric_limits::max()) != 128) + return false; + if (countl_one(std::numeric_limits::max()) != 128) + return false; + if (bit_width(uint128_t{0, 1}) != 65) + return false; + if (bit_floor(uint128_t{0, 3}) != uint128_t{0, 2}) + return false; + if (bit_ceil(uint128_t{0, 3}) != uint128_t{0, 4}) + return false; + if (!has_single_bit(uint128_t{0, 8}) || has_single_bit(uint128_t{3})) + return false; + + if (uint128_t::create_mask(0) != uint128_t{}) + return false; + if (uint128_t::create_mask(64) != uint128_t{~std::uint64_t{0}, 0}) + return false; + if (uint128_t::create_mask(128) != std::numeric_limits::max()) + return false; + if (uint128_t::create_mask<5>(62) != (uint128_t::create_mask(5) << 62)) + return false; + if (Bmi::blsi(lhs) != (lhs & -lhs)) + return false; + if (Bmi::blsr(lhs) != (lhs & (lhs - uint128_t{1}))) + return false; + if (Bmi::blsmsk(lhs) != (lhs ^ (lhs - uint128_t{1}))) + return false; + if (Bmi::bzhi(lhs, 65) != (lhs & uint128_t::create_mask(65))) + return false; + if (Bmi::andn(lhs, rhs) != (~lhs & rhs)) + return false; + if (Bmi::bextr(lhs, 17, 61) != ((lhs >> 61) & uint128_t::create_mask(17))) + return false; + + if (42_u128 != uint128_t{42}) + return false; + if (std::hash{}(lhs) != static_cast(lhs.low() ^ lhs.high())) + return false; static_assert(std::numeric_limits::digits == 128 && std::numeric_limits::is_modulo); return true; } +#if SIMDLIB_TEST_CONSTEXPR_ASSERTIONS static_assert(constexpr_contract()); +#endif static_assert(sizeof(uint128_t) == 16); static_assert(alignof(uint128_t) == 16); static_assert(std::is_standard_layout_v); @@ -261,7 +307,7 @@ struct operation_snapshot final std::uint64_t narrowed = 0; std::size_t hash = 0; - friend constexpr bool operator==(const operation_snapshot&, const operation_snapshot&) noexcept = default; + friend constexpr bool operator==(const operation_snapshot &, const operation_snapshot &) noexcept = default; }; [[nodiscard]] constexpr operation_snapshot snapshot(uint128_t lhs, const uint128_t rhs) noexcept @@ -330,8 +376,8 @@ struct operation_snapshot final }, { lhs == rhs, - lhs < rhs, - lhs > rhs, + lhs + rhs, static_cast(lhs), has_single_bit(lhs), std::numeric_limits::is_modulo, @@ -346,7 +392,7 @@ constexpr uint128_t snapshot_rhs{0x1111'2222'3333'4444ULL, 0x5555'6666'7777'8888 constexpr operation_snapshot constant_snapshot = snapshot(snapshot_lhs, snapshot_rhs); static_assert(constant_snapshot == snapshot(snapshot_lhs, snapshot_rhs)); -void mix_digest(std::uint64_t& digest, const uint128_t value) noexcept +void mix_digest(std::uint64_t &digest, const uint128_t value) noexcept { digest ^= value.low(); digest *= 1099511628211ULL; @@ -383,14 +429,7 @@ TEST_CASE("uint128 selected carry and borrow implementation executes with volati TEST_CASE("uint128 carry and borrow propagation matches the two-word oracle", "[simdlib][uint128][carry]") { constexpr std::array values{ - 0, - 1, - 2, - 0x7FFF'FFFF'FFFF'FFFFULL, - 0x8000'0000'0000'0000ULL, - 0xFFFF'FFFF'FFFF'FFFEULL, - 0xFFFF'FFFF'FFFF'FFFFULL, - 0xA5A5'5A5A'1234'FEDCULL}; + 0, 1, 2, 0x7FFF'FFFF'FFFF'FFFFULL, 0x8000'0000'0000'0000ULL, 0xFFFF'FFFF'FFFF'FFFEULL, 0xFFFF'FFFF'FFFF'FFFFULL, 0xA5A5'5A5A'1234'FEDCULL}; for (const auto lhs : values) { for (const auto rhs : values) @@ -434,7 +473,7 @@ TEST_CASE("uint128 integral construction and heterogeneous comparisons are expli heterogeneous_comparison_case{uint128_t{43}, 42, false, std::strong_ordering::greater}, heterogeneous_comparison_case{uint128_t{0, 1}, std::numeric_limits::max(), false, std::strong_ordering::greater}, }; - for (const auto& test : cases) + for (const auto &test : cases) { volatile std::uint64_t low = test.lhs.low(); volatile std::uint64_t high = test.lhs.high(); @@ -461,14 +500,10 @@ TEST_CASE("uint128 deprecated extraction remains compatible with Bmi bextr at bo volatile std::uint64_t sourceHigh = 0xFEDC'BA98'7654'3210ULL; const uint128_t source{sourceLow, sourceHigh}; const std::array cases{ - extraction_case{0, 0, {}}, - extraction_case{1, 127, uint128_t{1}}, - extraction_case{1, 128, {}}, - extraction_case{8, 200, {}}, - extraction_case{12, 60, uint128_t{0x100}}, - extraction_case{16, 120, uint128_t{0xFE}}, + extraction_case{0, 0, {}}, extraction_case{1, 127, uint128_t{1}}, extraction_case{1, 128, {}}, + extraction_case{8, 200, {}}, extraction_case{12, 60, uint128_t{0x100}}, extraction_case{16, 120, uint128_t{0xFE}}, }; - for (const auto& test : cases) + for (const auto &test : cases) { volatile std::uint8_t length = test.length; volatile std::uint8_t start = test.start; @@ -548,8 +583,7 @@ TEST_CASE("uint128 public integer surface remains constexpr-equivalent at runtim volatile std::uint64_t lhsHigh = snapshot_lhs.high(); volatile std::uint64_t rhsLow = snapshot_rhs.low(); volatile std::uint64_t rhsHigh = snapshot_rhs.high(); - const operation_snapshot runtimeSnapshot = snapshot( - uint128_t{lhsLow, lhsHigh}, uint128_t{rhsLow, rhsHigh}); + const operation_snapshot runtimeSnapshot = snapshot(uint128_t{lhsLow, lhsHigh}, uint128_t{rhsLow, rhsHigh}); CHECK(runtimeSnapshot == constant_snapshot); CHECK(std::numeric_limits::min() == uint128_t{}); CHECK(std::numeric_limits::lowest() == uint128_t{}); @@ -577,7 +611,7 @@ TEST_CASE("uint128 bit ceil covers identity rounding and overflow boundaries", " bit_ceil_case{uint128_t{1, std::uint64_t{1} << 63}, uint128_t{}}, bit_ceil_case{std::numeric_limits::max(), uint128_t{}}, }; - for (const auto& test : cases) + for (const auto &test : cases) { volatile std::uint64_t low = test.value.low(); volatile std::uint64_t high = test.value.high(); diff --git a/tests/availability/ApiDisabledProbe.cpp b/tests/availability/ApiDisabledProbe.cpp index fb3e6cb..97255d1 100644 --- a/tests/availability/ApiDisabledProbe.cpp +++ b/tests/availability/ApiDisabledProbe.cpp @@ -12,9 +12,6 @@ #include -template -concept HasNativeApi = requires { typename SimdLib::NativeApi; }; - static_assert(!SimdLib::is_api_available_v<128, int>); static_assert(!SimdLib::is_api_available_v<256, float>); -static_assert(!HasNativeApi); +static_assert(!SimdLib::NativeApiAvailable); diff --git a/tests/availability/ApiEnabledProbe.cpp b/tests/availability/ApiEnabledProbe.cpp index d89e1e2..a8ca46a 100644 --- a/tests/availability/ApiEnabledProbe.cpp +++ b/tests/availability/ApiEnabledProbe.cpp @@ -3,11 +3,10 @@ #include #include -template -consteval bool specialization_available() +template consteval bool specialization_available() { - using simd = SimdLib::Api; - return sizeof(typename simd::vector_t) == Width / 8 && simd::element_count == Width / (sizeof(Element) * 8); + using simd = SimdLib::Api; + return sizeof(typename simd::vector_t) == Width / 8 && simd::element_count == Width / (sizeof(Element) * 8); } /** @@ -15,11 +14,10 @@ consteval bool specialization_available() * @tparam Element SIMD lane element type. * @return `true` when `NativeApi` uses 256-bit registers. */ -template -consteval bool native_api_selects_widest_register() +template consteval bool native_api_selects_widest_register() { - using simd = SimdLib::NativeApi; - return simd::register_width == 256; + using simd = SimdLib::NativeApi; + return simd::register_width == 256; } static_assert(specialization_available<128, std::int8_t>()); @@ -61,19 +59,15 @@ static_assert(!SimdLib::is_api_available_v<128, long double>); static_assert(SimdLib::Api<128, std::int32_t>::element_width == 32); static_assert(SimdLib::Api<128, float>::element_width == 32); -static_assert(requires(SimdLib::Api<128, std::int32_t>::int_vector_t value) { - SimdLib::Api<128, std::int32_t>::convert_to_float(value); -}); -static_assert(requires(SimdLib::Api<128, float>::float_vector_t value) { - SimdLib::Api<128, float>::convert_to_int(value); -}); +static_assert(requires(SimdLib::Api<128, std::int32_t>::int_vector_t value) { SimdLib::Api<128, std::int32_t>::convert_to_float(value); }); +static_assert(requires(SimdLib::Api<128, float>::float_vector_t value) { SimdLib::Api<128, float>::convert_to_int(value); }); consteval bool constexpr_paths_match() { - using simd = SimdLib::Api<128, std::uint64_t>; - constexpr auto input = simd::setr(1, 2); - constexpr auto shifted = simd::template bit_shift_left<64>(input); - return simd::to_array(shifted) == std::array{0, 1}; + using simd = SimdLib::Api<128, std::uint64_t>; + constexpr auto input = simd::setr(1, 2); + constexpr auto shifted = simd::template shift_bits_left<64>(input); + return simd::to_array(shifted) == std::array{0, 1}; } static_assert(constexpr_paths_match()); diff --git a/tests/availability/CompleteRegisterShiftProbe.cpp b/tests/availability/CompleteRegisterShiftProbe.cpp new file mode 100644 index 0000000..521fe8f --- /dev/null +++ b/tests/availability/CompleteRegisterShiftProbe.cpp @@ -0,0 +1,67 @@ +#define SIMDLIB_HAS_SSE42 1 +#define SIMDLIB_HAS_AVX2 1 +#include +#include +#include + +#include + +namespace +{ + +/** @brief Reports whether an Api accepts an unsuffixed runtime byte count. */ +template +concept api_accepts_runtime_byte_shift = requires(typename api_t::int_vector_t value, int count) { + api_t::shift_bytes_left(value, count); + api_t::shift_bytes_right(value, count); +}; + +/** @brief Reports whether an implementation accepts an unsuffixed runtime byte count. */ +template +concept implementation_accepts_runtime_byte_shift = requires(typename implementation_t::int_vector_t value, int count) { + implementation_t::shift_bytes_left(value, count); + implementation_t::shift_bytes_right(value, count); +}; + +/** @brief Reports whether a Register accepts an unsuffixed runtime byte count. */ +template +concept register_accepts_runtime_byte_shift = requires(register_t value, int count) { + value.shift_bytes_left(count); + value.shift_bytes_right(count); +}; + +/** @brief Verifies immediate byte-shift availability for one integral lane type. */ +template consteval bool integral_availability_contract() +{ + using api128 = SimdLib::Api<128, element_t>; + using api256 = SimdLib::Api<256, element_t>; + using implementation128 = SimdLib::Detail::SimdMappings<128, element_t>; + using implementation256 = SimdLib::Detail::SimdMappings<256, element_t>; + using register128 = SimdLib::Register; + using register256 = SimdLib::Register; + return SimdLib::IApi::ShiftBytesLeft && SimdLib::IApi::ShiftBytesRight && SimdLib::IApi::ShiftBytesLeft && + SimdLib::IApi::ShiftBytesRight && SimdLib::IImpl::ShiftBytesLeft && + SimdLib::IImpl::ShiftBytesRight && SimdLib::IImpl::ShiftBytesLeft && + SimdLib::IImpl::ShiftBytesRight && SimdLib::IRegister::ShiftBytesLeft && + SimdLib::IRegister::ShiftBytesRight && SimdLib::IRegister::ShiftBytesLeft && + SimdLib::IRegister::ShiftBytesRight && !api_accepts_runtime_byte_shift && !api_accepts_runtime_byte_shift && + !implementation_accepts_runtime_byte_shift && !implementation_accepts_runtime_byte_shift && + !register_accepts_runtime_byte_shift && !register_accepts_runtime_byte_shift; +} + +} // namespace + +static_assert(integral_availability_contract()); +static_assert(integral_availability_contract()); +static_assert(integral_availability_contract()); +static_assert(integral_availability_contract()); +static_assert(integral_availability_contract()); +static_assert(integral_availability_contract()); +static_assert(integral_availability_contract()); +static_assert(integral_availability_contract()); +static_assert(!SimdLib::IApi::ShiftBytesLeft, 1>); +static_assert(!SimdLib::IApi::ShiftBytesRight, 1>); +static_assert(!SimdLib::IRegister::ShiftBytesLeft, 1>); +static_assert(!SimdLib::IRegister::ShiftBytesRight, 1>); +static_assert(!SimdLib::IApi::ShiftBytesLeft, -1>); +static_assert(!SimdLib::IRegister::ShiftBytesRight, -1>); \ No newline at end of file diff --git a/tests/availability/ImmediateControlSlowPathProbe.cpp b/tests/availability/ImmediateControlSlowPathProbe.cpp new file mode 100644 index 0000000..e8d51b0 --- /dev/null +++ b/tests/availability/ImmediateControlSlowPathProbe.cpp @@ -0,0 +1,133 @@ +#define SIMDLIB_HAS_SSE42 1 +#define SIMDLIB_HAS_AVX2 1 +#include +#include +#include + +#include +#include + +namespace +{ + +/** @brief Selects the implementation mapping for one public Api specialization. */ +template using implementation_t = SimdLib::Detail::SimdMappings; + +/** + * @brief Verifies runtime-selected lane slow paths in both public and implementation layers. + * @tparam Width SIMD register width in bits. + * @tparam Element Logical lane type. + * @return `true` when extraction and insertion slow signatures are available in both layers. + */ +template consteval bool lane_slow_paths_available() +{ + using api = SimdLib::Api; + using implementation = implementation_t; + return SimdLib::IApi::ExtractSlow && SimdLib::IApi::InsertSlow && SimdLib::IImpl::ExtractSlow && + SimdLib::IImpl::InsertSlow; +} + +/** + * @brief Verifies scalar-controlled blend slow paths in both public and implementation layers. + * @tparam Width SIMD register width in bits. + * @tparam Element Logical lane type. + * @return `true` when both slow blend signatures are available. + */ +template consteval bool blend_slow_path_available() +{ + using api = SimdLib::Api; + using implementation = implementation_t; + using vector = typename implementation::vector_t; + return SimdLib::IApi::BlendSlow && SimdLib::IImpl::BlendSlow; +} + +/** + * @brief Verifies scalar-controlled floating shuffle slow paths in both layers. + * @tparam Width SIMD register width in bits. + * @tparam Element Floating-point lane type. + * @return `true` when both slow shuffle signatures are available. + */ +template consteval bool floating_shuffle_slow_path_available() +{ + using api = SimdLib::Api; + using implementation = implementation_t; + using vector = typename implementation::vector_t; + return SimdLib::IApi::ShuffleSlow && SimdLib::IImpl::ShuffleSlow; +} + +/** + * @brief Verifies low- and high-half shuffle slow paths in both layers. + * @tparam Width SIMD register width in bits. + * @return `true` when all four slow signatures are available. + */ +template consteval bool half_shuffle_slow_paths_available() +{ + using api = SimdLib::Api; + using implementation = implementation_t; + using vector = typename implementation::vector_t; + return SimdLib::IApi::ShuffleLowSlow && SimdLib::IApi::ShuffleHighSlow && SimdLib::IImpl::ShuffleLowSlow && + SimdLib::IImpl::ShuffleHighSlow; +} + +/** + * @brief Verifies 32-bit shuffle slow paths in both layers. + * @tparam Width SIMD register width in bits. + * @return `true` when both slow signatures are available. + */ +template consteval bool shuffle_32_slow_path_available() +{ + using api = SimdLib::Api; + using implementation = implementation_t; + return SimdLib::IApi::Shuffle32Slow && SimdLib::IImpl::Shuffle32Slow; +} + +} // namespace + +#define SIMDLIB_ASSERT_LANE_SLOW_PATHS(width) \ + static_assert(lane_slow_paths_available()); \ + static_assert(lane_slow_paths_available()); \ + static_assert(lane_slow_paths_available()); \ + static_assert(lane_slow_paths_available()); \ + static_assert(lane_slow_paths_available()); \ + static_assert(lane_slow_paths_available()); \ + static_assert(lane_slow_paths_available()); \ + static_assert(lane_slow_paths_available()); \ + static_assert(lane_slow_paths_available()); \ + static_assert(lane_slow_paths_available()) + +#define SIMDLIB_ASSERT_BLEND_SLOW_PATHS(width) \ + static_assert(blend_slow_path_available()); \ + static_assert(blend_slow_path_available()); \ + static_assert(blend_slow_path_available()); \ + static_assert(blend_slow_path_available()); \ + static_assert(blend_slow_path_available()); \ + static_assert(blend_slow_path_available()) + +SIMDLIB_ASSERT_LANE_SLOW_PATHS(128); +SIMDLIB_ASSERT_LANE_SLOW_PATHS(256); +SIMDLIB_ASSERT_BLEND_SLOW_PATHS(128); +SIMDLIB_ASSERT_BLEND_SLOW_PATHS(256); +static_assert(floating_shuffle_slow_path_available<128, float>()); +static_assert(floating_shuffle_slow_path_available<128, double>()); +static_assert(floating_shuffle_slow_path_available<256, float>()); +static_assert(floating_shuffle_slow_path_available<256, double>()); +static_assert(half_shuffle_slow_paths_available<128>()); +static_assert(half_shuffle_slow_paths_available<256>()); +static_assert(shuffle_32_slow_path_available<128>()); +static_assert(shuffle_32_slow_path_available<256>()); +static_assert(SimdLib::IApi::ShiftBytesSlow>); +static_assert(SimdLib::IApi::ShiftBitsSlow>); +static_assert(SimdLib::IApi::ShiftBits, 1>); +static_assert(SimdLib::IImpl::ShiftBytesSlow>); +static_assert(SimdLib::IImpl::ShiftBitsSlow>); +static_assert(SimdLib::IImpl::ShiftBits, 1>); + +static_assert(SimdLib::IApi::RegisterShuffle>); +static_assert(SimdLib::IApi::RegisterShuffle>); +static_assert(SimdLib::IApi::RegisterBlend>); +static_assert(SimdLib::IApi::RegisterBlend>); +static_assert(requires(SimdLib::Api<128, std::uint32_t>::vector_t value) { SimdLib::Api<128, std::uint32_t>::shift_left(value, 1); }); +static_assert(requires(SimdLib::Api<256, std::uint32_t>::vector_t value) { SimdLib::Api<256, std::uint32_t>::shift_left(value, 1); }); + +#undef SIMDLIB_ASSERT_BLEND_SLOW_PATHS +#undef SIMDLIB_ASSERT_LANE_SLOW_PATHS \ No newline at end of file diff --git a/tests/availability/RegisterClangClFallbackExclusionProbe.cpp b/tests/availability/RegisterClangClFallbackExclusionProbe.cpp new file mode 100644 index 0000000..ba094a1 --- /dev/null +++ b/tests/availability/RegisterClangClFallbackExclusionProbe.cpp @@ -0,0 +1,18 @@ +#if !defined(__clang__) || !defined(_MSC_VER) +#error "The clang-cl fallback-exclusion probe requires clang-cl" +#endif + +#ifdef __cpp_explicit_this_parameter +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wbuiltin-macro-redefined" +#undef __cpp_explicit_this_parameter +#pragma clang diagnostic pop +#endif + +#include + +static_assert(_MSC_VER >= 1944); +static_assert(_MSVC_LANG > 202002L); +static_assert(SIMDLIB_COMPILER_CLANG == 1); +static_assert(SIMDLIB_COMPILER_MSVC == 0); +static_assert(SIMDLIB_REGISTER_INTERFACE_AVAILABLE == 0); diff --git a/tests/availability/RegisterCxx20UmbrellaProbe.cpp b/tests/availability/RegisterCxx20UmbrellaProbe.cpp new file mode 100644 index 0000000..a175dc2 --- /dev/null +++ b/tests/availability/RegisterCxx20UmbrellaProbe.cpp @@ -0,0 +1,5 @@ +#include + +static_assert(SIMDLIB_REGISTER_INTERFACE_AVAILABLE == 0); +static_assert(SIMDLIB_REQUIRE_REGISTER_INTERFACE == 0); +static_assert(SimdLib::version_major == SimdLib::Config::version_major); diff --git a/tests/availability/RegisterEnabledProbe.cpp b/tests/availability/RegisterEnabledProbe.cpp new file mode 100644 index 0000000..532f3e0 --- /dev/null +++ b/tests/availability/RegisterEnabledProbe.cpp @@ -0,0 +1,80 @@ +#include + +#if !SIMDLIB_REGISTER_INTERFACE_AVAILABLE +#error "The Register positive probe requires the computed interface availability" +#endif + +#if !SIMDLIB_REQUIRE_REGISTER_INTERFACE +#error "SimdLib::Register must publish its requirement signal" +#endif + +#if defined(__clang__) || defined(__GNUC__) +#if !defined(__cpp_explicit_this_parameter) || __cpp_explicit_this_parameter < 202110L +#error "Clang and GCC Register support must use the standard explicit-object feature macro" +#endif +#endif + +#if defined(_MSC_VER) && !defined(__clang__) && _MSVC_LANG <= 202002L +#error "Microsoft C++ Register support requires a post-C++20 language mode" +#endif + +/** @brief Exercises the explicit-object declaration forms required by Register. */ +struct RegisterExplicitObjectProbe +{ + int value; + + /** + * @brief Returns the stored value through a by-value explicit object parameter. + * @return Stored probe value. + */ + [[nodiscard]] constexpr int SIMD_FLAGS(Neither) get(this RegisterExplicitObjectProbe self) noexcept + { + return self.value; + } + + /** + * @brief Adds two probe values through a by-value explicit object operator. + * @param rhs Right operand. + * @return Sum of both probe values. + */ + [[nodiscard]] constexpr RegisterExplicitObjectProbe SIMD_FLAGS(Neither) operator+(this RegisterExplicitObjectProbe lhs, + const RegisterExplicitObjectProbe rhs) noexcept + { + return {lhs.value + rhs.value}; + } + + /** + * @brief Mutates a probe through a reference explicit object parameter. + * @param rhs Value added to the probe. + * @return Reference to the mutated probe. + */ + constexpr auto SIMD_FLAGS(Neither) operator+=(this RegisterExplicitObjectProbe &self, const RegisterExplicitObjectProbe rhs) noexcept + -> RegisterExplicitObjectProbe & + { + self.value += rhs.value; + return self; + } + + /** + * @brief Compares two probes through a by-value explicit object operator. + * @param rhs Right operand. + * @return `true` when both values are equal. + */ + [[nodiscard]] constexpr bool SIMD_FLAGS(Neither) operator==(this RegisterExplicitObjectProbe lhs, const RegisterExplicitObjectProbe rhs) noexcept + { + return lhs.value == rhs.value; + } +}; + +/** + * @brief Verifies named, arithmetic, comparison, and mutating explicit-object declarations. + * @return `true` when every declaration produces its expected value. + */ +consteval bool register_explicit_object_probe_succeeds() +{ + RegisterExplicitObjectProbe value{1}; + value += RegisterExplicitObjectProbe{2}; + return value.get() == 3 && value + RegisterExplicitObjectProbe{4} == RegisterExplicitObjectProbe{7}; +} + +static_assert(register_explicit_object_probe_succeeds()); diff --git a/tests/availability/RegisterMsvcFallbackProbe.cpp b/tests/availability/RegisterMsvcFallbackProbe.cpp new file mode 100644 index 0000000..8accad6 --- /dev/null +++ b/tests/availability/RegisterMsvcFallbackProbe.cpp @@ -0,0 +1,14 @@ +#if !defined(_MSC_VER) || defined(__clang__) +#error "The Microsoft fallback probe requires Microsoft C++" +#endif + +#ifdef __cpp_explicit_this_parameter +#undef __cpp_explicit_this_parameter +#endif + +#include + +static_assert(_MSC_VER >= 1944); +static_assert(_MSVC_LANG > 202002L); +static_assert(SIMDLIB_REGISTER_INTERFACE_AVAILABLE == 1); +static_assert(SIMDLIB_REQUIRE_REGISTER_INTERFACE == 1); diff --git a/tests/cmake/artifact_aggregates/CMakeLists.txt b/tests/cmake/artifact_aggregates/CMakeLists.txt new file mode 100644 index 0000000..41b6b79 --- /dev/null +++ b/tests/cmake/artifact_aggregates/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.31) + +project(SimdLibArtifactAggregateFixture LANGUAGES NONE) + +if(NOT DEFINED SIMDLIB_SOURCE_DIRECTORY) + message(FATAL_ERROR "SIMDLIB_SOURCE_DIRECTORY is required") +endif() +if(NOT DEFINED SIMDLIB_ARTIFACT_FAILURE_CASE) + message(FATAL_ERROR "SIMDLIB_ARTIFACT_FAILURE_CASE is required") +endif() + +set(SIMDLIB_VALIDATION_PROFILE CUSTOM) +set(SIMDLIB_REGISTER_COMPILER_SUPPORTED OFF) +set(SIMDLIB_BUILD_REGISTER_CODEGEN_GATES OFF) +set(SIMDLIB_REGISTER_CODEGEN_MODE OFF) +include("${SIMDLIB_SOURCE_DIRECTORY}/cmake/development/ArtifactOwnership.cmake") + +if(SIMDLIB_ARTIFACT_FAILURE_CASE STREQUAL "UNOWNED") + add_custom_target(UnownedFixture) +elseif(SIMDLIB_ARTIFACT_FAILURE_CASE STREQUAL "MULTIPLE") + add_custom_target(MultipleFixture) + simdlib_register_development_target(MultipleFixture RUNTIME_VALIDATION) + simdlib_register_development_target(MultipleFixture CHECKS_VALIDATION) +elseif(SIMDLIB_ARTIFACT_FAILURE_CASE STREQUAL "EXCLUDED") + set(SIMDLIB_VALIDATION_PROFILE SANITIZER) + set(SIMDLIB_DEFAULT_CHECKS_PROBE DEBUG) + add_custom_target(ExcludedFixture) + simdlib_register_development_target(ExcludedFixture COMPILER_CONTRACT) +else() + message(FATAL_ERROR + "Unsupported fixture case ${SIMDLIB_ARTIFACT_FAILURE_CASE}") +endif() + +include("${SIMDLIB_SOURCE_DIRECTORY}/cmake/development/ArtifactAggregates.cmake") diff --git a/tests/codegen/RegisterAbi.cpp b/tests/codegen/RegisterAbi.cpp new file mode 100644 index 0000000..7950b77 --- /dev/null +++ b/tests/codegen/RegisterAbi.cpp @@ -0,0 +1,108 @@ +#include + +#include + +#if defined(__clang__) || defined(__GNUC__) +#define SIMDLIB_ABI_NOINLINE __attribute__((noinline, used)) +#elif SIMDLIB_COMPILER_MSVC +#define SIMDLIB_ABI_NOINLINE __declspec(noinline) __declspec(dllexport) +#else +#define SIMDLIB_ABI_NOINLINE __attribute__((noinline)) +#endif + +using api_type = SimdLib::Api; +using native_type = typename api_type::vector_t; +using register_type = SimdLib::Register; +using mask_type = typename register_type::mask_type; + +/** @brief Test-only one-vector predicate used to mirror RegisterMask call boundaries. */ +class AbiMask final +{ + public: + /** @brief Owns the native predicate value represented by this aggregate mirror. */ + [[maybe_unused]] native_type m_data = api_type::setzero(); +}; + +/** @brief Test-only one-vector value used to validate explicit-object call boundaries. */ +class AbiRegister final +{ + public: + /** @brief Owns the native register value represented by this aggregate mirror. */ + native_type m_data = api_type::setzero(); + + /** @brief Mirrors a unary explicit-object member boundary. */ + SIMDLIB_ABI_NOINLINE AbiRegister SIMD_FLAGS(InOut, RegisterOnly) simdlib_abi_unary(this AbiRegister value) noexcept + { + return AbiRegister{api_type::bitwise_not(value.m_data)}; + } + + /** @brief Mirrors a binary explicit-object member boundary. */ + SIMDLIB_ABI_NOINLINE AbiRegister SIMD_FLAGS(InOut, RegisterOnly) simdlib_abi_binary(this AbiRegister lhs, AbiRegister rhs) noexcept + { + return AbiRegister{api_type::add(lhs.m_data, rhs.m_data)}; + } + + /** @brief Mirrors a ternary explicit-object member boundary. */ + SIMDLIB_ABI_NOINLINE AbiRegister SIMD_FLAGS(InOut, RegisterOnly) simdlib_abi_ternary(this AbiRegister lhs, AbiRegister rhs, AbiRegister addend) noexcept + { + return AbiRegister{api_type::add(api_type::multiply(lhs.m_data, rhs.m_data), addend.m_data)}; + } + + /** @brief Mirrors a scalar-result explicit-object member boundary. */ + SIMDLIB_ABI_NOINLINE std::uint32_t SIMD_FLAGS(In, RegisterOnly) simdlib_abi_scalar(this AbiRegister value) noexcept + { + return api_type::movemask(value.m_data); + } + + /** @brief Mirrors a register-shaped mask-result explicit-object member boundary. */ + SIMDLIB_ABI_NOINLINE AbiMask SIMD_FLAGS(InOut, RegisterOnly) simdlib_abi_mask(this AbiRegister value) noexcept + { + (void)value; + return AbiMask{api_type::setzero()}; + } + + /** @brief Mirrors a native-result explicit-object member boundary. */ + SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) simdlib_abi_native(this AbiRegister value) noexcept + { + return value.m_data; + } + + /** @brief Mirrors a store explicit-object member boundary. */ + SIMDLIB_ABI_NOINLINE void SIMD_FLAGS(In) simdlib_abi_store(this AbiRegister value, float *destination) noexcept + { + api_type::store(value.m_data, std::span(destination, api_type::element_count)); + } + + /** @brief Mirrors a mutating-reference explicit-object member boundary. */ + SIMDLIB_ABI_NOINLINE auto SIMD_FLAGS(In) simdlib_abi_mutate(this AbiRegister &lhs, AbiRegister rhs) noexcept -> AbiRegister & + { + lhs.m_data = api_type::add(lhs.m_data, rhs.m_data); + return lhs; + } +}; + +/** @brief Returns a real Register across a separately compiled consumer boundary. */ +SIMDLIB_ABI_NOINLINE register_type SIMD_FLAGS(InOut, RegisterOnly) simdlib_consumer_abi_register_return(register_type lhs, register_type rhs) noexcept +{ + return register_type{api_type::add(lhs.native, rhs.native)}; +} + +/** @brief Passes a real Register across a separately compiled consumer boundary. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) simdlib_consumer_abi_register_pass(register_type value) noexcept +{ + return value.native; +} + +/** @brief Returns a real RegisterMask across a separately compiled ABI boundary. */ +SIMDLIB_ABI_NOINLINE mask_type SIMD_FLAGS(In, RegisterOnly) simdlib_consumer_abi_mask_return(register_type lhs, register_type rhs) noexcept +{ + return lhs.compare_equal(rhs); +} + +/** @brief Passes a real RegisterMask across a separately compiled ABI boundary. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(Out, RegisterOnly) simdlib_consumer_abi_mask_pass(mask_type value) noexcept +{ + return value.native; +} + +#undef SIMDLIB_ABI_NOINLINE diff --git a/tests/codegen/RegisterAbiRaw.cpp b/tests/codegen/RegisterAbiRaw.cpp new file mode 100644 index 0000000..0011b46 --- /dev/null +++ b/tests/codegen/RegisterAbiRaw.cpp @@ -0,0 +1,88 @@ +#include + +#include + +#if SIMDLIB_COMPILER_MSVC +#define SIMDLIB_ABI_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_ABI_NOINLINE __attribute__((noinline)) +#endif + +using api_type = SimdLib::Api; +using native_type = typename api_type::vector_t; + +/** @brief Raw unary ABI mirror. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_abi_unary(native_type value) noexcept +{ + return api_type::bitwise_not(value); +} + +/** @brief Raw binary ABI mirror. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_abi_binary(native_type lhs, native_type rhs) noexcept +{ + return api_type::add(lhs, rhs); +} + +/** @brief Raw ternary ABI mirror. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_abi_ternary(native_type lhs, native_type rhs, native_type addend) noexcept +{ + return api_type::add(api_type::multiply(lhs, rhs), addend); +} + +/** @brief Raw scalar-result ABI mirror. */ +SIMDLIB_ABI_NOINLINE std::uint32_t SIMD_FLAGS(In) simdlib_abi_scalar(native_type value) noexcept +{ + return api_type::movemask(value); +} + +/** @brief Raw register-shaped mask-result ABI mirror. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_abi_mask(native_type value) noexcept +{ + (void)value; + return api_type::setzero(); +} + +/** @brief Raw native-result ABI mirror. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_abi_native(native_type value) noexcept +{ + return value; +} + +/** @brief Raw store ABI mirror. */ +SIMDLIB_ABI_NOINLINE void SIMD_FLAGS(In) simdlib_abi_store(native_type value, float *destination) noexcept +{ + api_type::store(value, std::span(destination, api_type::element_count)); +} + +/** @brief Raw mutating-reference ABI mirror. */ +SIMDLIB_ABI_NOINLINE auto SIMD_FLAGS(In) simdlib_abi_mutate(native_type &lhs, native_type rhs) noexcept -> native_type & +{ + lhs = api_type::add(lhs, rhs); + return lhs; +} + +/** @brief Returns a raw vector across the Register consumer-boundary mirror. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_consumer_abi_register_return(native_type lhs, native_type rhs) noexcept +{ + return api_type::add(lhs, rhs); +} + +/** @brief Passes a raw vector across the Register consumer-boundary mirror. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_consumer_abi_register_pass(native_type value) noexcept +{ + return value; +} + +/** @brief Returns a raw predicate across a separately compiled ABI boundary. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_consumer_abi_mask_return(native_type lhs, native_type rhs) noexcept +{ + return api_type::compare_equal(lhs, rhs); +} + +/** @brief Passes a raw predicate across a separately compiled ABI boundary. */ +SIMDLIB_ABI_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_consumer_abi_mask_pass(native_type value) noexcept +{ + return value; +} + +#undef SIMDLIB_ABI_NOINLINE diff --git a/tests/codegen/RegisterCodegen.cpp b/tests/codegen/RegisterCodegen.cpp new file mode 100644 index 0000000..c677fcf --- /dev/null +++ b/tests/codegen/RegisterCodegen.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 1 +#include "RegisterCodegenFixture.h" diff --git a/tests/codegen/RegisterCodegenFixture.h b/tests/codegen/RegisterCodegenFixture.h new file mode 100644 index 0000000..a82e719 --- /dev/null +++ b/tests/codegen/RegisterCodegenFixture.h @@ -0,0 +1,522 @@ +#pragma once + +#if SIMDLIB_CODEGEN_USE_WRAPPER +#include +#else +#include +#endif + +#include +#include +#include + +#if SIMDLIB_COMPILER_MSVC +#define SIMDLIB_CODEGEN_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_CODEGEN_NOINLINE __attribute__((noinline)) +#endif + +namespace SimdLibCodegen +{ + +using api_type = SimdLib::Api; +using native_type = typename api_type::vector_t; +#if SIMDLIB_CODEGEN_USE_WRAPPER +using register_type = SimdLib::Register; +#endif +using uint_api_type = SimdLib::Api; +using uint_native_type = typename uint_api_type::vector_t; +#if SIMDLIB_CODEGEN_USE_WRAPPER +using uint_register_type = SimdLib::Register; +#endif + +#if SIMDLIB_CODEGEN_USE_WRAPPER +using value_type = register_type; +#else +using value_type = native_type; +#endif + +/** @brief Converts the fixture value to its native vector representation. */ +native_type SIMD_FLAGS(Out, RegisterOnly, ForceInline) unwrap(value_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return value.native; +#else + return value; +#endif +} + +/** @brief Converts a native vector to the fixture value representation. */ +value_type SIMD_FLAGS(In, RegisterOnly, ForceInline) wrap(native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return value_type{value}; +#else + return value; +#endif +} + +} // namespace SimdLibCodegen + +using SimdLibCodegen::native_type; +using SimdLibCodegen::value_type; + +/** @brief Opaque call boundary used to keep a register value live across a separately compiled call. */ +SIMDLIB_CODEGEN_NOINLINE void SIMD_FLAGS(In) simdlib_codegen_opaque_sink(native_type value) noexcept; + +/** @brief Forced-inline ternary expression fixture. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) simdlib_codegen_ternary(native_type lhs, native_type rhs, native_type addend) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return ((SimdLibCodegen::register_type{lhs} * SimdLibCodegen::register_type{rhs}) + SimdLibCodegen::register_type{addend}).native; +#else + return SimdLibCodegen::api_type::add(SimdLibCodegen::api_type::multiply(lhs, rhs), addend); +#endif +} + +/** @brief Compare-and-combine mask fixture. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) simdlib_codegen_mask_combine(native_type lhs, native_type rhs) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + const SimdLibCodegen::register_type left{lhs}; + const SimdLibCodegen::register_type right{rhs}; + return (left.compare_equal(right) | left.compare_greater(right)).native; +#else + return SimdLibCodegen::api_type::bitwise_or(SimdLibCodegen::api_type::compare_equal(lhs, rhs), SimdLibCodegen::api_type::compare_greater(lhs, rhs)); +#endif +} + +/** @brief Compare-and-select mask fixture. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_mask_select(native_type lhs, native_type rhs, native_type when_true, native_type when_false) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::register_type{lhs} + .compare_greater(SimdLibCodegen::register_type{rhs}) + .select(SimdLibCodegen::register_type{when_true}, SimdLibCodegen::register_type{when_false}) + .native; +#else + const native_type condition = SimdLibCodegen::api_type::compare_greater(lhs, rhs); + return SimdLibCodegen::api_type::select(condition, when_true, when_false); +#endif +} + +/** @brief Compact predicate-bit fixture. */ +SIMDLIB_CODEGEN_NOINLINE std::uint32_t SIMD_FLAGS(In, RegisterOnly) simdlib_codegen_mask_bits(native_type lhs, native_type rhs) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::register_type{lhs}.compare_equal(SimdLibCodegen::register_type{rhs}).bits(); +#else + return static_cast(SimdLibCodegen::api_type::movemask_slim(SimdLibCodegen::api_type::compare_equal(lhs, rhs))); +#endif +} + +/** @brief Any-lane predicate reduction fixture. */ +SIMDLIB_CODEGEN_NOINLINE bool SIMD_FLAGS(In, RegisterOnly) simdlib_codegen_mask_any(native_type lhs, native_type rhs) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::register_type{lhs}.compare_equal(SimdLibCodegen::register_type{rhs}).any(); +#else + return SimdLibCodegen::api_type::movemask_slim(SimdLibCodegen::api_type::compare_equal(lhs, rhs)) != 0; +#endif +} + +/** @brief All-lane predicate reduction fixture. */ +SIMDLIB_CODEGEN_NOINLINE bool SIMD_FLAGS(In, RegisterOnly) simdlib_codegen_mask_all(native_type lhs, native_type rhs) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::register_type{lhs}.compare_equal(SimdLibCodegen::register_type{rhs}).all(); +#else + constexpr std::uint32_t all_bits = (std::uint32_t{1} << SimdLibCodegen::api_type::element_count) - 1; + return static_cast(SimdLibCodegen::api_type::movemask_slim(SimdLibCodegen::api_type::compare_equal(lhs, rhs))) == all_bits; +#endif +} + +/** @brief Native-result fixture. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) simdlib_codegen_native(native_type value) noexcept +{ + return SimdLibCodegen::unwrap(SimdLibCodegen::wrap(value)); +} + +/** @brief Broadcast-reuse fixture. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(Out, RegisterOnly) simdlib_codegen_broadcast_reuse(float value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + const auto broadcast = SimdLibCodegen::register_type::broadcast(value); + return (broadcast + broadcast).native; +#else + const auto broadcast = SimdLibCodegen::api_type::set1(value); + return SimdLibCodegen::api_type::add(broadcast, broadcast); +#endif +} + +/** @brief Highest-lane observation fixture. */ +SIMDLIB_CODEGEN_NOINLINE float SIMD_FLAGS(In, RegisterOnly) simdlib_codegen_lane_last(native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::register_type{value}.template lane(); +#else + return SimdLibCodegen::api_type::template extract(SimdLibCodegen::api_type::element_count - 1)>(value); +#endif +} + +/** @brief Full-register load, operation, and store fixture. */ +SIMDLIB_CODEGEN_NOINLINE void simdlib_codegen_load_operate_store(const float *source, float *destination) noexcept +{ + constexpr auto count = SimdLibCodegen::api_type::element_count; +#if SIMDLIB_CODEGEN_USE_WRAPPER + const auto value = SimdLibCodegen::register_type::load(std::span{source, count}); + (value + value).store(std::span{destination, count}); +#else + const auto value = SimdLibCodegen::api_type::load(std::span{source, count}); + SimdLibCodegen::api_type::store(SimdLibCodegen::api_type::add(value, value), std::span{destination, count}); +#endif +} + +/** @brief Aligned full-register load/store fixture. */ +SIMDLIB_CODEGEN_NOINLINE void simdlib_codegen_aligned_transfer(const float *source, float *destination) noexcept +{ + constexpr auto count = SimdLibCodegen::api_type::element_count; +#if SIMDLIB_CODEGEN_USE_WRAPPER + SimdLibCodegen::register_type::load_aligned(std::span{source, count}).store_aligned(std::span{destination, count}); +#else + SimdLibCodegen::api_type::store_aligned(SimdLibCodegen::api_type::load_aligned(std::span{source, count}), + std::span{destination, count}); +#endif +} + +/** @brief Exact-byte load/store fixture. */ +SIMDLIB_CODEGEN_NOINLINE void simdlib_codegen_byte_transfer(const std::byte *source, std::byte *destination) noexcept +{ + constexpr auto count = SimdLibCodegen::api_type::byte_count; +#if SIMDLIB_CODEGEN_USE_WRAPPER + SimdLibCodegen::register_type::load_bytes(std::span{source, count}).store_bytes(std::span{destination, count}); +#else + SimdLibCodegen::api_type::store(SimdLibCodegen::api_type::load(std::span{source, count}), + std::span{destination, count}); +#endif +} + +/** @brief Copy/move special-member fixture. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) simdlib_codegen_special_members(native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + SimdLibCodegen::register_type first{value}; + const SimdLibCodegen::register_type second{first}; + first = second; + return first.native; +#else + native_type first = value; + const native_type second = first; + first = second; + return first; +#endif +} + +/** @brief Mutating-reference fixture. */ +SIMDLIB_CODEGEN_NOINLINE void SIMD_FLAGS(In) simdlib_codegen_mutate(native_type &lhs, native_type rhs) noexcept +{ + value_type wrapped_lhs = SimdLibCodegen::wrap(lhs); + const value_type wrapped_rhs = SimdLibCodegen::wrap(rhs); +#if SIMDLIB_CODEGEN_USE_WRAPPER + wrapped_lhs = wrapped_lhs + wrapped_rhs; +#else + wrapped_lhs = SimdLibCodegen::api_type::add(wrapped_lhs, wrapped_rhs); +#endif + lhs = SimdLibCodegen::unwrap(wrapped_lhs); +} + +/** @brief Controlled register-pressure fixture. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_pressure(native_type a, native_type b, native_type c, native_type d, native_type e, native_type f, native_type g, native_type h) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + const SimdLibCodegen::register_type ab = SimdLibCodegen::register_type{a} + SimdLibCodegen::register_type{b}; + const SimdLibCodegen::register_type cd = SimdLibCodegen::register_type{c} + SimdLibCodegen::register_type{d}; + const SimdLibCodegen::register_type ef = SimdLibCodegen::register_type{e} + SimdLibCodegen::register_type{f}; + const SimdLibCodegen::register_type gh = SimdLibCodegen::register_type{g} + SimdLibCodegen::register_type{h}; + return ((ab + cd) + (ef + gh)).native; +#else + const native_type ab = SimdLibCodegen::api_type::add(a, b); + const native_type cd = SimdLibCodegen::api_type::add(c, d); + const native_type ef = SimdLibCodegen::api_type::add(e, f); + const native_type gh = SimdLibCodegen::api_type::add(g, h); + return SimdLibCodegen::unwrap( + SimdLibCodegen::wrap(SimdLibCodegen::api_type::add(SimdLibCodegen::api_type::add(ab, cd), SimdLibCodegen::api_type::add(ef, gh)))); +#endif +} + +/** @brief Chained bitwise-expression fixture including the public andnot polarity. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) simdlib_codegen_basic_bitwise(native_type lhs, native_type rhs) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + const SimdLibCodegen::register_type left{lhs}; + const SimdLibCodegen::register_type right{rhs}; + return ((left & right) | (left ^ ~right)).andnot(right).native; +#else + const native_type combined = SimdLibCodegen::api_type::bitwise_or(SimdLibCodegen::api_type::bitwise_and(lhs, rhs), + SimdLibCodegen::api_type::bitwise_xor(lhs, SimdLibCodegen::api_type::bitwise_not(rhs))); + return SimdLibCodegen::api_type::bitwise_andnot(combined, rhs); +#endif +} + +/** @brief Local reassignment expression fixture. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_reassignment_arithmetic(native_type lhs, native_type rhs, native_type multiplier) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + SimdLibCodegen::register_type result{lhs}; + result = result + SimdLibCodegen::register_type{rhs}; + result = result * SimdLibCodegen::register_type{multiplier}; + return result.native; +#else + return SimdLibCodegen::api_type::multiply(SimdLibCodegen::api_type::add(lhs, rhs), multiplier); +#endif +} + +/** @brief Explicit scalar-broadcast arithmetic-chain fixture. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_basic_broadcast_chain(native_type value, float scale, float offset) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return ((SimdLibCodegen::register_type{value} * SimdLibCodegen::register_type::broadcast(scale)) + SimdLibCodegen::register_type::broadcast(offset)).native; +#else + return SimdLibCodegen::api_type::add(SimdLibCodegen::api_type::multiply(value, SimdLibCodegen::api_type::set1(scale)), + SimdLibCodegen::api_type::set1(offset)); +#endif +} + +/** @brief Immediate per-lane unsigned left-shift fixture. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_basic_shift_left_immediate(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return (SimdLibCodegen::uint_register_type{value} << 3).native; +#else + return SimdLibCodegen::uint_api_type::shift_left(value, 3); +#endif +} + +#if SIMDLIB_REGISTER_TEST_WIDTH == 128 +/** @brief Static complete-register bit-shift fixture. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut) + simdlib_codegen_complete_shift_static(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bits_left<19>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bits_left<19>(value); +#endif +} + +/** @brief Runtime complete-register bit-shift fixture. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut) + simdlib_codegen_complete_shift_runtime(SimdLibCodegen::uint_native_type value, int count) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.shift_bits_right_slow(count).native; +#else + return SimdLibCodegen::uint_api_type::shift_bits_right_slow(value, count); +#endif +} + +/** @brief Runtime complete-register byte-shift fixture. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut) + simdlib_codegen_complete_byte_shift(SimdLibCodegen::uint_native_type value, int count) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.shift_bytes_left_slow(count).native; +#else + return SimdLibCodegen::uint_api_type::shift_bytes_left_slow(value, count); +#endif +} +#endif + +/** @brief Immediate complete-register byte left shift by 0 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_left_0(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_left<0>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_left<0>(value); +#endif +} + +/** @brief Immediate complete-register byte right shift by 0 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_right_0(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_right<0>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_right<0>(value); +#endif +} + +/** @brief Immediate complete-register byte left shift by 1 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_left_1(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_left<1>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_left<1>(value); +#endif +} + +/** @brief Immediate complete-register byte right shift by 1 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_right_1(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_right<1>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_right<1>(value); +#endif +} + +/** @brief Immediate complete-register byte left shift by 7 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_left_7(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_left<7>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_left<7>(value); +#endif +} + +/** @brief Immediate complete-register byte right shift by 7 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_right_7(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_right<7>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_right<7>(value); +#endif +} + +/** @brief Immediate complete-register byte left shift by 15 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_left_15(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_left<15>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_left<15>(value); +#endif +} + +/** @brief Immediate complete-register byte right shift by 15 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_right_15(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_right<15>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_right<15>(value); +#endif +} + +/** @brief Immediate complete-register byte left shift by 16 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_left_16(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_left<16>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_left<16>(value); +#endif +} + +/** @brief Immediate complete-register byte right shift by 16 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_right_16(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_right<16>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_right<16>(value); +#endif +} + +/** @brief Immediate complete-register byte left shift by 17 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_left_17(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_left<17>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_left<17>(value); +#endif +} + +/** @brief Immediate complete-register byte right shift by 17 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_right_17(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_right<17>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_right<17>(value); +#endif +} + +#if SIMDLIB_REGISTER_TEST_WIDTH == 256 +/** @brief Immediate complete-register byte left shift by 31 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_left_31(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_left<31>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_left<31>(value); +#endif +} + +/** @brief Immediate complete-register byte right shift by 31 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_right_31(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_right<31>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_right<31>(value); +#endif +} + +/** @brief Immediate complete-register byte left shift by 32 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_left_32(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_left<32>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_left<32>(value); +#endif +} + +/** @brief Immediate complete-register byte right shift by 32 bytes. */ +SIMDLIB_CODEGEN_NOINLINE SimdLibCodegen::uint_native_type SIMD_FLAGS(InOut, RegisterOnly) + simdlib_codegen_shift_bytes_right_32(SimdLibCodegen::uint_native_type value) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLibCodegen::uint_register_type{value}.template shift_bytes_right<32>().native; +#else + return SimdLibCodegen::uint_api_type::template shift_bytes_right<32>(value); +#endif +} + +#endif + +/** @brief Opaque-call fixture used to compare wrapper and raw spill behavior. */ +SIMDLIB_CODEGEN_NOINLINE native_type SIMD_FLAGS(InOut) simdlib_codegen_opaque(native_type value) noexcept +{ + const value_type wrapped = SimdLibCodegen::wrap(value); + simdlib_codegen_opaque_sink(SimdLibCodegen::unwrap(wrapped)); + return SimdLibCodegen::unwrap(wrapped); +} + +#undef SIMDLIB_CODEGEN_NOINLINE diff --git a/tests/codegen/RegisterCodegenRaw.cpp b/tests/codegen/RegisterCodegenRaw.cpp new file mode 100644 index 0000000..8c708d2 --- /dev/null +++ b/tests/codegen/RegisterCodegenRaw.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 0 +#include "RegisterCodegenFixture.h" diff --git a/tests/codegen/RegisterDefaultAbi.cpp b/tests/codegen/RegisterDefaultAbi.cpp new file mode 100644 index 0000000..d309c36 --- /dev/null +++ b/tests/codegen/RegisterDefaultAbi.cpp @@ -0,0 +1,18 @@ +#include + +#if SIMDLIB_COMPILER_MSVC +#define SIMDLIB_CODEGEN_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_CODEGEN_NOINLINE __attribute__((noinline)) +#endif + +using api_type = SimdLib::Api; +using register_type = SimdLib::Register; + +/** @brief Records wrapper behavior under the platform-default calling convention. */ +SIMDLIB_CODEGEN_NOINLINE register_type simdlib_codegen_default(register_type lhs, register_type rhs) noexcept +{ + return register_type{api_type::add(lhs.native, rhs.native)}; +} + +#undef SIMDLIB_CODEGEN_NOINLINE diff --git a/tests/codegen/RegisterDefaultAbiRaw.cpp b/tests/codegen/RegisterDefaultAbiRaw.cpp new file mode 100644 index 0000000..977d3b8 --- /dev/null +++ b/tests/codegen/RegisterDefaultAbiRaw.cpp @@ -0,0 +1,18 @@ +#include + +#if SIMDLIB_COMPILER_MSVC +#define SIMDLIB_CODEGEN_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_CODEGEN_NOINLINE __attribute__((noinline)) +#endif + +using api_type = SimdLib::Api; +using native_type = typename api_type::vector_t; + +/** @brief Records raw-vector behavior under the platform-default calling convention. */ +SIMDLIB_CODEGEN_NOINLINE native_type simdlib_codegen_default(native_type lhs, native_type rhs) noexcept +{ + return api_type::add(lhs, rhs); +} + +#undef SIMDLIB_CODEGEN_NOINLINE diff --git a/tests/codegen/RegisterFmaCodegen.cpp b/tests/codegen/RegisterFmaCodegen.cpp new file mode 100644 index 0000000..6077a0f --- /dev/null +++ b/tests/codegen/RegisterFmaCodegen.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 1 +#include "RegisterFmaCodegenFixture.h" \ No newline at end of file diff --git a/tests/codegen/RegisterFmaCodegenFixture.h b/tests/codegen/RegisterFmaCodegenFixture.h new file mode 100644 index 0000000..a2508fc --- /dev/null +++ b/tests/codegen/RegisterFmaCodegenFixture.h @@ -0,0 +1,62 @@ +#pragma once + +#include + +#if SIMDLIB_COMPILER_MSVC +#define SIMDLIB_FMA_CODEGEN_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_FMA_CODEGEN_NOINLINE __attribute__((noinline)) +#endif + +namespace SimdLibFmaCodegen +{ + +/** @brief Native single-precision register used by the isolated FMA fixture. */ +using float_native_t = typename SimdLib::Api::vector_t; + +/** @brief Native double-precision register used by the isolated FMA fixture. */ +using double_native_t = typename SimdLib::Api::vector_t; + +} // namespace SimdLibFmaCodegen + +/** + * @brief Compares single-precision Register multiply-add against the raw Api expression. + * @param lhs Multiplicand register. + * @param rhs Multiplier register. + * @param addend Addend register. + * @return Per-lane multiply-add result. + */ +SIMDLIB_FMA_CODEGEN_NOINLINE SimdLibFmaCodegen::float_native_t SIMD_FLAGS(Neither, RegisterOnly) + simdlib_fma_codegen_multiply_add_f32(SimdLibFmaCodegen::float_native_t lhs, SimdLibFmaCodegen::float_native_t rhs, + SimdLibFmaCodegen::float_native_t addend) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLib::Register{lhs} + .multiply_add(SimdLib::Register{rhs}, SimdLib::Register{addend}) + .native; +#else + return SimdLib::Api::multiply_add(lhs, rhs, addend); +#endif +} + +/** + * @brief Compares double-precision Register multiply-add against the raw Api expression. + * @param lhs Multiplicand register. + * @param rhs Multiplier register. + * @param addend Addend register. + * @return Per-lane multiply-add result. + */ +SIMDLIB_FMA_CODEGEN_NOINLINE SimdLibFmaCodegen::double_native_t SIMD_FLAGS(Neither, RegisterOnly) + simdlib_fma_codegen_multiply_add_f64(SimdLibFmaCodegen::double_native_t lhs, SimdLibFmaCodegen::double_native_t rhs, + SimdLibFmaCodegen::double_native_t addend) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return SimdLib::Register{lhs} + .multiply_add(SimdLib::Register{rhs}, SimdLib::Register{addend}) + .native; +#else + return SimdLib::Api::multiply_add(lhs, rhs, addend); +#endif +} + +#undef SIMDLIB_FMA_CODEGEN_NOINLINE \ No newline at end of file diff --git a/tests/codegen/RegisterFmaCodegenRaw.cpp b/tests/codegen/RegisterFmaCodegenRaw.cpp new file mode 100644 index 0000000..42583a3 --- /dev/null +++ b/tests/codegen/RegisterFmaCodegenRaw.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 0 +#include "RegisterFmaCodegenFixture.h" \ No newline at end of file diff --git a/tests/codegen/RegisterRearrangementCodegen.cpp b/tests/codegen/RegisterRearrangementCodegen.cpp new file mode 100644 index 0000000..9ebebad --- /dev/null +++ b/tests/codegen/RegisterRearrangementCodegen.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 1 +#include "RegisterRearrangementCodegenFixture.h" diff --git a/tests/codegen/RegisterRearrangementCodegenFixture.h b/tests/codegen/RegisterRearrangementCodegenFixture.h new file mode 100644 index 0000000..0ee31f6 --- /dev/null +++ b/tests/codegen/RegisterRearrangementCodegenFixture.h @@ -0,0 +1,279 @@ +#pragma once + +#if SIMDLIB_CODEGEN_USE_WRAPPER +#include +#else +#include +#endif + +#include +#include + +#if SIMDLIB_COMPILER_MSVC +#define SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE __attribute__((noinline)) +#endif + +namespace SimdLibRearrangementCodegen +{ + +/** @brief Native register type for one code-generation element type and width. */ +template using native_t = typename SimdLib::Api::vector_t; + +} // namespace SimdLibRearrangementCodegen + +#if SIMDLIB_CODEGEN_USE_WRAPPER +#define SIMDLIB_REARRANGE_UNARY(type, member, api, value) (SimdLib::Register{value}.member().native) +#define SIMDLIB_REARRANGE_BINARY(type, member, api, lhs, rhs) \ + (SimdLib::Register{lhs}.member(SimdLib::Register{rhs}).native) +#define SIMDLIB_REARRANGE_INDEXED_UNARY(type, member, api, immediate, value) \ + (SimdLib::Register{value}.template member().native) +#define SIMDLIB_REARRANGE_INDEXED_BINARY(type, member, api, immediate, lhs, rhs) \ + (SimdLib::Register{lhs}.template member(SimdLib::Register{rhs}).native) +#define SIMDLIB_REARRANGE_BIT_CAST(source_type, target_type, value) \ + (SimdLib::Register{value}.template bit_cast().native) +#define SIMDLIB_REARRANGE_CONVERT(source_type, target_type, value) \ + (SimdLib::Register{value}.template convert().native) +#define SIMDLIB_REARRANGE_LOWER(type, value) (SimdLib::Register{value}.lower_half().native) +#define SIMDLIB_REARRANGE_WIDEN(source_type, target_type, target_bits, value) \ + (SimdLib::Register{value}.template widen_low().native) +#define SIMDLIB_REARRANGE_LOGICAL_SHUFFLE(type, value, ...) (SimdLib::Register{value}.template shuffle<__VA_ARGS__>().native) +#define SIMDLIB_REARRANGE_BYTE_SHUFFLE(type, value, ...) \ + (SimdLib::Register{value}.template shuffle_bytes<__VA_ARGS__>().native) +#else +#define SIMDLIB_REARRANGE_UNARY(type, member, api, value) (SimdLib::Api::api(value)) +#define SIMDLIB_REARRANGE_BINARY(type, member, api, lhs, rhs) (SimdLib::Api::api(lhs, rhs)) +#define SIMDLIB_REARRANGE_INDEXED_UNARY(type, member, api, immediate, value) (SimdLib::Api::template api(value)) +#define SIMDLIB_REARRANGE_INDEXED_BINARY(type, member, api, immediate, lhs, rhs) \ + (SimdLib::Api::template api(lhs, rhs)) +#define SIMDLIB_REARRANGE_BIT_CAST(source_type, target_type, value) \ + (SimdLib::Api::template bit_cast(value)) +#define SIMDLIB_REARRANGE_CONVERT(source_type, target_type, value) \ + (SimdLib::Api::template convert(value)) +#define SIMDLIB_REARRANGE_LOWER(type, value) (SimdLib::Api<256, type>::lower_half(value)) +#define SIMDLIB_REARRANGE_WIDEN(source_type, target_type, target_bits, value) \ + (SimdLib::Api<128, source_type>::template widen>(value)) +#define SIMDLIB_REARRANGE_LOGICAL_SHUFFLE(type, value, ...) (SimdLib::Api::template shuffle<__VA_ARGS__>(value)) +#define SIMDLIB_REARRANGE_BYTE_SHUFFLE(type, value, ...) \ + (SimdLib::Api::template bit_cast( \ + SimdLib::Api::template shuffle<__VA_ARGS__>( \ + SimdLib::Api::template bit_cast(value)))) +#endif + +#define SIMDLIB_DEFINE_REARRANGE_UNARY(operation, token, type, member, api) \ + /** @brief Compares one unary rearrangement wrapper against its Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_##operation##_##token(SimdLibRearrangementCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_REARRANGE_UNARY(type, member, api, value); \ + } + +#define SIMDLIB_DEFINE_REARRANGE_BINARY(operation, token, type, member, api) \ + /** @brief Compares one binary rearrangement wrapper against its Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_##operation##_##token(SimdLibRearrangementCodegen::native_t lhs, \ + SimdLibRearrangementCodegen::native_t rhs) noexcept \ + { \ + return SIMDLIB_REARRANGE_BINARY(type, member, api, lhs, rhs); \ + } + +#define SIMDLIB_DEFINE_REARRANGE_INDEXED_UNARY(operation, token, type, member, api, immediate) \ + /** @brief Compares one immediate unary rearrangement wrapper against its Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_##operation##_##token(SimdLibRearrangementCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_REARRANGE_INDEXED_UNARY(type, member, api, immediate, value); \ + } + +#define SIMDLIB_DEFINE_REARRANGE_INDEXED_BINARY(operation, token, type, member, api, immediate) \ + /** @brief Compares one immediate binary rearrangement wrapper against its Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_##operation##_##token(SimdLibRearrangementCodegen::native_t lhs, \ + SimdLibRearrangementCodegen::native_t rhs) noexcept \ + { \ + return SIMDLIB_REARRANGE_INDEXED_BINARY(type, member, api, immediate, lhs, rhs); \ + } + +#define SIMDLIB_FOR_EACH_REGISTER_TYPE(macro, operation, member, api) \ + macro(operation, i8, std::int8_t, member, api) macro(operation, u8, std::uint8_t, member, api) macro(operation, i16, std::int16_t, member, api) \ + macro(operation, u16, std::uint16_t, member, api) macro(operation, i32, std::int32_t, member, api) macro(operation, u32, std::uint32_t, member, api) \ + macro(operation, i64, std::int64_t, member, api) macro(operation, u64, std::uint64_t, member, api) macro(operation, f32, float, member, api) \ + macro(operation, f64, double, member, api) + +SIMDLIB_FOR_EACH_REGISTER_TYPE(SIMDLIB_DEFINE_REARRANGE_BINARY, unpack_low, unpack_low, unpack_lo) +SIMDLIB_FOR_EACH_REGISTER_TYPE(SIMDLIB_DEFINE_REARRANGE_BINARY, unpack_high, unpack_high, unpack_hi) + +SIMDLIB_DEFINE_REARRANGE_INDEXED_UNARY(shuffle_low, i16, std::int16_t, shuffle_low, shuffle_lo, 0x1B) +SIMDLIB_DEFINE_REARRANGE_INDEXED_UNARY(shuffle_low, u16, std::uint16_t, shuffle_low, shuffle_lo, 0x1B) +SIMDLIB_DEFINE_REARRANGE_INDEXED_UNARY(shuffle_high, i16, std::int16_t, shuffle_high, shuffle_hi, 0x1B) +SIMDLIB_DEFINE_REARRANGE_INDEXED_UNARY(shuffle_high, u16, std::uint16_t, shuffle_high, shuffle_hi, 0x1B) +SIMDLIB_DEFINE_REARRANGE_INDEXED_BINARY(blend, i16, std::int16_t, blend, blend, 0xA5) +SIMDLIB_DEFINE_REARRANGE_INDEXED_BINARY(blend, u16, std::uint16_t, blend, blend, 0xA5) +SIMDLIB_DEFINE_REARRANGE_INDEXED_BINARY(blend, i32, std::int32_t, blend, blend, 0xA5) +SIMDLIB_DEFINE_REARRANGE_INDEXED_BINARY(blend, u32, std::uint32_t, blend, blend, 0xA5) +SIMDLIB_DEFINE_REARRANGE_INDEXED_BINARY(blend, f32, float, blend, blend, 0xA5) +SIMDLIB_DEFINE_REARRANGE_INDEXED_BINARY(blend, f64, double, blend, blend, 0xA5) + +#define SIMDLIB_DEFINE_LOGICAL_SHUFFLE(token, type, ...) \ + /** @brief Compares one complete logical shuffle wrapper against its Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_logical_shuffle_##token(SimdLibRearrangementCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_REARRANGE_LOGICAL_SHUFFLE(type, value, __VA_ARGS__); \ + } + +#if SIMDLIB_REGISTER_TEST_WIDTH == 128 +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(i8, std::int8_t, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(u8, std::uint8_t, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(i16, std::int16_t, 7, 6, 5, 4, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(u16, std::uint16_t, 7, 6, 5, 4, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(i32, std::int32_t, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(u32, std::uint32_t, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(i64, std::int64_t, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(u64, std::uint64_t, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(f32, float, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(f64, double, 1, 0) +#else +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(i8, std::int8_t, 0, 17, 2, 19, 4, 21, 6, 23, 8, 25, 10, 27, 12, 29, 14, 31, 16, 1, 18, 3, 20, 5, 22, 7, 24, 9, 26, 11, 28, 13, + 30, 15) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(u8, std::uint8_t, 0, 17, 2, 19, 4, 21, 6, 23, 8, 25, 10, 27, 12, 29, 14, 31, 16, 1, 18, 3, 20, 5, 22, 7, 24, 9, 26, 11, 28, 13, + 30, 15) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(i16, std::int16_t, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12, 5, 14, 7) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(u16, std::uint16_t, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12, 5, 14, 7) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(i32, std::int32_t, 7, 6, 5, 4, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(u32, std::uint32_t, 7, 6, 5, 4, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(i64, std::int64_t, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(u64, std::uint64_t, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(f32, float, 7, 6, 5, 4, 3, 2, 1, 0) +SIMDLIB_DEFINE_LOGICAL_SHUFFLE(f64, double, 3, 2, 1, 0) +#define SIMDLIB_DEFINE_LOWER(token, type) \ + /** @brief Compares one lower-half wrapper against its Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_lower_half_##token(SimdLibRearrangementCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_REARRANGE_LOWER(type, value); \ + } +SIMDLIB_DEFINE_LOWER(i8, std::int8_t) +SIMDLIB_DEFINE_LOWER(u8, std::uint8_t) +SIMDLIB_DEFINE_LOWER(i16, std::int16_t) +SIMDLIB_DEFINE_LOWER(u16, std::uint16_t) +SIMDLIB_DEFINE_LOWER(i32, std::int32_t) +SIMDLIB_DEFINE_LOWER(u32, std::uint32_t) +SIMDLIB_DEFINE_LOWER(i64, std::int64_t) +SIMDLIB_DEFINE_LOWER(u64, std::uint64_t) +SIMDLIB_DEFINE_LOWER(f32, float) +SIMDLIB_DEFINE_LOWER(f64, double) +#undef SIMDLIB_DEFINE_LOWER +#endif + +#define SIMDLIB_DEFINE_BYTE_SHUFFLE(token, type, ...) \ + /** @brief Compares one complete byte shuffle wrapper against its direct Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_byte_shuffle_##token(SimdLibRearrangementCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_REARRANGE_BYTE_SHUFFLE(type, value, __VA_ARGS__); \ + } + +#if SIMDLIB_REGISTER_TEST_WIDTH == 128 +SIMDLIB_DEFINE_BYTE_SHUFFLE(i32, std::int32_t, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#else +SIMDLIB_DEFINE_BYTE_SHUFFLE(i32_local, std::int32_t, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, + 18, 17, 16) +SIMDLIB_DEFINE_BYTE_SHUFFLE(i32_cross, std::int32_t, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, + 3, 2, 1, 0) +SIMDLIB_DEFINE_BYTE_SHUFFLE(i32_mixed, std::int32_t, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31) +#endif + +#define SIMDLIB_DEFINE_BIT_CAST(source_token, source_type, target_token, target_type) \ + /** @brief Compares one full-width bit reinterpretation wrapper against its Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_bit_cast_##source_token##_##target_token(SimdLibRearrangementCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_REARRANGE_BIT_CAST(source_type, target_type, value); \ + } + +#define SIMDLIB_FOR_EACH_BIT_CAST_TARGET(macro, source_token, source_type) \ + macro(source_token, source_type, i8, std::int8_t) macro(source_token, source_type, u8, std::uint8_t) macro(source_token, source_type, i16, std::int16_t) \ + macro(source_token, source_type, u16, std::uint16_t) macro(source_token, source_type, i32, std::int32_t) \ + macro(source_token, source_type, u32, std::uint32_t) macro(source_token, source_type, i64, std::int64_t) \ + macro(source_token, source_type, u64, std::uint64_t) macro(source_token, source_type, f32, float) \ + macro(source_token, source_type, f64, double) + +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, i8, std::int8_t) +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, u8, std::uint8_t) +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, i16, std::int16_t) +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, u16, std::uint16_t) +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, i32, std::int32_t) +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, u32, std::uint32_t) +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, i64, std::int64_t) +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, u64, std::uint64_t) +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, f32, float) +SIMDLIB_FOR_EACH_BIT_CAST_TARGET(SIMDLIB_DEFINE_BIT_CAST, f64, double) + +#define SIMDLIB_DEFINE_CONVERT(source_token, source_type, target_token, target_type) \ + /** @brief Compares one complete numeric conversion wrapper against its Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_convert_##source_token##_##target_token(SimdLibRearrangementCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_REARRANGE_CONVERT(source_type, target_type, value); \ + } +SIMDLIB_DEFINE_CONVERT(i32, std::int32_t, f32, float) +SIMDLIB_DEFINE_CONVERT(u32, std::uint32_t, f32, float) +SIMDLIB_DEFINE_CONVERT(f32, float, i32, std::int32_t) + +#if SIMDLIB_REGISTER_TEST_WIDTH == 128 +#define SIMDLIB_DEFINE_WIDEN(source_token, source_type, target_token, target_type, target_bits) \ + /** @brief Compares one explicit low-lane widening wrapper against its Api expression. */ \ + SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE SimdLibRearrangementCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_rearrangement_codegen_widen_##source_token##_##target_token##_##target_bits( \ + SimdLibRearrangementCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_REARRANGE_WIDEN(source_type, target_type, target_bits, value); \ + } +#if SIMDLIB_HAS_AVX2 +#define SIMDLIB_DEFINE_WIDEN_WIDTHS(source_token, source_type, target_token, target_type) \ + SIMDLIB_DEFINE_WIDEN(source_token, source_type, target_token, target_type, 128) \ + SIMDLIB_DEFINE_WIDEN(source_token, source_type, target_token, target_type, 256) +#else +#define SIMDLIB_DEFINE_WIDEN_WIDTHS(source_token, source_type, target_token, target_type) \ + SIMDLIB_DEFINE_WIDEN(source_token, source_type, target_token, target_type, 128) +#endif +SIMDLIB_DEFINE_WIDEN_WIDTHS(i8, std::int8_t, i16, std::int16_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(i8, std::int8_t, i32, std::int32_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(i8, std::int8_t, i64, std::int64_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(u8, std::uint8_t, u16, std::uint16_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(u8, std::uint8_t, u32, std::uint32_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(u8, std::uint8_t, u64, std::uint64_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(i16, std::int16_t, i32, std::int32_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(i16, std::int16_t, i64, std::int64_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(u16, std::uint16_t, u32, std::uint32_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(u16, std::uint16_t, u64, std::uint64_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(i32, std::int32_t, i64, std::int64_t) +SIMDLIB_DEFINE_WIDEN_WIDTHS(u32, std::uint32_t, u64, std::uint64_t) +#undef SIMDLIB_DEFINE_WIDEN_WIDTHS +#undef SIMDLIB_DEFINE_WIDEN +#endif + +#undef SIMDLIB_DEFINE_CONVERT +#undef SIMDLIB_FOR_EACH_BIT_CAST_TARGET +#undef SIMDLIB_DEFINE_BIT_CAST +#undef SIMDLIB_FOR_EACH_REGISTER_TYPE +#undef SIMDLIB_DEFINE_REARRANGE_INDEXED_BINARY +#undef SIMDLIB_DEFINE_REARRANGE_INDEXED_UNARY +#undef SIMDLIB_DEFINE_REARRANGE_BINARY +#undef SIMDLIB_DEFINE_REARRANGE_UNARY +#undef SIMDLIB_DEFINE_LOGICAL_SHUFFLE +#undef SIMDLIB_DEFINE_BYTE_SHUFFLE +#undef SIMDLIB_REARRANGE_LOGICAL_SHUFFLE +#undef SIMDLIB_REARRANGE_BYTE_SHUFFLE +#undef SIMDLIB_REARRANGE_WIDEN +#undef SIMDLIB_REARRANGE_LOWER +#undef SIMDLIB_REARRANGE_CONVERT +#undef SIMDLIB_REARRANGE_BIT_CAST +#undef SIMDLIB_REARRANGE_INDEXED_BINARY +#undef SIMDLIB_REARRANGE_INDEXED_UNARY +#undef SIMDLIB_REARRANGE_BINARY +#undef SIMDLIB_REARRANGE_UNARY +#undef SIMDLIB_REARRANGEMENT_CODEGEN_NOINLINE diff --git a/tests/codegen/RegisterRearrangementCodegenRaw.cpp b/tests/codegen/RegisterRearrangementCodegenRaw.cpp new file mode 100644 index 0000000..8fce6e2 --- /dev/null +++ b/tests/codegen/RegisterRearrangementCodegenRaw.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 0 +#include "RegisterRearrangementCodegenFixture.h" diff --git a/tests/codegen/RegisterSpecializedCodegen.cpp b/tests/codegen/RegisterSpecializedCodegen.cpp new file mode 100644 index 0000000..68e85c0 --- /dev/null +++ b/tests/codegen/RegisterSpecializedCodegen.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 1 +#include "RegisterSpecializedCodegenFixture.h" diff --git a/tests/codegen/RegisterSpecializedCodegenFixture.h b/tests/codegen/RegisterSpecializedCodegenFixture.h new file mode 100644 index 0000000..52da465 --- /dev/null +++ b/tests/codegen/RegisterSpecializedCodegenFixture.h @@ -0,0 +1,180 @@ +#pragma once + +#include + +#include +#include + +#if SIMDLIB_COMPILER_MSVC +#define SIMDLIB_SPECIALIZED_CODEGEN_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_SPECIALIZED_CODEGEN_NOINLINE __attribute__((noinline)) +#endif + +namespace SimdLibSpecializedCodegen +{ + +/** @brief Native register type for one specialized-operation source type. */ +template using native_t = typename SimdLib::Api::vector_t; + +} // namespace SimdLibSpecializedCodegen + +#if SIMDLIB_CODEGEN_USE_WRAPPER +#define SIMDLIB_SPECIALIZED_UNARY_EXPRESSION(type, member, api, value) (SimdLib::Register{value}.member().native) +#define SIMDLIB_SPECIALIZED_BINARY_EXPRESSION(type, member, api, lhs, rhs) \ + (SimdLib::Register{lhs}.member(SimdLib::Register{rhs}).native) +#define SIMDLIB_SPECIALIZED_SCALAR_EXPRESSION(type, member, api, value) (SimdLib::Register{value}.member()) +#define SIMDLIB_SPECIALIZED_PROMOTED_EXPRESSION(type, member, api, lhs, rhs) \ + (SimdLib::Register{lhs}.member(SimdLib::Register{rhs}).native) +#define SIMDLIB_SPECIALIZED_MULTI_SAD_EXPRESSION(type, lhs, rhs) \ + (SimdLib::Register{lhs} \ + .template multi_sum_absolute_byte_differences<0x1B>(SimdLib::Register{rhs}) \ + .native) +#define SIMDLIB_SPECIALIZED_DOT_EXPRESSION(type, lhs, rhs) \ + (SimdLib::Register{lhs}.template dot_product<0xD3>(SimdLib::Register{rhs}).native) +#else +#define SIMDLIB_SPECIALIZED_UNARY_EXPRESSION(type, member, api, value) (SimdLib::Api::api(value)) +#define SIMDLIB_SPECIALIZED_BINARY_EXPRESSION(type, member, api, lhs, rhs) (SimdLib::Api::api(lhs, rhs)) +#define SIMDLIB_SPECIALIZED_SCALAR_EXPRESSION(type, member, api, value) (SimdLib::Api::api(value)) +#define SIMDLIB_SPECIALIZED_PROMOTED_EXPRESSION(type, member, api, lhs, rhs) (SimdLib::Api::api(lhs, rhs)) +#define SIMDLIB_SPECIALIZED_MULTI_SAD_EXPRESSION(type, lhs, rhs) \ + (SimdLib::Api::template multi_sum_absolute_byte_differences<0x1B>(lhs, rhs)) +#define SIMDLIB_SPECIALIZED_DOT_EXPRESSION(type, lhs, rhs) (SimdLib::Api::template dot_product<0xD3>(lhs, rhs)) +#endif + +#define SIMDLIB_DEFINE_SPECIALIZED_UNARY(operation, token, type, member, api) \ + /** @brief Compares one unary Register specialized operation against its raw Api expression. */ \ + SIMDLIB_SPECIALIZED_CODEGEN_NOINLINE SimdLibSpecializedCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_specialized_codegen_##operation##_##token(SimdLibSpecializedCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_SPECIALIZED_UNARY_EXPRESSION(type, member, api, value); \ + } + +#define SIMDLIB_DEFINE_SPECIALIZED_BINARY(operation, token, type, member, api) \ + /** @brief Compares one binary Register specialized operation against its raw Api expression. */ \ + SIMDLIB_SPECIALIZED_CODEGEN_NOINLINE SimdLibSpecializedCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_specialized_codegen_##operation##_##token(SimdLibSpecializedCodegen::native_t lhs, \ + SimdLibSpecializedCodegen::native_t rhs) noexcept \ + { \ + return SIMDLIB_SPECIALIZED_BINARY_EXPRESSION(type, member, api, lhs, rhs); \ + } + +#define SIMDLIB_DEFINE_SPECIALIZED_SCALAR(operation, token, type, member, api) \ + /** @brief Compares one scalar-result Register specialized operation against its raw Api expression. */ \ + SIMDLIB_SPECIALIZED_CODEGEN_NOINLINE std::size_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_specialized_codegen_##operation##_##token(SimdLibSpecializedCodegen::native_t value) noexcept \ + { \ + return SIMDLIB_SPECIALIZED_SCALAR_EXPRESSION(type, member, api, value); \ + } + +#define SIMDLIB_DEFINE_SPECIALIZED_PROMOTED(operation, token, type, member, api) \ + /** @brief Compares one promoted-result Register specialized operation against its raw Api expression. */ \ + SIMDLIB_SPECIALIZED_CODEGEN_NOINLINE SimdLibSpecializedCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_specialized_codegen_##operation##_##token(SimdLibSpecializedCodegen::native_t lhs, \ + SimdLibSpecializedCodegen::native_t rhs) noexcept \ + { \ + return SIMDLIB_SPECIALIZED_PROMOTED_EXPRESSION(type, member, api, lhs, rhs); \ + } + +#define SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD(token, type) \ + /** @brief Compares immediate-controlled multi-SAD Register code against its raw Api expression. */ \ + SIMDLIB_SPECIALIZED_CODEGEN_NOINLINE SimdLibSpecializedCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_specialized_codegen_multi_sad_##token(SimdLibSpecializedCodegen::native_t lhs, SimdLibSpecializedCodegen::native_t rhs) noexcept \ + { \ + return SIMDLIB_SPECIALIZED_MULTI_SAD_EXPRESSION(type, lhs, rhs); \ + } + +#define SIMDLIB_DEFINE_SPECIALIZED_DOT(token, type) \ + /** @brief Compares immediate-controlled dot-product Register code against its raw Api expression. */ \ + SIMDLIB_SPECIALIZED_CODEGEN_NOINLINE SimdLibSpecializedCodegen::native_t SIMD_FLAGS(In, RegisterOnly) \ + simdlib_specialized_codegen_dot_product_##token(SimdLibSpecializedCodegen::native_t lhs, SimdLibSpecializedCodegen::native_t rhs) noexcept \ + { \ + return SIMDLIB_SPECIALIZED_DOT_EXPRESSION(type, lhs, rhs); \ + } + +#define SIMDLIB_FOR_EACH_SPECIALIZED_TYPE(macro, operation, member, api) \ + macro(operation, i8, std::int8_t, member, api) macro(operation, u8, std::uint8_t, member, api) macro(operation, i16, std::int16_t, member, api) \ + macro(operation, u16, std::uint16_t, member, api) macro(operation, i32, std::int32_t, member, api) macro(operation, u32, std::uint32_t, member, api) \ + macro(operation, i64, std::int64_t, member, api) macro(operation, u64, std::uint64_t, member, api) macro(operation, f32, float, member, api) \ + macro(operation, f64, double, member, api) + +#define SIMDLIB_FOR_EACH_SPECIALIZED_INTEGER(macro, operation, member, api) \ + macro(operation, i8, std::int8_t, member, api) macro(operation, u8, std::uint8_t, member, api) macro(operation, i16, std::int16_t, member, api) \ + macro(operation, u16, std::uint16_t, member, api) macro(operation, i32, std::int32_t, member, api) macro(operation, u32, std::uint32_t, member, api) \ + macro(operation, i64, std::int64_t, member, api) macro(operation, u64, std::uint64_t, member, api) + +SIMDLIB_FOR_EACH_SPECIALIZED_TYPE(SIMDLIB_DEFINE_SPECIALIZED_BINARY, min, min, min) +SIMDLIB_FOR_EACH_SPECIALIZED_TYPE(SIMDLIB_DEFINE_SPECIALIZED_BINARY, max, max, max) +SIMDLIB_FOR_EACH_SPECIALIZED_TYPE(SIMDLIB_DEFINE_SPECIALIZED_UNARY, absolute, absolute, absolute) +SIMDLIB_FOR_EACH_SPECIALIZED_TYPE(SIMDLIB_DEFINE_SPECIALIZED_UNARY, sqrt, sqrt, sqrt) +SIMDLIB_FOR_EACH_SPECIALIZED_TYPE(SIMDLIB_DEFINE_SPECIALIZED_UNARY, magnitude, magnitude, magnitude) +SIMDLIB_FOR_EACH_SPECIALIZED_INTEGER(SIMDLIB_DEFINE_SPECIALIZED_UNARY, magnitude_checked, magnitude_checked, magnitude_checked) + +SIMDLIB_DEFINE_SPECIALIZED_UNARY(normalize, f32, float, normalize, normalize) +SIMDLIB_DEFINE_SPECIALIZED_UNARY(normalize, f64, double, normalize, normalize) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(average, u8, std::uint8_t, average, avg) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(average, u16, std::uint16_t, average, avg) + +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_add, i16, std::int16_t, horizontal_add, add_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_add, u16, std::uint16_t, horizontal_add, add_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_add, i32, std::int32_t, horizontal_add, add_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_add, u32, std::uint32_t, horizontal_add, add_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_add, f32, float, horizontal_add, add_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_add, f64, double, horizontal_add, add_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_subtract, i16, std::int16_t, horizontal_subtract, subtract_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_subtract, u16, std::uint16_t, horizontal_subtract, subtract_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_subtract, i32, std::int32_t, horizontal_subtract, subtract_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_subtract, u32, std::uint32_t, horizontal_subtract, subtract_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_subtract, f32, float, horizontal_subtract, subtract_horizontal) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_subtract, f64, double, horizontal_subtract, subtract_horizontal) + +SIMDLIB_DEFINE_SPECIALIZED_BINARY(add_saturated, i8, std::int8_t, add_saturated, add_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(add_saturated, u8, std::uint8_t, add_saturated, add_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(add_saturated, i16, std::int16_t, add_saturated, add_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(add_saturated, u16, std::uint16_t, add_saturated, add_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(subtract_saturated, i8, std::int8_t, subtract_saturated, subtract_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(subtract_saturated, u8, std::uint8_t, subtract_saturated, subtract_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(subtract_saturated, i16, std::int16_t, subtract_saturated, subtract_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(subtract_saturated, u16, std::uint16_t, subtract_saturated, subtract_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_add_saturated, i16, std::int16_t, horizontal_add_saturated, hadd_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_add_saturated, u16, std::uint16_t, horizontal_add_saturated, hadd_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_subtract_saturated, i16, std::int16_t, horizontal_subtract_saturated, hsubtract_saturated) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(horizontal_subtract_saturated, u16, std::uint16_t, horizontal_subtract_saturated, hsubtract_saturated) + +SIMDLIB_DEFINE_SPECIALIZED_BINARY(add_subtract, f32, float, add_subtract, add_subtract) +SIMDLIB_DEFINE_SPECIALIZED_BINARY(add_subtract, f64, double, add_subtract, add_subtract) +SIMDLIB_DEFINE_SPECIALIZED_DOT(f32, float) +SIMDLIB_DEFINE_SPECIALIZED_DOT(f64, double) + +SIMDLIB_FOR_EACH_SPECIALIZED_INTEGER(SIMDLIB_DEFINE_SPECIALIZED_SCALAR, min_position, min_position, min_position) +SIMDLIB_FOR_EACH_SPECIALIZED_INTEGER(SIMDLIB_DEFINE_SPECIALIZED_SCALAR, max_position, max_position, max_position) +SIMDLIB_FOR_EACH_SPECIALIZED_INTEGER(SIMDLIB_DEFINE_SPECIALIZED_PROMOTED, multiply_add_adjacent, multiply_add_adjacent, multiply_add_adjacent) +SIMDLIB_FOR_EACH_SPECIALIZED_INTEGER(SIMDLIB_DEFINE_SPECIALIZED_PROMOTED, byte_multiply_add, multiply_add_unsigned_signed_bytes, + multiply_add_unsigned_signed_bytes) +SIMDLIB_FOR_EACH_SPECIALIZED_INTEGER(SIMDLIB_DEFINE_SPECIALIZED_PROMOTED, sum_absolute_byte_differences, sum_absolute_byte_differences, + sum_absolute_byte_differences) + +SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD(i8, std::int8_t) +SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD(u8, std::uint8_t) +SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD(i16, std::int16_t) +SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD(u16, std::uint16_t) +SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD(i32, std::int32_t) +SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD(u32, std::uint32_t) +SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD(i64, std::int64_t) +SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD(u64, std::uint64_t) + +#undef SIMDLIB_FOR_EACH_SPECIALIZED_INTEGER +#undef SIMDLIB_FOR_EACH_SPECIALIZED_TYPE +#undef SIMDLIB_DEFINE_SPECIALIZED_DOT +#undef SIMDLIB_DEFINE_SPECIALIZED_MULTI_SAD +#undef SIMDLIB_DEFINE_SPECIALIZED_PROMOTED +#undef SIMDLIB_DEFINE_SPECIALIZED_SCALAR +#undef SIMDLIB_DEFINE_SPECIALIZED_BINARY +#undef SIMDLIB_DEFINE_SPECIALIZED_UNARY +#undef SIMDLIB_SPECIALIZED_DOT_EXPRESSION +#undef SIMDLIB_SPECIALIZED_MULTI_SAD_EXPRESSION +#undef SIMDLIB_SPECIALIZED_PROMOTED_EXPRESSION +#undef SIMDLIB_SPECIALIZED_SCALAR_EXPRESSION +#undef SIMDLIB_SPECIALIZED_BINARY_EXPRESSION +#undef SIMDLIB_SPECIALIZED_UNARY_EXPRESSION +#undef SIMDLIB_SPECIALIZED_CODEGEN_NOINLINE \ No newline at end of file diff --git a/tests/codegen/RegisterSpecializedCodegenRaw.cpp b/tests/codegen/RegisterSpecializedCodegenRaw.cpp new file mode 100644 index 0000000..97f87b5 --- /dev/null +++ b/tests/codegen/RegisterSpecializedCodegenRaw.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 0 +#include "RegisterSpecializedCodegenFixture.h" diff --git a/tests/codegen/RegisterTypeMatrixCodegen.cpp b/tests/codegen/RegisterTypeMatrixCodegen.cpp new file mode 100644 index 0000000..148cc49 --- /dev/null +++ b/tests/codegen/RegisterTypeMatrixCodegen.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 1 +#include "RegisterTypeMatrixCodegenFixture.h" diff --git a/tests/codegen/RegisterTypeMatrixCodegenFixture.h b/tests/codegen/RegisterTypeMatrixCodegenFixture.h new file mode 100644 index 0000000..af4dd4f --- /dev/null +++ b/tests/codegen/RegisterTypeMatrixCodegenFixture.h @@ -0,0 +1,517 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#if SIMDLIB_COMPILER_MSVC +#define SIMDLIB_TYPE_MATRIX_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_TYPE_MATRIX_NOINLINE __attribute__((noinline)) +#endif + +namespace SimdLibTypeMatrixCodegen +{ + +/** @brief Api specialization for one common-operation fixture element type. */ +template using api_t = SimdLib::Api; + +/** @brief Native vector for one common-operation fixture element type. */ +template using native_t = typename api_t::vector_t; + +/** @brief Register wrapper for one common-operation fixture element type. */ +template using register_t = SimdLib::Register; + +/** @brief Register-mask wrapper for one common-operation fixture element type. */ +template using register_mask_t = SimdLib::RegisterMask; + +/** @brief Complete fixed-size lane array for one common-operation fixture element type. */ +template using array_t = std::array::lane_count>; + +/** @brief Returns the compact all-lane predicate for one fixture element type. */ +template [[nodiscard]] consteval typename api_t::mask_t all_lane_bits() noexcept +{ + using mask_t = typename api_t::mask_t; + if constexpr (register_t::lane_count == std::numeric_limits::digits) + return std::numeric_limits::max(); + else + return static_cast((mask_t{1} << register_t::lane_count) - 1); +} + +/** @brief Identifies one isolated native-result operation in the type matrix. */ +enum class vector_operation +{ + zero, + broadcast, + add, + subtract, + multiply, + divide, + modulus, + negate, + bitwise_and, + bitwise_or, + bitwise_xor, + bitwise_not, + bitwise_andnot, + compare_equal, + compare_greater, + compare_greater_equal, + compare_less, + compare_less_equal, + mask_and, + mask_or, + mask_xor, + mask_not, + select, + insert_last, + shift_left, + logical_shift_right, + shift_right, +}; + +/** + * @brief Emits one isolated native-result operation for exact wrapper/raw comparison. + * @tparam operation Operation selected at compile time. + * @param lhs First native operand. + * @param rhs Second native operand. + * @param third Third native operand. + * @param scalar Scalar operand for broadcasts and insertion. + * @param count Runtime shift count. + * @return Native result of the selected operation. + */ +template +[[nodiscard]] native_t SIMD_FLAGS(InOut, ForceInline) + vector_result(native_t lhs, native_t rhs, native_t third, element_t scalar, int count) noexcept +{ + using api_type [[maybe_unused]] = api_t; + using register_type = register_t; +#if SIMDLIB_CODEGEN_USE_WRAPPER + const register_type left{lhs}; + const register_type right{rhs}; + const register_type other{third}; + if constexpr (operation == vector_operation::zero) + return register_type::zero().native; + else if constexpr (operation == vector_operation::broadcast) + return register_type::broadcast(scalar).native; + else if constexpr (operation == vector_operation::add && SimdLib::IRegister::Add) + return (left + right).native; + else if constexpr (operation == vector_operation::subtract && SimdLib::IRegister::Subtract) + return (left - right).native; + else if constexpr (operation == vector_operation::multiply && SimdLib::IRegister::Multiply) + return (left * right).native; + else if constexpr (operation == vector_operation::divide && SimdLib::IRegister::Divide) + return (left / right).native; + else if constexpr (operation == vector_operation::modulus && SimdLib::IRegister::Modulus) + return (left % right).native; + else if constexpr (operation == vector_operation::negate && SimdLib::IRegister::Negate) + return (-left).native; + else if constexpr (operation == vector_operation::bitwise_and) + return (left & right).native; + else if constexpr (operation == vector_operation::bitwise_or) + return (left | right).native; + else if constexpr (operation == vector_operation::bitwise_xor) + return (left ^ right).native; + else if constexpr (operation == vector_operation::bitwise_not) + return (~left).native; + else if constexpr (operation == vector_operation::bitwise_andnot) + return left.andnot(right).native; + else if constexpr (operation == vector_operation::compare_equal) + return left.compare_equal(right).native; + else if constexpr (operation == vector_operation::compare_greater) + return left.compare_greater(right).native; + else if constexpr (operation == vector_operation::compare_greater_equal) + return left.compare_greater_equal(right).native; + else if constexpr (operation == vector_operation::compare_less) + return left.compare_less(right).native; + else if constexpr (operation == vector_operation::compare_less_equal) + return left.compare_less_equal(right).native; + else if constexpr (operation == vector_operation::mask_and) + return (register_mask_t{lhs} & register_mask_t{rhs}).native; + else if constexpr (operation == vector_operation::mask_or) + return (register_mask_t{lhs} | register_mask_t{rhs}).native; + else if constexpr (operation == vector_operation::mask_xor) + return (register_mask_t{lhs} ^ register_mask_t{rhs}).native; + else if constexpr (operation == vector_operation::mask_not) + return (~register_mask_t{lhs}).native; + else if constexpr (operation == vector_operation::select) + return register_mask_t{lhs}.select(right, other).native; + else if constexpr (operation == vector_operation::insert_last) + return left.template with_lane(scalar).native; + else if constexpr (operation == vector_operation::shift_left && SimdLib::IRegister::ShiftLeft) + return (left << count).native; + else if constexpr (operation == vector_operation::logical_shift_right && SimdLib::IRegister::LogicalShiftRight) + return left.logical_shift_right(count).native; + else if constexpr (operation == vector_operation::shift_right && SimdLib::IRegister::ShiftRight) + return (left >> count).native; +#else + if constexpr (operation == vector_operation::zero) + return api_type::setzero(); + else if constexpr (operation == vector_operation::broadcast) + return api_type::set1(scalar); + else if constexpr (operation == vector_operation::add && SimdLib::IRegister::Add) + return api_type::add(lhs, rhs); + else if constexpr (operation == vector_operation::subtract && SimdLib::IRegister::Subtract) + return api_type::subtract(lhs, rhs); + else if constexpr (operation == vector_operation::multiply && SimdLib::IRegister::Multiply) + return api_type::multiply(lhs, rhs); + else if constexpr (operation == vector_operation::divide && SimdLib::IRegister::Divide) + return api_type::divide(lhs, rhs); + else if constexpr (operation == vector_operation::modulus && SimdLib::IRegister::Modulus) + return api_type::modulus(lhs, rhs); + else if constexpr (operation == vector_operation::negate && SimdLib::IRegister::Negate) + return api_type::negate(lhs); + else if constexpr (operation == vector_operation::bitwise_and || operation == vector_operation::mask_and) + return api_type::bitwise_and(lhs, rhs); + else if constexpr (operation == vector_operation::bitwise_or || operation == vector_operation::mask_or) + return api_type::bitwise_or(lhs, rhs); + else if constexpr (operation == vector_operation::bitwise_xor || operation == vector_operation::mask_xor) + return api_type::bitwise_xor(lhs, rhs); + else if constexpr (operation == vector_operation::bitwise_not || operation == vector_operation::mask_not) + return api_type::bitwise_not(lhs); + else if constexpr (operation == vector_operation::bitwise_andnot) + return api_type::bitwise_andnot(lhs, rhs); + else if constexpr (operation == vector_operation::compare_equal) + return api_type::compare_equal(lhs, rhs); + else if constexpr (operation == vector_operation::compare_greater) + return api_type::compare_greater(lhs, rhs); + else if constexpr (operation == vector_operation::compare_greater_equal) + return api_type::compare_greater_equal(lhs, rhs); + else if constexpr (operation == vector_operation::compare_less) + return api_type::compare_less(lhs, rhs); + else if constexpr (operation == vector_operation::compare_less_equal) + return api_type::compare_less_equal(lhs, rhs); + else if constexpr (operation == vector_operation::select) + return api_type::select(lhs, rhs, third); + else if constexpr (operation == vector_operation::insert_last) + return api_type::template insert(lhs, scalar); + else if constexpr (operation == vector_operation::shift_left && SimdLib::IRegister::ShiftLeft) + return api_type::shift_left(lhs, count); + else if constexpr (operation == vector_operation::logical_shift_right && SimdLib::IRegister::LogicalShiftRight) + return api_type::shift_right(lhs, count); + else if constexpr (operation == vector_operation::shift_right && SimdLib::IRegister::ShiftRight) + { + if constexpr (std::is_signed_v) + return api_type::shift_right_arithmetic(lhs, count); + else + return api_type::shift_right(lhs, count); + } +#endif + else + { + static_assert(SimdLib::Detail::dependent_false_v, "The selected Register operation is unavailable for this element type."); + } +} + +/** @brief Identifies one isolated scalar-result operation in the type matrix. */ +enum class scalar_operation +{ + movemask, + lane_sign_bits, + mask_bits, + mask_any, + mask_all, + mask_none, + equal, + not_equal, + extract_first, +}; + +/** + * @brief Emits one isolated scalar-result operation for exact wrapper/raw comparison. + * @tparam operation Operation selected at compile time. + * @param lhs First native operand. + * @param rhs Second native operand. + * @return Compact scalar result of the selected operation. + */ +template +[[nodiscard]] typename api_t::mask_t SIMD_FLAGS(In, ForceInline) scalar_result(native_t lhs, native_t rhs) noexcept +{ + using api_type = api_t; + using register_type [[maybe_unused]] = register_t; + using mask_bits_t = typename api_type::mask_t; +#if SIMDLIB_CODEGEN_USE_WRAPPER + const register_type left{lhs}; + const register_type right{rhs}; + const register_mask_t mask{lhs}; + if constexpr (operation == scalar_operation::movemask) + return left.movemask(); + else if constexpr (operation == scalar_operation::lane_sign_bits) + return left.lane_sign_bits(); + else if constexpr (operation == scalar_operation::mask_bits) + return mask.bits(); + else if constexpr (operation == scalar_operation::mask_any) + return static_cast(mask.any()); + else if constexpr (operation == scalar_operation::mask_all) + return static_cast(mask.all()); + else if constexpr (operation == scalar_operation::mask_none) + return static_cast(mask.none()); + else if constexpr (operation == scalar_operation::equal) + return static_cast(left == right); + else if constexpr (operation == scalar_operation::not_equal) + return static_cast(left != right); + else + return static_cast(left.template lane<0>()); +#else + if constexpr (operation == scalar_operation::movemask) + return api_type::movemask(lhs); + else if constexpr (operation == scalar_operation::lane_sign_bits || operation == scalar_operation::mask_bits) + return api_type::movemask_slim(lhs); + else if constexpr (operation == scalar_operation::mask_any) + return static_cast(api_type::movemask_slim(lhs) != 0); + else if constexpr (operation == scalar_operation::mask_all) + return static_cast(api_type::movemask_slim(lhs) == all_lane_bits()); + else if constexpr (operation == scalar_operation::mask_none) + return static_cast(api_type::movemask_slim(lhs) == 0); + else if constexpr (operation == scalar_operation::equal) + return static_cast(api_type::movemask_slim(api_type::compare_equal(lhs, rhs)) == all_lane_bits()); + else if constexpr (operation == scalar_operation::not_equal) + return static_cast(api_type::movemask_slim(api_type::compare_equal(lhs, rhs)) != all_lane_bits()); + else + return static_cast(api_type::template extract<0>(lhs)); +#endif +} + +/** @brief Returns a register constructed from a fixed array. */ +template [[nodiscard]] native_t SIMD_FLAGS(Out, ForceInline) construct_array(const array_t &source) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return register_t::from_array(source).native; +#else + return api_t::construct(source); +#endif +} + +/** @brief Returns a register loaded from an unaligned fixed-size span. */ +template [[nodiscard]] native_t SIMD_FLAGS(Out, ForceInline) load(const element_t *source) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return register_t::load(std::span::lane_count>{source, register_t::lane_count}).native; +#else + return api_t::load(std::span::lane_count>{source, register_t::lane_count}); +#endif +} + +/** @brief Returns a register loaded from an aligned fixed-size span. */ +template [[nodiscard]] native_t SIMD_FLAGS(Out, ForceInline) load_aligned(const element_t *source) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return register_t::load_aligned(std::span::lane_count>{source, register_t::lane_count}).native; +#else + return api_t::load_aligned(std::span::lane_count>{source, register_t::lane_count}); +#endif +} + +/** @brief Returns a register loaded from a fixed-size byte span. */ +template [[nodiscard]] native_t SIMD_FLAGS(Out, ForceInline) load_bytes(const std::byte *source) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return register_t::load_bytes(std::span::byte_count>{source, register_t::byte_count}).native; +#else + return api_t::load(std::span::byte_count>{source, register_t::byte_count}); +#endif +} + +/** @brief Stores a native register through the unaligned fixed-size span API. */ +template void SIMD_FLAGS(In, ForceInline) store(native_t value, element_t *destination) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + register_t{value}.store(std::span::lane_count>{destination, register_t::lane_count}); +#else + api_t::store(value, std::span::lane_count>{destination, register_t::lane_count}); +#endif +} + +/** @brief Stores a native register through the aligned fixed-size span API. */ +template void SIMD_FLAGS(In, ForceInline) store_aligned(native_t value, element_t *destination) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + register_t{value}.store_aligned(std::span::lane_count>{destination, register_t::lane_count}); +#else + api_t::store_aligned(value, std::span::lane_count>{destination, register_t::lane_count}); +#endif +} + +/** @brief Stores a native register through the fixed-size byte-span API. */ +template void SIMD_FLAGS(In, ForceInline) store_bytes(native_t value, std::byte *destination) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + register_t{value}.store_bytes(std::span::byte_count>{destination, register_t::byte_count}); +#else + api_t::store(value, std::span::byte_count>{destination, register_t::byte_count}); +#endif +} + +/** @brief Stores a native register through the fixed-array observation API. */ +template void SIMD_FLAGS(In, ForceInline) observe_array(native_t value, array_t &destination) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + destination = register_t{value}.to_array(); +#else + destination = api_t::to_array(value); +#endif +} + +/** @brief Expands a complete array through the lane-list construction overload. */ +template +native_t SIMD_FLAGS(Out, RegisterOnly, ForceInline) from_lanes(const array_t &source, std::index_sequence) noexcept +{ +#if SIMDLIB_CODEGEN_USE_WRAPPER + return register_t::from_lanes(source[indices]...).native; +#else + return api_t::setr(source[indices]...); +#endif +} + +} // namespace SimdLibTypeMatrixCodegen + +#define SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, operation) \ + /** @brief Compares one isolated native-result operation with its raw Api expression. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE SimdLibTypeMatrixCodegen::native_t SIMD_FLAGS(In) \ + simdlib_type_matrix_##operation##_##token(SimdLibTypeMatrixCodegen::native_t lhs, SimdLibTypeMatrixCodegen::native_t rhs, \ + SimdLibTypeMatrixCodegen::native_t third, element_type scalar, int count) noexcept \ + { \ + return SimdLibTypeMatrixCodegen::vector_result(lhs, rhs, third, scalar, count); \ + } + +#define SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, operation) \ + /** @brief Compares one isolated scalar-result operation with its raw Api expression. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE typename SimdLibTypeMatrixCodegen::api_t::mask_t SIMD_FLAGS(In) simdlib_type_matrix_##operation##_##token( \ + SimdLibTypeMatrixCodegen::native_t lhs, SimdLibTypeMatrixCodegen::native_t rhs) noexcept \ + { \ + return SimdLibTypeMatrixCodegen::scalar_result(lhs, rhs); \ + } + +#define SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(token, element_type) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, zero) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, broadcast) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, add) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, subtract) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, multiply) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, divide) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, negate) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, bitwise_and) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, bitwise_or) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, bitwise_xor) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, bitwise_not) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, bitwise_andnot) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, compare_equal) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, compare_greater) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, compare_greater_equal) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, compare_less) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, compare_less_equal) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, mask_and) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, mask_or) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, mask_xor) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, mask_not) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, select) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, insert_last) \ + SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, movemask) \ + SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, lane_sign_bits) \ + SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, mask_bits) \ + SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, mask_any) \ + SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, mask_all) \ + SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, mask_none) \ + SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, equal) \ + SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, not_equal) \ + SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR(token, element_type, extract_first) \ + /** @brief Compares fixed-array construction for one element type. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE SimdLibTypeMatrixCodegen::native_t SIMD_FLAGS(Neither) \ + simdlib_type_matrix_construct_array_##token(const SimdLibTypeMatrixCodegen::array_t &source) noexcept \ + { \ + return SimdLibTypeMatrixCodegen::construct_array(source); \ + } \ + /** @brief Compares lane-list construction for one element type. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE SimdLibTypeMatrixCodegen::native_t SIMD_FLAGS(Neither) \ + simdlib_type_matrix_construct_lanes_##token(const SimdLibTypeMatrixCodegen::array_t &source) noexcept \ + { \ + return SimdLibTypeMatrixCodegen::from_lanes(source, \ + std::make_index_sequence::lane_count>{}); \ + } \ + /** @brief Compares unaligned loading for one element type. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE SimdLibTypeMatrixCodegen::native_t SIMD_FLAGS(Neither) \ + simdlib_type_matrix_load_##token(const element_type *source) noexcept \ + { \ + return SimdLibTypeMatrixCodegen::load(source); \ + } \ + /** @brief Compares aligned loading for one element type. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE SimdLibTypeMatrixCodegen::native_t SIMD_FLAGS(Neither) \ + simdlib_type_matrix_load_aligned_##token(const element_type *source) noexcept \ + { \ + return SimdLibTypeMatrixCodegen::load_aligned(source); \ + } \ + /** @brief Compares byte-span loading for one element type. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE SimdLibTypeMatrixCodegen::native_t SIMD_FLAGS(Neither) \ + simdlib_type_matrix_load_bytes_##token(const std::byte *source) noexcept \ + { \ + return SimdLibTypeMatrixCodegen::load_bytes(source); \ + } \ + /** @brief Compares unaligned storage for one element type. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE void SIMD_FLAGS(In) \ + simdlib_type_matrix_store_##token(SimdLibTypeMatrixCodegen::native_t value, element_type *destination) noexcept \ + { \ + SimdLibTypeMatrixCodegen::store(value, destination); \ + } \ + /** @brief Compares aligned storage for one element type. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE void SIMD_FLAGS(In) \ + simdlib_type_matrix_store_aligned_##token(SimdLibTypeMatrixCodegen::native_t value, element_type *destination) noexcept \ + { \ + SimdLibTypeMatrixCodegen::store_aligned(value, destination); \ + } \ + /** @brief Compares byte-span storage for one element type. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE void SIMD_FLAGS(In) \ + simdlib_type_matrix_store_bytes_##token(SimdLibTypeMatrixCodegen::native_t value, std::byte *destination) noexcept \ + { \ + SimdLibTypeMatrixCodegen::store_bytes(value, destination); \ + } \ + /** @brief Compares fixed-array observation for one element type. */ \ + SIMDLIB_TYPE_MATRIX_NOINLINE void SIMD_FLAGS(In) simdlib_type_matrix_observe_array_##token( \ + SimdLibTypeMatrixCodegen::native_t value, SimdLibTypeMatrixCodegen::array_t &destination) noexcept \ + { \ + SimdLibTypeMatrixCodegen::observe_array(value, destination); \ + } + +#define SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES(token, element_type) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, modulus) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, shift_left) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, logical_shift_right) \ + SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR(token, element_type, shift_right) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(i8, std::int8_t) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(u8, std::uint8_t) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(i16, std::int16_t) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(u16, std::uint16_t) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(i32, std::int32_t) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(u32, std::uint32_t) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(i64, std::int64_t) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(u64, std::uint64_t) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(f32, float) +SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES(f64, double) + +SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES(i8, std::int8_t) +SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES(u8, std::uint8_t) +SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES(i16, std::int16_t) +SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES(u16, std::uint16_t) +SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES(i32, std::int32_t) +SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES(u32, std::uint32_t) +SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES(i64, std::int64_t) +SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES(u64, std::uint64_t) +static_assert(!SimdLib::IRegister::Modulus>); +static_assert(!SimdLib::IRegister::Modulus>); +static_assert(!SimdLib::IRegister::ShiftLeft>); +static_assert(!SimdLib::IRegister::ShiftLeft>); +static_assert(!SimdLib::IRegister::LogicalShiftRight>); +static_assert(!SimdLib::IRegister::LogicalShiftRight>); +static_assert(!SimdLib::IRegister::ShiftRight>); +static_assert(!SimdLib::IRegister::ShiftRight>); + +#undef SIMDLIB_DEFINE_TYPE_MATRIX_INTEGER_FIXTURES +#undef SIMDLIB_DEFINE_TYPE_MATRIX_COMMON_FIXTURES +#undef SIMDLIB_DEFINE_TYPE_MATRIX_SCALAR +#undef SIMDLIB_DEFINE_TYPE_MATRIX_VECTOR +#undef SIMDLIB_TYPE_MATRIX_NOINLINE diff --git a/tests/codegen/RegisterTypeMatrixCodegenRaw.cpp b/tests/codegen/RegisterTypeMatrixCodegenRaw.cpp new file mode 100644 index 0000000..c3dd991 --- /dev/null +++ b/tests/codegen/RegisterTypeMatrixCodegenRaw.cpp @@ -0,0 +1,2 @@ +#define SIMDLIB_CODEGEN_USE_WRAPPER 0 +#include "RegisterTypeMatrixCodegenFixture.h" diff --git a/tests/compile_fail/api/ApiInvalidShuffleSelector.cpp b/tests/compile_fail/api/ApiInvalidShuffleSelector.cpp new file mode 100644 index 0000000..4340684 --- /dev/null +++ b/tests/compile_fail/api/ApiInvalidShuffleSelector.cpp @@ -0,0 +1,19 @@ +#define SIMDLIB_HAS_SSE42 1 +#define SIMDLIB_HAS_AVX2 1 +#include + +#include + +using dword_api = SimdLib::Api<128, std::uint32_t>; +using qword_api = SimdLib::Api<128, std::int64_t>; +using wide_float_api = SimdLib::Api<256, float>; + +/** @brief Reports whether a 32-bit Api accepts a selector outside the source register. */ +template +concept accepts_out_of_range_dword_selector = requires(typename api_t::vector_t value) { api_t::template shuffle<0, 1, 2, 4>(value); }; + +/** @brief Reports whether a 64-bit Api accepts a selector outside the source register. */ +template +concept accepts_out_of_range_qword_selector = requires(typename api_t::vector_t value) { api_t::template shuffle<0, 2>(value); }; + +static_assert(accepts_out_of_range_dword_selector || accepts_out_of_range_qword_selector, "SIMDLIB_API_REJECTS_INVALID_SHUFFLE_SELECTOR"); diff --git a/tests/compile_fail/api/ApiNegativeCompleteByteShift.cpp b/tests/compile_fail/api/ApiNegativeCompleteByteShift.cpp new file mode 100644 index 0000000..1eb0946 --- /dev/null +++ b/tests/compile_fail/api/ApiNegativeCompleteByteShift.cpp @@ -0,0 +1,14 @@ +#define SIMDLIB_HAS_SSE42 1 +#define SIMDLIB_HAS_AVX2 1 +#include + +#include + +using api = SimdLib::Api<256, std::uint8_t>; + +/** @brief Instantiates invalid negative immediate byte counts in both directions. */ +void invalid_negative_complete_byte_shifts(api::int_vector_t value) +{ + (void)api::template shift_bytes_left<-1>(value); + (void)api::template shift_bytes_right<-1>(value); +} \ No newline at end of file diff --git a/tests/compile_fail/api/ApiUnsuffixedRuntimeImmediate.cpp b/tests/compile_fail/api/ApiUnsuffixedRuntimeImmediate.cpp new file mode 100644 index 0000000..0b667fd --- /dev/null +++ b/tests/compile_fail/api/ApiUnsuffixedRuntimeImmediate.cpp @@ -0,0 +1,100 @@ +#define SIMDLIB_HAS_SSE42 1 +#define SIMDLIB_HAS_AVX2 1 +#include + +#include + +using byte_api = SimdLib::Api<128, std::uint8_t>; +using half_api = SimdLib::Api<128, std::uint16_t>; +using word_api = SimdLib::Api<128, std::uint32_t>; +using float_api = SimdLib::Api<128, float>; +using byte_impl = SimdLib::Detail::SimdMappings<128, std::uint8_t>; +using half_impl = SimdLib::Detail::SimdMappings<128, std::uint16_t>; +using word_impl = SimdLib::Detail::SimdMappings<128, std::uint32_t>; +using float_impl = SimdLib::Detail::SimdMappings<128, float>; + +/** @brief Reports whether unsuffixed Api extraction accepts a runtime lane index. */ +template +concept api_accepts_runtime_extract = requires(typename api_t::vector_t value, int control) { api_t::extract(value, control); }; +/** @brief Reports whether unsuffixed Api insertion accepts a runtime lane index. */ +template +concept api_accepts_runtime_insert = + requires(typename api_t::vector_t value, typename api_t::element_type lane, int control) { api_t::insert(value, lane, control); }; +/** @brief Reports whether unsuffixed Api blend accepts a runtime immediate mask. */ +template +concept api_accepts_runtime_blend = requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs, int control) { api_t::blend(lhs, rhs, control); }; +/** @brief Reports whether unsuffixed Api floating shuffle accepts a runtime immediate mask. */ +template +concept api_accepts_runtime_shuffle = requires(typename api_t::vector_t lhs, typename api_t::vector_t rhs, int control) { api_t::shuffle(lhs, rhs, control); }; +/** @brief Reports whether an unsuffixed native register-selector shuffle accepts a scalar selector. */ +template +concept api_accepts_scalar_shuffle_selector = requires(typename api_t::vector_t value, int control) { api_t::shuffle(value, control); }; +/** @brief Reports whether unsuffixed Api low-half shuffle accepts a runtime immediate mask. */ +template +concept api_accepts_runtime_shuffle_low = requires(typename api_t::vector_t value, int control) { api_t::shuffle_lo(value, control); }; +/** @brief Reports whether unsuffixed Api high-half shuffle accepts a runtime immediate mask. */ +template +concept api_accepts_runtime_shuffle_high = requires(typename api_t::vector_t value, int control) { api_t::shuffle_hi(value, control); }; +/** @brief Reports whether unsuffixed Api 32-bit shuffle accepts a runtime immediate mask. */ +template +concept api_accepts_runtime_shuffle_32 = requires(typename api_t::int_vector_t value, int control) { api_t::shuffle_32(value, control); }; +/** @brief Reports whether unsuffixed Api byte shift accepts a runtime count. */ +template +concept api_accepts_runtime_byte_shift = requires(typename api_t::int_vector_t value, int control) { + api_t::shift_bytes_left(value, control); + api_t::shift_bytes_right(value, control); +}; +/** @brief Reports whether unsuffixed Api complete-register bit shift accepts a runtime count. */ +template +concept api_accepts_runtime_bit_shift = requires(typename api_t::int_vector_t value, int control) { + api_t::shift_bits_left(value, control); + api_t::shift_bits_right(value, control); +}; + +/** @brief Reports whether unsuffixed implementation extraction accepts a runtime lane index. */ +template +concept impl_accepts_runtime_extract = requires(typename impl_t::vector_t value, int control) { impl_t::extract(value, control); }; +/** @brief Reports whether unsuffixed implementation insertion accepts a runtime lane index. */ +template +concept impl_accepts_runtime_insert = requires(typename impl_t::vector_t value, std::uint32_t lane, int control) { impl_t::insert(value, lane, control); }; +/** @brief Reports whether unsuffixed implementation blend accepts a runtime immediate mask. */ +template +concept impl_accepts_runtime_blend = requires(typename impl_t::vector_t lhs, typename impl_t::vector_t rhs, int control) { impl_t::blend(lhs, rhs, control); }; +/** @brief Reports whether unsuffixed implementation floating shuffle accepts a runtime immediate mask. */ +template +concept impl_accepts_runtime_shuffle = + requires(typename impl_t::vector_t lhs, typename impl_t::vector_t rhs, int control) { impl_t::shuffle(lhs, rhs, control); }; +/** @brief Reports whether a native implementation shuffle accepts a scalar selector. */ +template +concept impl_accepts_scalar_shuffle_selector = requires(typename impl_t::vector_t value, int control) { impl_t::shuffle(value, control); }; +/** @brief Reports whether unsuffixed implementation low-half shuffle accepts a runtime immediate mask. */ +template +concept impl_accepts_runtime_shuffle_low = requires(typename impl_t::vector_t value, int control) { impl_t::shuffle_lo(value, control); }; +/** @brief Reports whether unsuffixed implementation high-half shuffle accepts a runtime immediate mask. */ +template +concept impl_accepts_runtime_shuffle_high = requires(typename impl_t::vector_t value, int control) { impl_t::shuffle_hi(value, control); }; +/** @brief Reports whether unsuffixed implementation 32-bit shuffle accepts a runtime immediate mask. */ +template +concept impl_accepts_runtime_shuffle_32 = requires(typename impl_t::int_vector_t value, int control) { impl_t::shuffle_32(value, control); }; +/** @brief Reports whether unsuffixed implementation byte shift accepts a runtime count. */ +template +concept impl_accepts_runtime_byte_shift = requires(typename impl_t::int_vector_t value, int control) { + impl_t::shift_bytes_left(value, control); + impl_t::shift_bytes_right(value, control); +}; +/** @brief Reports whether unsuffixed implementation complete-register bit shift accepts a runtime count. */ +template +concept impl_accepts_runtime_bit_shift = requires(typename impl_t::int_vector_t value, int control) { + impl_t::shift_bits_left(value, control); + impl_t::shift_bits_right(value, control); +}; + +static_assert(api_accepts_runtime_extract || api_accepts_runtime_insert || api_accepts_runtime_blend || + api_accepts_runtime_blend || api_accepts_runtime_shuffle || api_accepts_scalar_shuffle_selector || + api_accepts_runtime_shuffle_low || api_accepts_runtime_shuffle_high || api_accepts_runtime_shuffle_32 || + api_accepts_runtime_byte_shift || api_accepts_runtime_bit_shift || impl_accepts_runtime_extract || + impl_accepts_runtime_insert || impl_accepts_runtime_blend || impl_accepts_runtime_blend || + impl_accepts_runtime_shuffle || impl_accepts_scalar_shuffle_selector || impl_accepts_runtime_shuffle_low || + impl_accepts_runtime_shuffle_high || impl_accepts_runtime_shuffle_32 || impl_accepts_runtime_byte_shift || + impl_accepts_runtime_bit_shift, + "SIMDLIB_REJECTS_UNSUFFIXED_RUNTIME_IMMEDIATE_CONTROLS"); \ No newline at end of file diff --git a/tests/compile_fail/api/ApiWrongShuffleSelectorCount.cpp b/tests/compile_fail/api/ApiWrongShuffleSelectorCount.cpp new file mode 100644 index 0000000..65120d6 --- /dev/null +++ b/tests/compile_fail/api/ApiWrongShuffleSelectorCount.cpp @@ -0,0 +1,18 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using byte_api = SimdLib::Api<128, std::uint8_t>; +using word_api = SimdLib::Api<128, std::int16_t>; + +/** @brief Reports whether an Api accepts fewer logical selectors than output lanes. */ +template +concept accepts_too_few_shuffle_selectors = + requires(typename api_t::vector_t value) { api_t::template shuffle<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14>(value); }; + +/** @brief Reports whether an Api accepts more logical selectors than output lanes. */ +template +concept accepts_too_many_shuffle_selectors = requires(typename api_t::vector_t value) { api_t::template shuffle<0, 1, 2, 3, 4, 5, 6, 7, 0>(value); }; + +static_assert(accepts_too_few_shuffle_selectors || accepts_too_many_shuffle_selectors, "SIMDLIB_API_REJECTS_WRONG_SHUFFLE_SELECTOR_COUNT"); diff --git a/tests/compile_fail/register/RegisterAvailabilityOverride.cpp b/tests/compile_fail/register/RegisterAvailabilityOverride.cpp new file mode 100644 index 0000000..00edf8a --- /dev/null +++ b/tests/compile_fail/register/RegisterAvailabilityOverride.cpp @@ -0,0 +1,8 @@ +#define SIMDLIB_REGISTER_INTERFACE_AVAILABLE 1 +#include + +/** @brief Provides an entry point when a computed-availability override unexpectedly compiles. */ +int main() +{ + return 0; +} diff --git a/tests/compile_fail/register/RegisterCollectionOperations.cpp b/tests/compile_fail/register/RegisterCollectionOperations.cpp new file mode 100644 index 0000000..8e963e3 --- /dev/null +++ b/tests/compile_fail/register/RegisterCollectionOperations.cpp @@ -0,0 +1,22 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether packed collection transformation leaks into the preferred Register surface. */ +template +concept has_transform_pack = requires(value_t value, std::span data) { value.transform_pack(data); }; + +/** @brief Reports whether unary collection transformation leaks into the preferred Register surface. */ +template +concept has_unary_transform = requires(value_t value, std::span data) { value.transform(data); }; + +/** @brief Reports whether binary collection transformation leaks into the preferred Register surface. */ +template +concept has_binary_transform = requires(value_t value, std::span data) { value.transform(data, data); }; + +static_assert(has_transform_pack || has_unary_transform || has_binary_transform, + "SIMDLIB_REGISTER_REJECTS_COLLECTION_OPERATIONS"); diff --git a/tests/compile_fail/register/RegisterCompatibilityRearrangement.cpp b/tests/compile_fail/register/RegisterCompatibilityRearrangement.cpp new file mode 100644 index 0000000..e86171a --- /dev/null +++ b/tests/compile_fail/register/RegisterCompatibilityRearrangement.cpp @@ -0,0 +1,35 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether runtime-selected lane extraction leaks into the preferred Register surface. */ +template +concept has_runtime_extract = requires(value_t value, std::size_t index) { value.extract(index); }; + +/** @brief Reports whether an implementation-specific generic shuffle leaks into the preferred Register surface. */ +template +concept has_generic_shuffle = requires(value_t value) { value.shuffle(value); }; + +/** @brief Reports whether an ambiguous expansion operation leaks into the preferred Register surface. */ +template +concept has_expand = requires(value_t value) { value.expand(value); }; + +/** @brief Reports whether an ambiguous compression operation leaks into the preferred Register surface. */ +template +concept has_compress = requires(value_t value) { value.compress(value); }; + +/** @brief Reports whether implementation-specific generic insertion leaks into the preferred Register surface. */ +template +concept has_generic_insert = requires(value_t value) { value.insert(value); }; + +/** @brief Reports whether complementary-type conversion inference leaks into the preferred Register surface. */ +template +concept has_inferred_convert = requires(value_t value) { value.convert(); }; + +static_assert(has_runtime_extract || has_generic_shuffle || has_expand || has_compress || + has_generic_insert || has_inferred_convert, + "SIMDLIB_REGISTER_REJECTS_COMPATIBILITY_REARRANGEMENT"); diff --git a/tests/compile_fail/register/RegisterDynamicTransfer.cpp b/tests/compile_fail/register/RegisterDynamicTransfer.cpp new file mode 100644 index 0000000..f7a37ec --- /dev/null +++ b/tests/compile_fail/register/RegisterDynamicTransfer.cpp @@ -0,0 +1,31 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether a dynamic-extent load bypasses the exact-width contract. */ +template +concept accepts_dynamic_load = requires(std::span source) { value_t::load(source); }; + +/** @brief Reports whether a partial-load escape hatch is exposed. */ +template +concept has_partial_load = requires(std::span source) { value_t::template load_partial<1>(source); }; + +/** @brief Reports whether an unsafe dynamic-load escape hatch is exposed. */ +template +concept has_unsafe_load = requires(std::span source) { value_t::load_unsafe(source); }; + +/** @brief Reports whether a partial-store escape hatch is exposed. */ +template +concept has_partial_store = requires(value_t value, std::span destination) { value.template store_partial<1>(destination); }; + +/** @brief Reports whether an unsafe dynamic-store escape hatch is exposed. */ +template +concept has_unsafe_store = requires(value_t value, std::span destination) { value.store_unsafe(destination); }; + +static_assert(accepts_dynamic_load || has_partial_load || has_unsafe_load || has_partial_store || + has_unsafe_store, + "SIMDLIB_REGISTER_REJECTS_DYNAMIC_TRANSFER"); diff --git a/tests/compile_fail/register/RegisterHeaderCxx20.cpp b/tests/compile_fail/register/RegisterHeaderCxx20.cpp new file mode 100644 index 0000000..41ad838 --- /dev/null +++ b/tests/compile_fail/register/RegisterHeaderCxx20.cpp @@ -0,0 +1,7 @@ +#include + +/** @brief Provides an entry point when an invalid Register header inclusion unexpectedly compiles. */ +int main() +{ + return 0; +} diff --git a/tests/compile_fail/register/RegisterImplicitNative.cpp b/tests/compile_fail/register/RegisterImplicitNative.cpp new file mode 100644 index 0000000..dab83a4 --- /dev/null +++ b/tests/compile_fail/register/RegisterImplicitNative.cpp @@ -0,0 +1,9 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include +#include + +using register_type = SimdLib::Register; + +static_assert(std::is_convertible_v, "SIMDLIB_REGISTER_REJECTS_IMPLICIT_NATIVE"); diff --git a/tests/compile_fail/register/RegisterImplicitScalar.cpp b/tests/compile_fail/register/RegisterImplicitScalar.cpp new file mode 100644 index 0000000..d51e6e5 --- /dev/null +++ b/tests/compile_fail/register/RegisterImplicitScalar.cpp @@ -0,0 +1,9 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include +#include + +using register_type = SimdLib::Register; + +static_assert(std::is_convertible_v, "SIMDLIB_REGISTER_REJECTS_IMPLICIT_SCALAR"); diff --git a/tests/compile_fail/register/RegisterInvalidByteShuffleSelector.cpp b/tests/compile_fail/register/RegisterInvalidByteShuffleSelector.cpp new file mode 100644 index 0000000..e2b4a71 --- /dev/null +++ b/tests/compile_fail/register/RegisterInvalidByteShuffleSelector.cpp @@ -0,0 +1,12 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using byte_register = SimdLib::Register; + +/** @brief Reports whether a byte shuffle accepts a selector outside the source register. */ +template +concept accepts_out_of_range_byte_selector = requires(value_t value) { value.template shuffle_bytes<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16>(); }; + +static_assert(accepts_out_of_range_byte_selector, "SIMDLIB_REGISTER_REJECTS_INVALID_BYTE_SHUFFLE_SELECTOR"); diff --git a/tests/compile_fail/register/RegisterInvalidRearrangementImmediate.cpp b/tests/compile_fail/register/RegisterInvalidRearrangementImmediate.cpp new file mode 100644 index 0000000..23258e8 --- /dev/null +++ b/tests/compile_fail/register/RegisterInvalidRearrangementImmediate.cpp @@ -0,0 +1,25 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether any immediate-controlled rearrangement accepts a negative control. */ +template +concept accepts_negative_immediate = requires(value_t value) { + value.template shuffle_low<-1>(); + value.template shuffle_high<-1>(); + value.template blend<-1>(value); +}; + +/** @brief Reports whether any immediate-controlled rearrangement accepts a control above one byte. */ +template +concept accepts_oversized_immediate = requires(value_t value) { + value.template shuffle_low<256>(); + value.template shuffle_high<256>(); + value.template blend<256>(value); +}; + +static_assert(accepts_negative_immediate || accepts_oversized_immediate, + "SIMDLIB_REGISTER_REJECTS_INVALID_REARRANGEMENT_IMMEDIATE"); diff --git a/tests/compile_fail/register/RegisterInvalidShuffleSelector.cpp b/tests/compile_fail/register/RegisterInvalidShuffleSelector.cpp new file mode 100644 index 0000000..487adc6 --- /dev/null +++ b/tests/compile_fail/register/RegisterInvalidShuffleSelector.cpp @@ -0,0 +1,20 @@ +#define SIMDLIB_HAS_SSE42 1 +#define SIMDLIB_HAS_AVX2 1 +#include + +#include + +using dword_register = SimdLib::Register; +using qword_register = SimdLib::Register; +using wide_float_register = SimdLib::Register; + +/** @brief Reports whether a 32-bit Register accepts a selector outside the source register. */ +template +concept accepts_out_of_range_dword_selector = requires(value_t value) { value.template shuffle<0, 1, 2, 4>(); }; + +/** @brief Reports whether a 64-bit Register accepts a selector outside the source register. */ +template +concept accepts_out_of_range_qword_selector = requires(value_t value) { value.template shuffle<0, 2>(); }; + +static_assert(accepts_out_of_range_dword_selector || accepts_out_of_range_qword_selector, + "SIMDLIB_REGISTER_REJECTS_INVALID_SHUFFLE_SELECTOR"); diff --git a/tests/compile_fail/register/RegisterNativeOrder.cpp b/tests/compile_fail/register/RegisterNativeOrder.cpp new file mode 100644 index 0000000..bbd0317 --- /dev/null +++ b/tests/compile_fail/register/RegisterNativeOrder.cpp @@ -0,0 +1,12 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether a native-order construction spelling is exposed. */ +template +concept has_native_order_constructor = requires { value_t::from_native_order(4, 3, 2, 1); }; + +static_assert(has_native_order_constructor, "SIMDLIB_REGISTER_REJECTS_NATIVE_ORDER_CONSTRUCTION"); diff --git a/tests/compile_fail/register/RegisterNegativeCompleteByteShift.cpp b/tests/compile_fail/register/RegisterNegativeCompleteByteShift.cpp new file mode 100644 index 0000000..670942d --- /dev/null +++ b/tests/compile_fail/register/RegisterNegativeCompleteByteShift.cpp @@ -0,0 +1,14 @@ +#define SIMDLIB_HAS_SSE42 1 +#define SIMDLIB_HAS_AVX2 1 +#include + +#include + +using register_type = SimdLib::Register; + +/** @brief Instantiates invalid negative Register immediate byte counts in both directions. */ +void invalid_negative_complete_byte_shifts(register_type value) +{ + (void)value.template shift_bytes_left<-1>(); + (void)value.template shift_bytes_right<-1>(); +} \ No newline at end of file diff --git a/tests/compile_fail/register/RegisterOversizedLaneList.cpp b/tests/compile_fail/register/RegisterOversizedLaneList.cpp new file mode 100644 index 0000000..194ffa6 --- /dev/null +++ b/tests/compile_fail/register/RegisterOversizedLaneList.cpp @@ -0,0 +1,12 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether an oversized logical lane list is accepted. */ +template +concept accepts_oversized_lane_list = requires { value_t::from_lanes(1, 2, 3, 4, 5); }; + +static_assert(accepts_oversized_lane_list, "SIMDLIB_REGISTER_REJECTS_OVERSIZED_LANE_LIST"); diff --git a/tests/compile_fail/register/RegisterPartialLaneList.cpp b/tests/compile_fail/register/RegisterPartialLaneList.cpp new file mode 100644 index 0000000..85693c7 --- /dev/null +++ b/tests/compile_fail/register/RegisterPartialLaneList.cpp @@ -0,0 +1,12 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether an incomplete logical lane list is accepted. */ +template +concept accepts_partial_lane_list = requires { value_t::from_lanes(1, 2, 3); }; + +static_assert(accepts_partial_lane_list, "SIMDLIB_REGISTER_REJECTS_PARTIAL_LANE_LIST"); diff --git a/tests/compile_fail/register/RegisterRequirementCxx20.cpp b/tests/compile_fail/register/RegisterRequirementCxx20.cpp new file mode 100644 index 0000000..ef6a419 --- /dev/null +++ b/tests/compile_fail/register/RegisterRequirementCxx20.cpp @@ -0,0 +1,8 @@ +#define SIMDLIB_REQUIRE_REGISTER_INTERFACE 1 +#include + +/** @brief Provides an entry point when an unavailable Register requirement unexpectedly compiles. */ +int main() +{ + return 0; +} diff --git a/tests/compile_fail/register/RegisterUnavailableWidthChange.cpp b/tests/compile_fail/register/RegisterUnavailableWidthChange.cpp new file mode 100644 index 0000000..787c919 --- /dev/null +++ b/tests/compile_fail/register/RegisterUnavailableWidthChange.cpp @@ -0,0 +1,13 @@ +#define SIMDLIB_HAS_SSE42 1 +#define SIMDLIB_HAS_AVX2 1 +#include + +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether widening accepts a 256-bit source that lacks a one-result backend mapping. */ +template +concept accepts_unavailable_width_change = requires(value_t value) { value.template widen_low(); }; + +static_assert(accepts_unavailable_width_change, "SIMDLIB_REGISTER_REJECTS_UNAVAILABLE_WIDTH_CHANGE"); diff --git a/tests/compile_fail/register/RegisterUninitialized.cpp b/tests/compile_fail/register/RegisterUninitialized.cpp new file mode 100644 index 0000000..2553f29 --- /dev/null +++ b/tests/compile_fail/register/RegisterUninitialized.cpp @@ -0,0 +1,14 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include +#include + +/** @brief Marker used to probe for an uninitialized construction escape hatch. */ +struct uninitialized_t final +{ +}; + +using register_type = SimdLib::Register; + +static_assert(std::is_constructible_v, "SIMDLIB_REGISTER_REJECTS_UNINITIALIZED_CONSTRUCTION"); diff --git a/tests/compile_fail/register/RegisterUnsuffixedRuntimeImmediate.cpp b/tests/compile_fail/register/RegisterUnsuffixedRuntimeImmediate.cpp new file mode 100644 index 0000000..f05f4f1 --- /dev/null +++ b/tests/compile_fail/register/RegisterUnsuffixedRuntimeImmediate.cpp @@ -0,0 +1,23 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether Register exposes an unsuffixed runtime complete-register byte shift. */ +template +concept accepts_runtime_byte_shift = requires(value_t value, int count) { + value.shift_bytes_left(count); + value.shift_bytes_right(count); +}; + +/** @brief Reports whether Register exposes an unsuffixed runtime complete-register bit shift. */ +template +concept accepts_runtime_bit_shift = requires(value_t value, int count) { + value.shift_bits_left(count); + value.shift_bits_right(count); +}; + +static_assert(accepts_runtime_byte_shift || accepts_runtime_bit_shift, + "SIMDLIB_REGISTER_REJECTS_UNSUFFIXED_RUNTIME_IMMEDIATE_CONTROLS"); \ No newline at end of file diff --git a/tests/compile_fail/register/RegisterUnsupportedCompiler.cpp b/tests/compile_fail/register/RegisterUnsupportedCompiler.cpp new file mode 100644 index 0000000..2103e10 --- /dev/null +++ b/tests/compile_fail/register/RegisterUnsupportedCompiler.cpp @@ -0,0 +1,8 @@ +#define SIMDLIB_REQUIRE_REGISTER_INTERFACE 1 +#include + +/** @brief Provides an entry point when an unsupported Register compiler unexpectedly compiles. */ +int main() +{ + return 0; +} diff --git a/tests/compile_fail/register/RegisterUnsupportedConversionTarget.cpp b/tests/compile_fail/register/RegisterUnsupportedConversionTarget.cpp new file mode 100644 index 0000000..b070bdf --- /dev/null +++ b/tests/compile_fail/register/RegisterUnsupportedConversionTarget.cpp @@ -0,0 +1,12 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using register_type = SimdLib::Register; + +/** @brief Reports whether numeric conversion accepts a target outside the complete-register backend contract. */ +template +concept accepts_unsupported_conversion_target = requires(value_t value) { value.template convert(); }; + +static_assert(accepts_unsupported_conversion_target, "SIMDLIB_REGISTER_REJECTS_UNSUPPORTED_CONVERSION_TARGET"); diff --git a/tests/compile_fail/register/RegisterWrongByteShuffleSelectorCount.cpp b/tests/compile_fail/register/RegisterWrongByteShuffleSelectorCount.cpp new file mode 100644 index 0000000..d6188eb --- /dev/null +++ b/tests/compile_fail/register/RegisterWrongByteShuffleSelectorCount.cpp @@ -0,0 +1,18 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using byte_register = SimdLib::Register; + +/** @brief Reports whether a byte shuffle accepts fewer selectors than register bytes. */ +template +concept accepts_too_few_byte_shuffle_selectors = requires(value_t value) { value.template shuffle_bytes<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14>(); }; + +/** @brief Reports whether a byte shuffle accepts more selectors than register bytes. */ +template +concept accepts_too_many_byte_shuffle_selectors = + requires(value_t value) { value.template shuffle_bytes<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0>(); }; + +static_assert(accepts_too_few_byte_shuffle_selectors || accepts_too_many_byte_shuffle_selectors, + "SIMDLIB_REGISTER_REJECTS_WRONG_BYTE_SHUFFLE_SELECTOR_COUNT"); diff --git a/tests/compile_fail/register/RegisterWrongShuffleSelectorCount.cpp b/tests/compile_fail/register/RegisterWrongShuffleSelectorCount.cpp new file mode 100644 index 0000000..e81db3a --- /dev/null +++ b/tests/compile_fail/register/RegisterWrongShuffleSelectorCount.cpp @@ -0,0 +1,18 @@ +#define SIMDLIB_HAS_SSE42 1 +#include + +#include + +using byte_register = SimdLib::Register; +using word_register = SimdLib::Register; + +/** @brief Reports whether a logical shuffle accepts fewer selectors than result lanes. */ +template +concept accepts_too_few_shuffle_selectors = requires(value_t value) { value.template shuffle<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14>(); }; + +/** @brief Reports whether a logical shuffle accepts more selectors than result lanes. */ +template +concept accepts_too_many_shuffle_selectors = requires(value_t value) { value.template shuffle<0, 1, 2, 3, 4, 5, 6, 7, 0>(); }; + +static_assert(accepts_too_few_shuffle_selectors || accepts_too_many_shuffle_selectors, + "SIMDLIB_REGISTER_REJECTS_WRONG_SHUFFLE_SELECTOR_COUNT"); \ No newline at end of file diff --git a/tests/config/ConfigClangUnsupportedTargetProbe.cpp b/tests/config/ConfigClangUnsupportedTargetProbe.cpp index 186b62a..8c7293d 100644 --- a/tests/config/ConfigClangUnsupportedTargetProbe.cpp +++ b/tests/config/ConfigClangUnsupportedTargetProbe.cpp @@ -7,7 +7,7 @@ static_assert(!SimdLib::Config::target_x86); static_assert(!SimdLib::Config::vectorcall_enabled); -int VECTORCALL ConfigClangUnsupportedTargetProbe() noexcept +int SIMD_FLAGS(Neither) ConfigClangUnsupportedTargetProbe() noexcept { return 0; } diff --git a/tests/config/ConfigDefaultChecksProbe.cpp b/tests/config/ConfigDefaultChecksProbe.cpp new file mode 100644 index 0000000..6ef78d8 --- /dev/null +++ b/tests/config/ConfigDefaultChecksProbe.cpp @@ -0,0 +1,11 @@ +#include + +#ifndef SIMDLIB_EXPECT_DEFAULT_CHECKS +#error "SIMDLIB_EXPECT_DEFAULT_CHECKS must be defined by the owning configuration profile" +#endif + +#if SIMDLIB_EXPECT_DEFAULT_CHECKS && defined(NDEBUG) +#error "The checks-enabled Debug configuration unexpectedly defines NDEBUG" +#endif + +static_assert(SIMDLIB_ENABLE_CHECKS == SIMDLIB_EXPECT_DEFAULT_CHECKS, "The default checks state does not match the owning configuration profile"); diff --git a/tests/config/ConfigDefaultProbe.cpp b/tests/config/ConfigDefaultProbe.cpp index 62e7de2..d554efd 100644 --- a/tests/config/ConfigDefaultProbe.cpp +++ b/tests/config/ConfigDefaultProbe.cpp @@ -1,31 +1,36 @@ #include -int VECTORCALL ConfigFreeFunction(const int value) noexcept +int SIMD_FLAGS(Neither) ConfigFreeFunction(const int value) noexcept { return value; } struct ConfigProbe { - static int VECTORCALL StaticFunction(const int value) noexcept + static int SIMD_FLAGS(Neither) StaticFunction(const int value) noexcept { return value; } - template - static value_t VECTORCALL TemplateFunction(const value_t value) noexcept + template static value_t SIMD_FLAGS(Neither) TemplateFunction(const value_t value) noexcept { return value; } }; -using ConfigFunctionPointer = int(VECTORCALL*)(int); +using ConfigFunctionPointer = int (*)(int); -SIMDLIB_FORCE_INLINE int ForceInlineFunction(const int value) noexcept +int SIMD_FLAGS(Neither, ForceInline) ForceInlineFunction(const int value) noexcept { return value + 1; } +/** @brief Exercises the default recursive-inlining annotation. */ +int SIMD_FLAGS(Neither, Flatten) FlattenFunction(const int value) noexcept +{ + return ForceInlineFunction(value); +} + static_assert(SimdLib::Config::target_x86 == (SIMDLIB_TARGET_X86 != 0)); static_assert(SimdLib::Config::target_x64 == (SIMDLIB_TARGET_X64 != 0)); static_assert(SimdLib::Config::compiler_clang == (SIMDLIB_COMPILER_CLANG != 0)); @@ -47,9 +52,12 @@ static_assert(SimdLib::version_major == 0 && SimdLib::version_minor == 2 && Simd #if SIMDLIB_COMPILER_MSVC && SIMDLIB_TARGET_X86 static_assert(SimdLib::Config::vectorcall_enabled); #endif +#if SIMDLIB_COMPILER_CLANG && !defined(_WIN32) +static_assert(!SimdLib::Config::vectorcall_enabled); +#endif int ConfigDefaultProbe() noexcept { const ConfigFunctionPointer function = &ConfigFreeFunction; - return function(ConfigProbe::StaticFunction(ConfigProbe::TemplateFunction(ForceInlineFunction(0)))); + return function(ConfigProbe::StaticFunction(ConfigProbe::TemplateFunction(FlattenFunction(0)))); } diff --git a/tests/config/ConfigOverrideForceInlineProbe.cpp b/tests/config/ConfigOverrideForceInlineProbe.cpp deleted file mode 100644 index a529bad..0000000 --- a/tests/config/ConfigOverrideForceInlineProbe.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#define SIMDLIB_FORCE_INLINE inline -#include - -SIMDLIB_FORCE_INLINE int ConfigOverrideForceInlineProbe() noexcept -{ - return 0; -} diff --git a/tests/config/ConfigOverridePreconditionProbe.cpp b/tests/config/ConfigOverridePreconditionProbe.cpp index 01281cd..2203d0c 100644 --- a/tests/config/ConfigOverridePreconditionProbe.cpp +++ b/tests/config/ConfigOverridePreconditionProbe.cpp @@ -1,11 +1,11 @@ inline int precondition_failures = 0; -#define SIMDLIB_PRECONDITION(condition, message) \ - do \ - { \ - (void)(message); \ - if (!(condition)) \ - ++precondition_failures; \ +#define SIMDLIB_PRECONDITION(condition, message) \ + do \ + { \ + (void)(message); \ + if (!(condition)) \ + ++precondition_failures; \ } while (false) #include diff --git a/tests/config/ConfigOverrideVectorcallProbe.cpp b/tests/config/ConfigOverrideVectorcallProbe.cpp deleted file mode 100644 index 9e43b09..0000000 --- a/tests/config/ConfigOverrideVectorcallProbe.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#define VECTORCALL -#define SIMDLIB_VECTORCALL_ENABLED 0 -#include - -static_assert(!SimdLib::Config::vectorcall_enabled); - -int VECTORCALL ConfigOverrideVectorcallProbe() noexcept -{ - return 0; -} diff --git a/tests/config/MethodFlagsConfigDefaultProbe.cpp b/tests/config/MethodFlagsConfigDefaultProbe.cpp new file mode 100644 index 0000000..b934720 --- /dev/null +++ b/tests/config/MethodFlagsConfigDefaultProbe.cpp @@ -0,0 +1,67 @@ +#include + +#include + +/** @brief Exercises the default Out boundary on a SIMD load. */ +[[nodiscard]] __m128 SIMD_FLAGS(Out) MethodFlagsDefaultLoad(const float *source) noexcept +{ + return _mm_loadu_ps(source); +} + +/** @brief Exercises the default In boundary on a memory-writing SIMD store. */ +void SIMD_FLAGS(In) MethodFlagsDefaultStore(const __m128 value, float *destination) noexcept +{ + _mm_storeu_ps(destination, value); +} + +/** @brief Exercises an In reduction with the independent RegisterOnly promise. */ +[[nodiscard]] float SIMD_FLAGS(In, RegisterOnly) MethodFlagsDefaultReduce(const __m128 value) noexcept +{ + return _mm_cvtss_f32(value); +} + +/** @brief Exercises the complete default InOut modifier composition. */ +[[nodiscard]] __m128 SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) MethodFlagsDefaultTransform(const __m128 value) noexcept +{ + return value; +} + +/** @brief Exercises an attribute-free scalar boundary. */ +[[nodiscard]] int SIMD_FLAGS(Neither) MethodFlagsDefaultScalar(const int value) noexcept +{ + return value; +} + +static_assert(SimdLib::Config::method_flags_has_vectorcall == (SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL != 0)); +static_assert(SimdLib::Config::method_flags_has_safe_buffers == (SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS != 0)); +static_assert(SimdLib::Config::method_flags_has_force_inline == (SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE != 0)); +static_assert(SimdLib::Config::method_flags_has_flatten == (SIMDLIB_METHOD_FLAGS_HAS_FLATTEN != 0)); + +#if SIMDLIB_COMPILER_MSVC +static_assert(SimdLib::Config::method_flags_has_vectorcall); +static_assert(SimdLib::Config::method_flags_has_safe_buffers); +#endif + +#if SIMDLIB_COMPILER_CLANG && defined(_WIN32) +static_assert(SimdLib::Config::method_flags_has_vectorcall); +static_assert(!SimdLib::Config::method_flags_has_safe_buffers); +#endif + +#if SIMDLIB_COMPILER_GCC || (SIMDLIB_COMPILER_CLANG && !defined(_WIN32)) +static_assert(!SimdLib::Config::method_flags_has_vectorcall); +static_assert(!SimdLib::Config::method_flags_has_safe_buffers); +#endif + +#if SIMDLIB_COMPILER_MSVC || SIMDLIB_COMPILER_CLANG || SIMDLIB_COMPILER_GCC +static_assert(SimdLib::Config::method_flags_has_force_inline); +static_assert(SimdLib::Config::method_flags_has_flatten); +#endif + +/** @brief Instantiates the default method-flags probe functions. */ +int MethodFlagsConfigDefaultProbe() noexcept +{ + alignas(16) float values[4]{}; + const __m128 loaded = MethodFlagsDefaultLoad(values); + MethodFlagsDefaultStore(MethodFlagsDefaultTransform(loaded), values); + return MethodFlagsDefaultScalar(static_cast(MethodFlagsDefaultReduce(loaded))); +} diff --git a/tests/config/MethodFlagsConfigDisabledVectorcallProbe.cpp b/tests/config/MethodFlagsConfigDisabledVectorcallProbe.cpp new file mode 100644 index 0000000..9ae20cd --- /dev/null +++ b/tests/config/MethodFlagsConfigDisabledVectorcallProbe.cpp @@ -0,0 +1,31 @@ +#define SIMDLIB_VECTORCALL_ENABLED 0 +#include + +#include + +static_assert(!SimdLib::Config::vectorcall_enabled); +static_assert(!SimdLib::Config::method_flags_has_vectorcall); + +/** @brief Exercises Out while the vector calling-convention mapping is disabled. */ +[[nodiscard]] __m128 SIMD_FLAGS(Out) MethodFlagsDisabledVectorcallOut() noexcept +{ + return _mm_setzero_ps(); +} + +/** @brief Exercises In while the vector calling-convention mapping is disabled. */ +[[nodiscard]] float SIMD_FLAGS(In) MethodFlagsDisabledVectorcallIn(const __m128 value) noexcept +{ + return _mm_cvtss_f32(value); +} + +/** @brief Exercises InOut while the vector calling-convention mapping is disabled. */ +[[nodiscard]] __m128 SIMD_FLAGS(InOut) MethodFlagsDisabledVectorcallInOut(const __m128 value) noexcept +{ + return value; +} + +/** @brief Instantiates every disabled-vectorcall boundary declaration. */ +int MethodFlagsConfigDisabledVectorcallProbe() noexcept +{ + return static_cast(MethodFlagsDisabledVectorcallIn(MethodFlagsDisabledVectorcallInOut(MethodFlagsDisabledVectorcallOut()))); +} diff --git a/tests/config/MethodFlagsConfigOverrideProbe.cpp b/tests/config/MethodFlagsConfigOverrideProbe.cpp new file mode 100644 index 0000000..5dfef57 --- /dev/null +++ b/tests/config/MethodFlagsConfigOverrideProbe.cpp @@ -0,0 +1,20 @@ +#define SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL 0 +#define SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS 1 +#define SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE 0 +#define SIMDLIB_METHOD_FLAGS_HAS_FLATTEN 1 +#define SIMDLIB_METHOD_FLAGS_VECTORCALL +#define SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS +#define SIMDLIB_METHOD_FLAGS_FORCE_INLINE inline +#define SIMDLIB_METHOD_FLAGS_FLATTEN +#include + +static_assert(!SimdLib::Config::method_flags_has_vectorcall); +static_assert(SimdLib::Config::method_flags_has_safe_buffers); +static_assert(!SimdLib::Config::method_flags_has_force_inline); +static_assert(SimdLib::Config::method_flags_has_flatten); + +/** @brief Exercises all caller-provided method-flags adapter definitions. */ +[[nodiscard]] int SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) MethodFlagsConfigOverrideProbe(const int value) noexcept +{ + return value; +} diff --git a/tests/config/MethodFlagsConfigUnsupportedTargetProbe.cpp b/tests/config/MethodFlagsConfigUnsupportedTargetProbe.cpp new file mode 100644 index 0000000..740d9ed --- /dev/null +++ b/tests/config/MethodFlagsConfigUnsupportedTargetProbe.cpp @@ -0,0 +1,20 @@ +#define SIMDLIB_COMPILER_CLANG 0 +#define SIMDLIB_COMPILER_MSVC 0 +#define SIMDLIB_COMPILER_GCC 0 +#define SIMDLIB_TARGET_X86 0 +#define SIMDLIB_TARGET_X64 0 +#define SIMDLIB_VECTORCALL_ENABLED 0 +#include + +static_assert(!SimdLib::Config::target_x86); +static_assert(!SimdLib::Config::target_x64); +static_assert(!SimdLib::Config::method_flags_has_vectorcall); +static_assert(!SimdLib::Config::method_flags_has_safe_buffers); +static_assert(!SimdLib::Config::method_flags_has_force_inline); +static_assert(!SimdLib::Config::method_flags_has_flatten); + +/** @brief Exercises every semantic flag when compiler mappings are unavailable. */ +[[nodiscard]] int SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) MethodFlagsConfigUnsupportedTargetProbe(const int value) noexcept +{ + return value; +} diff --git a/tests/constexpr/Api128Constexpr.tests.cpp b/tests/constexpr/Api128Constexpr.tests.cpp index 7ba8f00..051bf88 100644 --- a/tests/constexpr/Api128Constexpr.tests.cpp +++ b/tests/constexpr/Api128Constexpr.tests.cpp @@ -2,6 +2,17 @@ using namespace SimdLib::Tests::Constexpr; +static_assert(detail_lane_helper_contract<128, std::int8_t>()); +static_assert(detail_lane_helper_contract<128, std::uint8_t>()); +static_assert(detail_lane_helper_contract<128, std::int16_t>()); +static_assert(detail_lane_helper_contract<128, std::uint16_t>()); +static_assert(detail_lane_helper_contract<128, std::int32_t>()); +static_assert(detail_lane_helper_contract<128, std::uint32_t>()); +static_assert(detail_lane_helper_contract<128, std::int64_t>()); +static_assert(detail_lane_helper_contract<128, std::uint64_t>()); +static_assert(detail_lane_helper_contract<128, float>()); +static_assert(detail_lane_helper_contract<128, double>()); + static_assert(construction_contract<128, std::int8_t>()); static_assert(construction_contract<128, std::uint8_t>()); static_assert(construction_contract<128, std::int16_t>()); @@ -12,6 +23,7 @@ static_assert(construction_contract<128, std::int64_t>()); static_assert(construction_contract<128, std::uint64_t>()); static_assert(construction_contract<128, float>()); static_assert(construction_contract<128, double>()); +static_assert(setr_64bit_construction_contract()); static_assert(comparison_contract<128, std::int8_t>()); static_assert(comparison_contract<128, std::uint8_t>()); @@ -24,6 +36,17 @@ static_assert(comparison_contract<128, std::uint64_t>()); static_assert(comparison_contract<128, float>()); static_assert(comparison_contract<128, double>()); +static_assert(bitwise_contract<128, std::int8_t>()); +static_assert(bitwise_contract<128, std::uint8_t>()); +static_assert(bitwise_contract<128, std::int16_t>()); +static_assert(bitwise_contract<128, std::uint16_t>()); +static_assert(bitwise_contract<128, std::int32_t>()); +static_assert(bitwise_contract<128, std::uint32_t>()); +static_assert(bitwise_contract<128, std::int64_t>()); +static_assert(bitwise_contract<128, std::uint64_t>()); +static_assert(bitwise_contract<128, float>()); +static_assert(bitwise_contract<128, double>()); + static_assert(movemask_contract<128, std::int8_t>()); static_assert(movemask_contract<128, std::uint8_t>()); static_assert(movemask_contract<128, std::int16_t>()); @@ -52,5 +75,23 @@ static_assert(lane_shift_contract<128, std::int32_t>()); static_assert(lane_shift_contract<128, std::uint32_t>()); static_assert(lane_shift_contract<128, std::int64_t>()); static_assert(lane_shift_contract<128, std::uint64_t>()); +static_assert(immediate_blend_contract<128, std::int16_t>()); +static_assert(immediate_blend_contract<128, std::uint16_t>()); +static_assert(immediate_blend_contract<128, std::int32_t>()); +static_assert(immediate_blend_contract<128, std::uint32_t>()); +static_assert(immediate_blend_contract<128, float>()); +static_assert(immediate_blend_contract<128, double>()); +static_assert(logical_shuffle_contract<128, std::int8_t>()); +static_assert(logical_shuffle_contract<128, std::uint8_t>()); +static_assert(logical_shuffle_contract<128, std::int16_t>()); +static_assert(logical_shuffle_contract<128, std::uint16_t>()); +static_assert(logical_shuffle_contract<128, std::int32_t>()); +static_assert(logical_shuffle_contract<128, std::uint32_t>()); +static_assert(logical_shuffle_contract<128, std::int64_t>()); +static_assert(logical_shuffle_contract<128, std::uint64_t>()); +static_assert(logical_shuffle_contract<128, float>()); +static_assert(logical_shuffle_contract<128, double>()); + static_assert(whole_register_shift_contract()); -static_assert(simd_vector_contract<4>()); \ No newline at end of file +static_assert(immediate_byte_shift_contract<128>()); +static_assert(simd_vector_contract<4>()); diff --git a/tests/constexpr/Api256Constexpr.tests.cpp b/tests/constexpr/Api256Constexpr.tests.cpp index 938cd23..dd528a4 100644 --- a/tests/constexpr/Api256Constexpr.tests.cpp +++ b/tests/constexpr/Api256Constexpr.tests.cpp @@ -2,6 +2,17 @@ using namespace SimdLib::Tests::Constexpr; +static_assert(detail_lane_helper_contract<256, std::int8_t>()); +static_assert(detail_lane_helper_contract<256, std::uint8_t>()); +static_assert(detail_lane_helper_contract<256, std::int16_t>()); +static_assert(detail_lane_helper_contract<256, std::uint16_t>()); +static_assert(detail_lane_helper_contract<256, std::int32_t>()); +static_assert(detail_lane_helper_contract<256, std::uint32_t>()); +static_assert(detail_lane_helper_contract<256, std::int64_t>()); +static_assert(detail_lane_helper_contract<256, std::uint64_t>()); +static_assert(detail_lane_helper_contract<256, float>()); +static_assert(detail_lane_helper_contract<256, double>()); + static_assert(construction_contract<256, std::int8_t>()); static_assert(construction_contract<256, std::uint8_t>()); static_assert(construction_contract<256, std::int16_t>()); @@ -24,6 +35,17 @@ static_assert(comparison_contract<256, std::uint64_t>()); static_assert(comparison_contract<256, float>()); static_assert(comparison_contract<256, double>()); +static_assert(bitwise_contract<256, std::int8_t>()); +static_assert(bitwise_contract<256, std::uint8_t>()); +static_assert(bitwise_contract<256, std::int16_t>()); +static_assert(bitwise_contract<256, std::uint16_t>()); +static_assert(bitwise_contract<256, std::int32_t>()); +static_assert(bitwise_contract<256, std::uint32_t>()); +static_assert(bitwise_contract<256, std::int64_t>()); +static_assert(bitwise_contract<256, std::uint64_t>()); +static_assert(bitwise_contract<256, float>()); +static_assert(bitwise_contract<256, double>()); + static_assert(movemask_contract<256, std::int8_t>()); static_assert(movemask_contract<256, std::uint8_t>()); static_assert(movemask_contract<256, std::int16_t>()); @@ -52,4 +74,22 @@ static_assert(lane_shift_contract<256, std::int32_t>()); static_assert(lane_shift_contract<256, std::uint32_t>()); static_assert(lane_shift_contract<256, std::int64_t>()); static_assert(lane_shift_contract<256, std::uint64_t>()); +static_assert(immediate_blend_contract<256, std::int16_t>()); +static_assert(immediate_blend_contract<256, std::uint16_t>()); +static_assert(immediate_blend_contract<256, std::int32_t>()); +static_assert(immediate_blend_contract<256, std::uint32_t>()); +static_assert(immediate_blend_contract<256, float>()); +static_assert(immediate_blend_contract<256, double>()); +static_assert(logical_shuffle_contract<256, std::int8_t>()); +static_assert(logical_shuffle_contract<256, std::uint8_t>()); +static_assert(logical_shuffle_contract<256, std::int16_t>()); +static_assert(logical_shuffle_contract<256, std::uint16_t>()); +static_assert(logical_shuffle_contract<256, std::int32_t>()); +static_assert(logical_shuffle_contract<256, std::uint32_t>()); +static_assert(logical_shuffle_contract<256, std::int64_t>()); +static_assert(logical_shuffle_contract<256, std::uint64_t>()); +static_assert(logical_shuffle_contract<256, float>()); +static_assert(logical_shuffle_contract<256, double>()); + +static_assert(immediate_byte_shift_contract<256>()); static_assert(simd_vector_contract<8>()); diff --git a/tests/constexpr/ApiConstexprContracts.h b/tests/constexpr/ApiConstexprContracts.h index 6f3cd31..7b09094 100644 --- a/tests/constexpr/ApiConstexprContracts.h +++ b/tests/constexpr/ApiConstexprContracts.h @@ -1,5 +1,7 @@ #pragma once +#include "../LogicalShuffleTestSupport.h" + #include #include @@ -15,14 +17,67 @@ namespace SimdLib::Tests::Constexpr { + +/** + * @brief Expands one logical selector array into an Api shuffle during constant evaluation. + * @tparam api_t Api specialization under test. + * @tparam selectors Logical source-lane selectors. + * @tparam positions Output lane positions. + * @param value Source native register. + * @return Constant-evaluated shuffled native register. + */ +template +[[nodiscard]] constexpr auto logical_shuffle_value(typename api_t::vector_t value, std::index_sequence) noexcept +{ + return api_t::template shuffle(value); +} + +/** + * @brief Verifies one constant-evaluated Api shuffle against the scalar oracle. + * @tparam Width SIMD register width in bits. + * @tparam Element Logical lane type. + * @tparam selectors Logical source-lane selectors. + * @return True when every result lane preserves the oracle's object representation. + */ +template [[nodiscard]] consteval bool logical_shuffle_case() noexcept +{ + using api_t = Api; + constexpr auto source = LogicalShuffle::distinct_lanes(); + constexpr auto actual = + api_t::to_array(logical_shuffle_value(api_t::construct(source), std::make_index_sequence{})); + constexpr auto expected = LogicalShuffle::logical_shuffle_oracle(source); + return LogicalShuffle::same_object_representations(actual, expected); +} + +/** + * @brief Verifies nonidentity and repeated-selector constexpr logical shuffles. + * @tparam Width SIMD register width in bits. + * @tparam Element Logical lane type. + * @return True when every independent scalar-oracle comparison succeeds. + */ +template [[nodiscard]] consteval bool logical_shuffle_contract() noexcept +{ + if constexpr (Width == 256) + { + return logical_shuffle_case()>() && + logical_shuffle_case()>() && + logical_shuffle_case()>() && + logical_shuffle_case()>(); + } + else + { + return logical_shuffle_case()>() && + logical_shuffle_case()>(); + } +} + /** * @brief Creates a deterministic lane sequence for constexpr API contracts. * @tparam Width SIMD register width in bits. * @tparam Element SIMD lane type. * @return Lane values in increasing logical order. */ -template -[[nodiscard]] constexpr auto lane_values() noexcept +template [[nodiscard]] constexpr auto lane_values() noexcept { using simd = Api; std::array values{}; @@ -31,14 +86,40 @@ template return values; } +/** + * @brief Verifies the explicitly named portable lane helpers during constant evaluation. + * @tparam Width SIMD register width in bits. + * @tparam Element SIMD lane type. + * @return True when get, set, and value-returning insertion preserve the expected lanes. + */ +template [[nodiscard]] consteval bool detail_lane_helper_contract() noexcept +{ + using simd = Api; + constexpr auto values = lane_values(); + auto value = Detail::register_from_array(values); + + for (std::size_t index = 0; index < simd::element_count; ++index) + { + if (Detail::register_get_constexpr(value, index) != values[index]) + return false; + } + + constexpr std::size_t last = simd::element_count - 1; + value = Detail::register_insert_constexpr(value, values[0], last); + if (Detail::register_get_constexpr(value, last) != values[0]) + return false; + + Detail::register_set_constexpr(value, 0, values[last]); + return Detail::register_get_constexpr(value, 0) == values[last]; +} + /** * @brief Verifies constexpr construction, transfer, broadcast, and element access. * @tparam Width SIMD register width in bits. * @tparam Element SIMD lane type. * @return True when the public construction contract holds. */ -template -[[nodiscard]] consteval bool construction_contract() noexcept +template [[nodiscard]] consteval bool construction_contract() noexcept { using simd = Api; constexpr auto values = lane_values(); @@ -59,18 +140,37 @@ template return false; constexpr auto setrValue = [](std::index_sequence) constexpr noexcept - { - return simd::setr(static_cast(Indices + 1)...); - }(std::make_index_sequence{}); + { return simd::setr(static_cast(Indices + 1)...); }(std::make_index_sequence{}); if (simd::to_array(setrValue) != values) return false; - if (simd::get_element(constructed, 0) != values.front() || - simd::get_element(constructed, static_cast(simd::element_count - 1)) != values.back()) + if (simd::extract_slow(constructed, 0) != values.front() || simd::extract_slow(constructed, static_cast(simd::element_count - 1)) != values.back()) return false; constexpr Element replacement = static_cast(42); - const auto replaced = simd::set_element(constructed, static_cast(simd::element_count - 1), replacement); - return simd::get_element(replaced, static_cast(simd::element_count - 1)) == replacement; + const auto replaced = simd::insert_slow(constructed, replacement, static_cast(simd::element_count - 1)); + return simd::extract_slow(replaced, static_cast(simd::element_count - 1)) == replacement; +} + +/** + * @brief Verifies signed and unsigned 64-bit forward-order construction during constant evaluation. + * @return True when lane order and complete unsigned bit patterns are preserved. + */ +[[nodiscard]] consteval bool setr_64bit_construction_contract() noexcept +{ + using signed_words = Api<128, std::int64_t>; + constexpr std::array signed_values{ + std::numeric_limits::lowest(), + std::numeric_limits::max(), + }; + if (signed_words::to_array(signed_words::setr(signed_values[0], signed_values[1])) != signed_values) + return false; + + using unsigned_words = Api<128, std::uint64_t>; + constexpr std::array unsigned_values{ + 0x8000'0000'0000'0001ULL, + 0xFEDC'BA98'7654'3210ULL, + }; + return unsigned_words::to_array(unsigned_words::setr(unsigned_values[0], unsigned_values[1])) == unsigned_values; } /** @@ -80,8 +180,7 @@ template * @param offset Selects the left or right comparison pattern. * @return Comparison lanes containing equality and both order directions. */ -template -[[nodiscard]] constexpr auto comparison_values(const unsigned offset) noexcept +template [[nodiscard]] constexpr auto comparison_values(const unsigned offset) noexcept { using simd = Api; std::array values{}; @@ -104,10 +203,8 @@ template * @return Byte-granular comparison mask. */ template -[[nodiscard]] constexpr auto comparison_mask( - const std::array::element_count>& lhs, - const std::array::element_count>& rhs, - Predicate predicate) noexcept +[[nodiscard]] constexpr auto comparison_mask(const std::array::element_count> &lhs, + const std::array::element_count> &rhs, Predicate predicate) noexcept { using simd = Api; typename simd::mask_t result = 0; @@ -118,14 +215,35 @@ template return result; } +/** + * @brief Builds the lane-granular mask expected from a scalar comparison. + * @tparam Width SIMD register width in bits. + * @tparam Element SIMD lane type. + * @tparam Predicate Scalar comparison predicate type. + * @param lhs Left lane values. + * @param rhs Right lane values. + * @param predicate Scalar predicate applied to each lane pair. + * @return Mask containing one bit per matching lane. + */ +template +[[nodiscard]] constexpr auto comparison_slim_mask(const std::array::element_count> &lhs, + const std::array::element_count> &rhs, Predicate predicate) noexcept +{ + using simd = Api; + typename simd::mask_t result = 0; + for (std::size_t index = 0; index < lhs.size(); ++index) + if (predicate(lhs[index], rhs[index])) + result |= typename simd::mask_t{1} << index; + return result; +} + /** * @brief Verifies every public constexpr comparison helper for one SIMD shape. * @tparam Width SIMD register width in bits. * @tparam Element SIMD lane type. * @return True when equality and ordering masks match scalar predicates. */ -template -[[nodiscard]] consteval bool comparison_contract() noexcept +template [[nodiscard]] consteval bool comparison_contract() noexcept { using simd = Api; constexpr auto lhsValues = comparison_values(0); @@ -135,9 +253,76 @@ template constexpr auto equal = comparison_mask(lhsValues, rhsValues, [](const Element lhsValue, const Element rhsValue) { return lhsValue == rhsValue; }); constexpr auto greater = comparison_mask(lhsValues, rhsValues, [](const Element lhsValue, const Element rhsValue) { return lhsValue > rhsValue; }); constexpr auto less = comparison_mask(lhsValues, rhsValues, [](const Element lhsValue, const Element rhsValue) { return lhsValue < rhsValue; }); - return simd::cmp_eq(lhs, rhs) == equal && simd::cmp_eq_mask(lhs, rhs) == equal && - simd::cmp_gt(lhs, rhs) == greater && simd::cmp_ge(lhs, rhs) == (equal | greater) && - simd::cmp_lt(lhs, rhs) == less && simd::cmp_le(lhs, rhs) == (equal | less); + constexpr auto equalSlim = + comparison_slim_mask(lhsValues, rhsValues, [](const Element lhsValue, const Element rhsValue) { return lhsValue == rhsValue; }); + constexpr auto greaterSlim = + comparison_slim_mask(lhsValues, rhsValues, [](const Element lhsValue, const Element rhsValue) { return lhsValue > rhsValue; }); + constexpr auto lessSlim = + comparison_slim_mask(lhsValues, rhsValues, [](const Element lhsValue, const Element rhsValue) { return lhsValue < rhsValue; }); + using unsigned_element_t = select_unsigned_integer_t; + constexpr Element trueLane = std::bit_cast(std::numeric_limits::max()); + std::array equalLanes{}; + std::array greaterLanes{}; + std::array greaterEqualLanes{}; + std::array lessLanes{}; + std::array lessEqualLanes{}; + std::array selectedLanes{}; + for (std::size_t index = 0; index < lhsValues.size(); ++index) + { + equalLanes[index] = lhsValues[index] == rhsValues[index] ? trueLane : Element{}; + greaterLanes[index] = lhsValues[index] > rhsValues[index] ? trueLane : Element{}; + greaterEqualLanes[index] = lhsValues[index] >= rhsValues[index] ? trueLane : Element{}; + lessLanes[index] = lhsValues[index] < rhsValues[index] ? trueLane : Element{}; + lessEqualLanes[index] = lhsValues[index] <= rhsValues[index] ? trueLane : Element{}; + selectedLanes[index] = lhsValues[index] == rhsValues[index] ? lhsValues[index] : rhsValues[index]; + } + const auto matchesObjectRepresentation = [](const auto native, const auto &expected) constexpr noexcept + { return std::bit_cast>(simd::to_array(native)) == std::bit_cast>(expected); }; + return matchesObjectRepresentation(simd::compare_equal(lhs, rhs), equalLanes) && + matchesObjectRepresentation(simd::compare_greater(lhs, rhs), greaterLanes) && + matchesObjectRepresentation(simd::compare_greater_equal(lhs, rhs), greaterEqualLanes) && + matchesObjectRepresentation(simd::compare_less(lhs, rhs), lessLanes) && + matchesObjectRepresentation(simd::compare_less_equal(lhs, rhs), lessEqualLanes) && + matchesObjectRepresentation(simd::select(simd::compare_equal(lhs, rhs), lhs, rhs), selectedLanes) && simd::cmp_eq_mask(lhs, rhs) == equal && + simd::cmp_gt_mask(lhs, rhs) == greater && simd::cmp_ge_mask(lhs, rhs) == (equal | greater) && simd::cmp_lt_mask(lhs, rhs) == less && + simd::cmp_le_mask(lhs, rhs) == (equal | less) && simd::cmp_eq_slim(lhs, rhs) == equalSlim && simd::cmp_gt_slim(lhs, rhs) == greaterSlim && + simd::cmp_ge_slim(lhs, rhs) == (equalSlim | greaterSlim) && simd::cmp_lt_slim(lhs, rhs) == lessSlim && + simd::cmp_le_slim(lhs, rhs) == (equalSlim | lessSlim); +} + +/** + * @brief Verifies every public constexpr bitwise operation for one SIMD shape. + * @tparam Width SIMD register width in bits. + * @tparam Element SIMD lane type. + * @return True when all operations preserve the expected object-representation bits. + */ +template [[nodiscard]] consteval bool bitwise_contract() noexcept +{ + using simd = Api; + std::array left_bytes{}; + std::array right_bytes{}; + std::array expected_and{}; + std::array expected_or{}; + std::array expected_xor{}; + std::array expected_andnot{}; + std::array expected_not{}; + for (std::size_t byte = 0; byte < left_bytes.size(); ++byte) + { + left_bytes[byte] = static_cast(byte * 37u + 0x35u); + right_bytes[byte] = static_cast(byte * 19u + 0xA6u); + expected_and[byte] = left_bytes[byte] & right_bytes[byte]; + expected_or[byte] = left_bytes[byte] | right_bytes[byte]; + expected_xor[byte] = left_bytes[byte] ^ right_bytes[byte]; + expected_andnot[byte] = static_cast(~left_bytes[byte]) & right_bytes[byte]; + expected_not[byte] = static_cast(~left_bytes[byte]); + } + const auto lhs = simd::construct(std::bit_cast>(left_bytes)); + const auto rhs = simd::construct(std::bit_cast>(right_bytes)); + return std::bit_cast>(simd::to_array(simd::bitwise_and(lhs, rhs))) == expected_and && + std::bit_cast>(simd::to_array(simd::bitwise_or(lhs, rhs))) == expected_or && + std::bit_cast>(simd::to_array(simd::bitwise_xor(lhs, rhs))) == expected_xor && + std::bit_cast>(simd::to_array(simd::bitwise_andnot(lhs, rhs))) == expected_andnot && + std::bit_cast>(simd::to_array(simd::bitwise_not(lhs))) == expected_not; } /** @@ -145,8 +330,7 @@ template * @tparam Width SIMD register width in bits. * @return Byte sequence with varying sign bits. */ -template -[[nodiscard]] constexpr auto movemask_bytes() noexcept +template [[nodiscard]] constexpr auto movemask_bytes() noexcept { std::array bytes{}; for (std::size_t index = 0; index < bytes.size(); ++index) @@ -160,8 +344,7 @@ template * @tparam Element SIMD lane type. * @return Full register of lane values. */ -template -[[nodiscard]] constexpr auto movemask_values() noexcept +template [[nodiscard]] constexpr auto movemask_values() noexcept { using simd = Api; constexpr auto bytes = movemask_bytes(); @@ -175,8 +358,7 @@ template * @tparam Element SIMD lane type. * @return Expected byte-granular mask. */ -template -[[nodiscard]] constexpr auto expected_movemask() noexcept +template [[nodiscard]] constexpr auto expected_movemask() noexcept { using simd = Api; constexpr auto bytes = movemask_bytes(); @@ -192,8 +374,7 @@ template * @tparam Element SIMD lane type. * @return Expected element-granular mask. */ -template -[[nodiscard]] constexpr auto expected_movemask_slim() noexcept +template [[nodiscard]] constexpr auto expected_movemask_slim() noexcept { using simd = Api; constexpr auto bytes = movemask_bytes(); @@ -212,13 +393,11 @@ template * @tparam Element SIMD lane type. * @return True when both masks match scalar object-representation oracles. */ -template -[[nodiscard]] consteval bool movemask_contract() noexcept +template [[nodiscard]] consteval bool movemask_contract() noexcept { using simd = Api; constexpr auto value = simd::construct(movemask_values()); - return simd::movemask(value) == expected_movemask() && - simd::movemask_slim(value) == expected_movemask_slim(); + return simd::movemask(value) == expected_movemask() && simd::movemask_slim(value) == expected_movemask_slim(); } /** @@ -227,8 +406,7 @@ template * @tparam Element Integral SIMD lane type. * @return True when extrema positions match the prepared lane layout. */ -template -[[nodiscard]] consteval bool extrema_position_contract() noexcept +template [[nodiscard]] consteval bool extrema_position_contract() noexcept { using simd = Api; std::array values{}; @@ -239,8 +417,8 @@ template values[1] = std::numeric_limits::lowest(); const auto value = simd::construct(values); return simd::min_position(value) == 0 && simd::max_position(value) == simd::element_count - 1 && - simd::min_position(simd::set1(std::numeric_limits::lowest())) == 0 && - simd::max_position(simd::set1(std::numeric_limits::max())) == 0; + simd::min_position(simd::set1(std::numeric_limits::lowest())) == 0 && + simd::max_position(simd::set1(std::numeric_limits::max())) == 0; } /** @@ -249,28 +427,27 @@ template * @tparam Element Integral SIMD lane type. * @return True when zero, one, and final-valid-bit shifts match scalar values. */ -template -[[nodiscard]] consteval bool lane_shift_contract() noexcept +template [[nodiscard]] consteval bool lane_shift_contract() noexcept { using simd = Api; constexpr auto positive = simd::set1(static_cast(4)); if (simd::to_array(simd::shift_left(positive, 0)) != simd::to_array(positive) || - simd::get_element(simd::shift_left(positive, 1), 0) != static_cast(8) || - simd::get_element(simd::shift_right(positive, 1), 0) != static_cast(2)) + simd::extract_slow(simd::shift_left(positive, 1), 0) != static_cast(8) || + simd::extract_slow(simd::shift_right(positive, 1), 0) != static_cast(2)) return false; constexpr int finalShift = static_cast(sizeof(Element) * 8 - 1); constexpr int widthShift = static_cast(sizeof(Element) * 8); - if (simd::get_element(simd::shift_left(simd::set1(static_cast(1)), finalShift), 0) != - static_cast(std::make_unsigned_t{1} << finalShift) || + if (simd::extract_slow(simd::shift_left(simd::set1(static_cast(1)), finalShift), 0) != + static_cast(std::make_unsigned_t{1} << finalShift) || simd::to_array(simd::shift_left(positive, widthShift)) != std::array{} || simd::to_array(simd::shift_left(positive, widthShift + 1)) != std::array{} || simd::to_array(simd::shift_right(positive, widthShift)) != std::array{} || simd::to_array(simd::shift_right(positive, widthShift + 1)) != std::array{}) return false; if constexpr (std::is_signed_v) - return simd::get_element(simd::shift_right_arithmetic(simd::set1(static_cast(-8)), 1), 0) == static_cast(-4) && - simd::get_element(simd::shift_right_arithmetic(simd::set1(static_cast(-8)), widthShift), 0) == static_cast(-1) && - simd::get_element(simd::shift_right_arithmetic(simd::set1(static_cast(-8)), widthShift + 1), 0) == static_cast(-1); + return simd::extract_slow(simd::shift_right_arithmetic(simd::set1(static_cast(-8)), 1), 0) == static_cast(-4) && + simd::extract_slow(simd::shift_right_arithmetic(simd::set1(static_cast(-8)), widthShift), 0) == static_cast(-1) && + simd::extract_slow(simd::shift_right_arithmetic(simd::set1(static_cast(-8)), widthShift + 1), 0) == static_cast(-1); return true; } @@ -283,19 +460,41 @@ template using words = Api<128, std::uint64_t>; constexpr auto value = words::setr(std::uint64_t{1}, std::uint64_t{1} << 63); constexpr auto original = std::array{1, std::uint64_t{1} << 63}; - if (words::to_array(words::bit_shift_left(value, -1)) != original || - words::to_array(words::bit_shift_left(value, 0)) != original || - words::to_array(words::bit_shift_left(value, 64)) != std::array{0, 1} || - words::to_array(words::bit_shift_left(value, 127)) != std::array{0, std::uint64_t{1} << 63} || - words::to_array(words::bit_shift_left(value, 128)) != std::array{} || - words::to_array(words::bit_shift_left(value, 129)) != std::array{}) + if (words::to_array(words::shift_bits_left_slow(value, -1)) != original || words::to_array(words::shift_bits_left_slow(value, 0)) != original || + words::to_array(words::shift_bits_left_slow(value, 1)) != std::array{2, 0} || + words::to_array(words::shift_bits_left_slow(value, 63)) != std::array{std::uint64_t{1} << 63, 0} || + words::to_array(words::shift_bits_left_slow(value, 64)) != std::array{0, 1} || + words::to_array(words::shift_bits_left_slow(value, 65)) != std::array{0, 2} || + words::to_array(words::shift_bits_left_slow(value, 127)) != std::array{0, std::uint64_t{1} << 63} || + words::to_array(words::shift_bits_left_slow(value, 128)) != std::array{} || + words::to_array(words::shift_bits_left_slow(value, 129)) != std::array{}) + return false; + if (words::to_array(words::shift_bits_right_slow(value, -1)) != original || words::to_array(words::shift_bits_right_slow(value, 0)) != original || + words::to_array(words::shift_bits_right_slow(value, 1)) != std::array{0, std::uint64_t{1} << 62} || + words::to_array(words::shift_bits_right_slow(value, 63)) != std::array{0, 1} || + words::to_array(words::shift_bits_right_slow(value, 64)) != std::array{std::uint64_t{1} << 63, 0} || + words::to_array(words::shift_bits_right_slow(value, 65)) != std::array{std::uint64_t{1} << 62, 0} || + words::to_array(words::shift_bits_right_slow(value, 127)) != std::array{1, 0} || + words::to_array(words::shift_bits_right_slow(value, 128)) != std::array{} || + words::to_array(words::shift_bits_right_slow(value, 129)) != std::array{}) return false; - if (words::to_array(words::bit_shift_right(value, -1)) != original || - words::to_array(words::bit_shift_right(value, 0)) != original || - words::to_array(words::bit_shift_right(value, 64)) != std::array{std::uint64_t{1} << 63, 0} || - words::to_array(words::bit_shift_right(value, 127)) != std::array{1, 0} || - words::to_array(words::bit_shift_right(value, 128)) != std::array{} || - words::to_array(words::bit_shift_right(value, 129)) != std::array{}) + if (words::to_array(words::template shift_bits_left<0>(value)) != original || + words::to_array(words::template shift_bits_left<1>(value)) != std::array{2, 0} || + words::to_array(words::template shift_bits_left<63>(value)) != std::array{std::uint64_t{1} << 63, 0} || + words::to_array(words::template shift_bits_left<64>(value)) != std::array{0, 1} || + words::to_array(words::template shift_bits_left<65>(value)) != std::array{0, 2} || + words::to_array(words::template shift_bits_left<127>(value)) != std::array{0, std::uint64_t{1} << 63} || + words::to_array(words::template shift_bits_left<128>(value)) != std::array{} || + words::to_array(words::template shift_bits_left<129>(value)) != std::array{}) + return false; + if (words::to_array(words::template shift_bits_right<0>(value)) != original || + words::to_array(words::template shift_bits_right<1>(value)) != std::array{0, std::uint64_t{1} << 62} || + words::to_array(words::template shift_bits_right<63>(value)) != std::array{0, 1} || + words::to_array(words::template shift_bits_right<64>(value)) != std::array{std::uint64_t{1} << 63, 0} || + words::to_array(words::template shift_bits_right<65>(value)) != std::array{std::uint64_t{1} << 62, 0} || + words::to_array(words::template shift_bits_right<127>(value)) != std::array{1, 0} || + words::to_array(words::template shift_bits_right<128>(value)) != std::array{} || + words::to_array(words::template shift_bits_right<129>(value)) != std::array{}) return false; using bytes = Api<128, std::uint8_t>; @@ -305,21 +504,85 @@ template std::array right15{}; left15.back() = byteValues.front(); right15.front() = byteValues.back(); - return bytes::to_array(bytes::byte_shift_left(byteValue, -1)) == byteValues && - bytes::to_array(bytes::byte_shift_left(byteValue, 0)) == byteValues && - bytes::to_array(bytes::byte_shift_left(byteValue, 15)) == left15 && - bytes::to_array(bytes::byte_shift_left(byteValue, 16)) == std::array{} && - bytes::to_array(bytes::byte_shift_left(byteValue, 17)) == std::array{} && - bytes::to_array(bytes::byte_shift_right(byteValue, -1)) == byteValues && - bytes::to_array(bytes::byte_shift_right(byteValue, 0)) == byteValues && - bytes::to_array(bytes::byte_shift_right(byteValue, 15)) == right15 && - bytes::to_array(bytes::byte_shift_right(byteValue, 16)) == std::array{} && - bytes::to_array(bytes::byte_shift_right(byteValue, 17)) == std::array{}; + return bytes::to_array(bytes::shift_bytes_left_slow(byteValue, -1)) == byteValues && + bytes::to_array(bytes::shift_bytes_left_slow(byteValue, 0)) == byteValues && + bytes::to_array(bytes::shift_bytes_left_slow(byteValue, 15)) == left15 && + bytes::to_array(bytes::shift_bytes_left_slow(byteValue, 16)) == std::array{} && + bytes::to_array(bytes::shift_bytes_left_slow(byteValue, 17)) == std::array{} && + bytes::to_array(bytes::shift_bytes_right_slow(byteValue, -1)) == byteValues && + bytes::to_array(bytes::shift_bytes_right_slow(byteValue, 0)) == byteValues && + bytes::to_array(bytes::shift_bytes_right_slow(byteValue, 15)) == right15 && + bytes::to_array(bytes::shift_bytes_right_slow(byteValue, 16)) == std::array{} && + bytes::to_array(bytes::shift_bytes_right_slow(byteValue, 17)) == std::array{}; +} + +/** + * @brief Verifies one immediate complete-register byte shift during constant evaluation. + * @tparam Width SIMD register width in bits. + * @tparam Count Compile-time byte count. + * @return `true` when both directions match an independent scalar byte oracle. + */ +template [[nodiscard]] consteval bool immediate_byte_shift_count_contract() noexcept +{ + using api = Api; + std::array source{}; + std::array expected_left{}; + std::array expected_right{}; + for (std::size_t index = 0; index < source.size(); ++index) + source[index] = static_cast(index * 7 + 1); + if constexpr (Count < api::byte_count) + { + for (std::size_t index = Count; index < source.size(); ++index) + expected_left[index] = source[index - Count]; + for (std::size_t index = 0; index + Count < source.size(); ++index) + expected_right[index] = source[index + Count]; + } + const auto value = api::construct(source); + const auto left = api::to_array(api::template shift_bytes_left(Count)>(value)); + const auto right = api::to_array(api::template shift_bytes_right(Count)>(value)); + if (left != expected_left || right != expected_right) + return false; + if constexpr (Width == 128) + return left == api::to_array(api::template shift_bits_left(Count * 8)>(value)) && + right == api::to_array(api::template shift_bits_right(Count * 8)>(value)); + return true; } +/** + * @brief Verifies all required immediate byte-shift boundary counts during constant evaluation. + * @tparam Width SIMD register width in bits. + * @return `true` when every required count passes in both directions. + */ +template [[nodiscard]] consteval bool immediate_byte_shift_contract() noexcept +{ + return immediate_byte_shift_count_contract() && immediate_byte_shift_count_contract() && + immediate_byte_shift_count_contract() && immediate_byte_shift_count_contract() && + immediate_byte_shift_count_contract() && immediate_byte_shift_count_contract() && + immediate_byte_shift_count_contract() && immediate_byte_shift_count_contract() && + immediate_byte_shift_count_contract() && immediate_byte_shift_count_contract(); +} +/** + * @brief Verifies immediate blend through the implementation-layer constant-evaluation entry point. + * @tparam Width SIMD register width in bits. + * @tparam Element Lane type supported by immediate blend. + * @return `true` when the compile-time mask selects the expected lanes. + */ +template [[nodiscard]] consteval bool immediate_blend_contract() noexcept +{ + using api = Api; + std::array left{}; + std::array right{}; + std::array expected{}; + for (std::size_t index = 0; index < api::element_count; ++index) + { + left[index] = static_cast(index + 1); + right[index] = static_cast(index + 33); + expected[index] = (0xA5u & (1u << (index % 8))) != 0 ? right[index] : left[index]; + } + return api::to_array(api::template blend<0xA5>(api::construct(left), api::construct(right))) == expected; +} /** @brief Result bundle shared by constexpr and forced-runtime parity checks. */ -template -struct ApiContractSnapshot final +template struct ApiContractSnapshot final { using simd = Api; std::array lanes{}; @@ -329,7 +592,7 @@ struct ApiContractSnapshot final std::size_t maximumPosition{}; /** @brief Compares all observable snapshot fields. */ - friend constexpr bool operator==(const ApiContractSnapshot&, const ApiContractSnapshot&) noexcept = default; + friend constexpr bool operator==(const ApiContractSnapshot &, const ApiContractSnapshot &) noexcept = default; }; /** @@ -341,18 +604,14 @@ struct ApiContractSnapshot final */ template [[nodiscard]] constexpr ApiContractSnapshot evaluate_api_contract( - const std::array::element_count>& lhsValues, - const std::array::element_count>& rhsValues) noexcept + const std::array::element_count> &lhsValues, + const std::array::element_count> &rhsValues) noexcept { using simd = Api; const auto lhs = simd::construct(lhsValues); const auto rhs = simd::construct(rhsValues); return { - simd::to_array(simd::shift_left(lhs, 1)), - simd::cmp_eq(lhs, rhs), - simd::cmp_gt(lhs, rhs), - simd::min_position(lhs), - simd::max_position(lhs), + simd::to_array(simd::shift_left(lhs, 1)), simd::cmp_eq_mask(lhs, rhs), simd::cmp_gt_mask(lhs, rhs), simd::min_position(lhs), simd::max_position(lhs), }; } @@ -361,8 +620,7 @@ template * @tparam ElementCount Logical vector lane count. * @return True when the default, array, and broadcast constructors are constant evaluable. */ -template -[[nodiscard]] consteval bool simd_vector_contract() noexcept +template [[nodiscard]] consteval bool simd_vector_contract() noexcept { using vector = SimdVector(ElementCount)>; std::array values{}; @@ -376,4 +634,4 @@ template (void)broadcast; return true; } -} // namespace SimdLib::Tests::Constexpr \ No newline at end of file +} // namespace SimdLib::Tests::Constexpr diff --git a/tests/constexpr/BmiConstexpr.tests.cpp b/tests/constexpr/BmiConstexpr.tests.cpp index d2e48f8..3c7ab80 100644 --- a/tests/constexpr/BmiConstexpr.tests.cpp +++ b/tests/constexpr/BmiConstexpr.tests.cpp @@ -127,10 +127,10 @@ static_assert(extract_bits_higher_than(0b10111, 0b00001) == 0b101 /** * @brief Expands the BMI constexpr contract across one integral width and signedness. * @tparam Integer Integral type under test. - * @return True when representative generic helpers preserve their bit contracts. + * @return True when + * representative generic helpers preserve their bit contracts. */ -template -[[nodiscard]] consteval bool bmi_width_contract() noexcept +template [[nodiscard]] consteval bool bmi_width_contract() noexcept { using unsigned_type = std::make_unsigned_t; constexpr unsigned_type value = static_cast(0b10110100); @@ -140,12 +140,12 @@ template Integer high{}; const Integer low = mulx(static_cast(3), static_cast(7), high); return static_cast(andn(typedMask, typedValue)) == static_cast((~mask) & value) && - static_cast(bzhi(typedValue, 4)) == static_cast(value & 0x0F) && - static_cast(blsi(typedValue)) == static_cast(value & (unsigned_type{0} - value)) && - static_cast(blsr(typedValue)) == static_cast(value & (value - 1)) && - static_cast(pdep_u32(static_cast(value), static_cast(mask))) == 0x20u && - static_cast(pext_u32(static_cast(value), static_cast(mask))) == 0x04u && - low == static_cast(21) && high == Integer{}; + static_cast(bzhi(typedValue, 4)) == static_cast(value & 0x0F) && + static_cast(blsi(typedValue)) == static_cast(value & (unsigned_type{0} - value)) && + static_cast(blsr(typedValue)) == static_cast(value & (value - 1)) && + static_cast(pdep_u32(static_cast(value), static_cast(mask))) == 0x20u && + static_cast(pext_u32(static_cast(value), static_cast(mask))) == 0x04u && + low == static_cast(21) && high == Integer{}; } static_assert(blsmsk(0b10100) == 0b00111); @@ -161,4 +161,3 @@ static_assert(bmi_width_contract()); static_assert(bmi_width_contract()); static_assert(bmi_width_contract()); } // namespace SimdLib::Bmi - diff --git a/tests/constexpr/LogicalShuffleOracle.tests.cpp b/tests/constexpr/LogicalShuffleOracle.tests.cpp new file mode 100644 index 0000000..4645fde --- /dev/null +++ b/tests/constexpr/LogicalShuffleOracle.tests.cpp @@ -0,0 +1,134 @@ +#include "../LogicalShuffleTestSupport.h" + +#include +#include +#include +#include + +namespace +{ + +using namespace SimdLib::Tests::LogicalShuffle; + +/** + * @brief Reports whether every selector remains inside its output's 128-bit group. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @tparam selectors Logical source-lane selectors. + * @return True when the complete selector array obeys the group-local contract. + */ +template [[nodiscard]] consteval bool selectors_are_group_local() noexcept +{ + constexpr std::size_t lane_count = bits / (sizeof(element_t) * 8); + constexpr std::size_t lanes_per_group = 128 / (sizeof(element_t) * 8); + if (selectors.size() != lane_count) + return false; + for (std::size_t output = 0; output < lane_count; ++output) + if (selectors[output] >= lane_count || selectors[output] / lanes_per_group != output / lanes_per_group) + return false; + return true; +} + +/** + * @brief Validates selector construction and the scalar oracle for one SIMD shape. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return True when all required selector patterns produce their manually defined lanes. + */ +template [[nodiscard]] consteval bool oracle_contract() noexcept +{ + constexpr std::size_t lanes_per_group = 128 / (sizeof(element_t) * 8); + constexpr auto source = distinct_lanes(); + constexpr auto identity = identity_selectors(); + constexpr auto reverse = reverse_selectors(); + constexpr auto first = first_lane_selectors(); + constexpr auto last = last_lane_selectors(); + constexpr auto repeated = repeated_selectors(); + constexpr auto pair_swap = pair_swap_selectors(); + constexpr auto rotation = rotation_selectors(); + static_assert(selectors_are_group_local()); + static_assert(selectors_are_group_local()); + static_assert(selectors_are_group_local()); + static_assert(selectors_are_group_local()); + static_assert(selectors_are_group_local()); + static_assert(selectors_are_group_local()); + static_assert(selectors_are_group_local()); + + constexpr auto identity_result = logical_shuffle_oracle(source); + constexpr auto reverse_result = logical_shuffle_oracle(source); + constexpr auto first_result = logical_shuffle_oracle(source); + constexpr auto last_result = logical_shuffle_oracle(source); + constexpr auto repeated_result = logical_shuffle_oracle(source); + constexpr auto pair_swap_result = logical_shuffle_oracle(source); + constexpr auto rotation_result = logical_shuffle_oracle(source); + for (std::size_t lane = 0; lane < source.size(); ++lane) + { + const std::size_t group = lane / lanes_per_group * lanes_per_group; + const std::size_t local = lane % lanes_per_group; + if (std::bit_cast>(identity_result[lane]) != std::bit_cast>(source[lane]) || + std::bit_cast>(reverse_result[lane]) != + std::bit_cast>(source[group + lanes_per_group - 1 - local]) || + std::bit_cast>(first_result[lane]) != std::bit_cast>(source[group]) || + std::bit_cast>(last_result[lane]) != std::bit_cast>(source[group + lanes_per_group - 1]) || + std::bit_cast>(repeated_result[lane]) != std::bit_cast>(source[group + local / 2]) || + std::bit_cast>(pair_swap_result[lane]) != std::bit_cast>(source[lane ^ std::size_t{1}]) || + std::bit_cast>(rotation_result[lane]) != + std::bit_cast>(source[group + (local + 1) % lanes_per_group])) + return false; + } + if constexpr (bits == 256) + { + constexpr auto distinct_groups = distinct_group_selectors(); + static_assert(selectors_are_group_local()); + constexpr auto result = logical_shuffle_oracle(source); + for (std::size_t lane = 0; lane < lanes_per_group; ++lane) + { + if (std::bit_cast>(result[lane]) != std::bit_cast>(source[lane]) || + std::bit_cast>(result[lanes_per_group + lane]) != + std::bit_cast>(source[2 * lanes_per_group - 1 - lane])) + return false; + } + constexpr auto swapped_halves = swap_half_selectors(); + constexpr auto mixed_halves = mixed_half_selectors(); + constexpr auto full_reverse = full_reverse_selectors(); + static_assert(!selectors_are_group_local()); + static_assert(!selectors_are_group_local()); + static_assert(!selectors_are_group_local()); + constexpr auto swapped_result = logical_shuffle_oracle(source); + constexpr auto mixed_result = logical_shuffle_oracle(source); + constexpr auto full_reverse_result = logical_shuffle_oracle(source); + for (std::size_t lane = 0; lane < source.size(); ++lane) + { + const std::size_t opposite = (lane + lanes_per_group) % source.size(); + const std::size_t mixed_source = lane == 0 ? lanes_per_group : (lane == lanes_per_group ? 0 : lane); + if (std::bit_cast>(swapped_result[lane]) != std::bit_cast>(source[opposite]) || + std::bit_cast>(mixed_result[lane]) != std::bit_cast>(source[mixed_source]) || + std::bit_cast>(full_reverse_result[lane]) != std::bit_cast>(source[source.size() - 1 - lane])) + return false; + } + } + return true; +} + +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); +static_assert(oracle_contract()); + +} // namespace diff --git a/tests/constexpr/RegisterConstexpr.tests.cpp b/tests/constexpr/RegisterConstexpr.tests.cpp new file mode 100644 index 0000000..2854bfe --- /dev/null +++ b/tests/constexpr/RegisterConstexpr.tests.cpp @@ -0,0 +1,594 @@ +#include "../LogicalShuffleTestSupport.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +/** @brief Compile-time list of supported Register element types. */ +template struct register_element_types +{ +}; + +using supported_register_element_types = + register_element_types; + +/** + * @brief Expands one logical selector array into a Register shuffle during constant evaluation. + * @tparam register_t Register specialization under test. + * @tparam selectors Logical source-lane selectors. + * @tparam positions Output lane positions. + * @param value Source Register. + * @return Constant-evaluated shuffled Register. + */ +template +[[nodiscard]] consteval register_t register_logical_shuffle_value(register_t value, std::index_sequence) noexcept +{ + return value.template shuffle(); +} + +/** + * @brief Verifies one constant-evaluated Register shuffle against the scalar oracle. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @tparam selectors Logical source-lane selectors. + * @return True when every result lane preserves the oracle's object representation. + */ +template [[nodiscard]] consteval bool register_logical_shuffle_case() noexcept +{ + using register_t = SimdLib::Register; + constexpr auto source = SimdLib::Tests::LogicalShuffle::distinct_lanes(); + constexpr register_t source_register = register_t::from_array(source); + constexpr register_t shuffled = register_logical_shuffle_value(source_register, std::make_index_sequence{}); + constexpr auto actual = shuffled.to_array(); + constexpr auto expected = SimdLib::Tests::LogicalShuffle::logical_shuffle_oracle(source); + return SimdLib::Tests::LogicalShuffle::same_object_representations(actual, expected); +} + +/** + * @brief Verifies nonidentity and repeated-selector constexpr Register shuffles. + * @tparam element_t Logical lane type. + * @tparam bits Register width in bits. + * @return True when both independent scalar-oracle comparisons succeed. + */ +template [[nodiscard]] consteval bool register_logical_shuffle_contract() noexcept +{ + return register_logical_shuffle_case()>() && + register_logical_shuffle_case()>(); +} + +/** + * @brief Expands one byte-selector array into a Register byte shuffle during constant evaluation. + * @tparam register_t Register specialization under test. + * @tparam selectors Source-byte selectors. + * @tparam positions Output byte positions. + * @param value Source Register. + * @return Constant-evaluated byte-shuffled Register. + */ +template +[[nodiscard]] consteval register_t register_byte_shuffle_value(register_t value, std::index_sequence) noexcept +{ + return value.template shuffle_bytes(); +} + +/** + * @brief Verifies one constant-evaluated Register byte shuffle against the scalar byte oracle. + * @tparam element_t Logical lane type retained by the result. + * @tparam bits Register width in bits. + * @tparam selectors Source-byte selectors. + * @return True when every result byte matches the independently selected source byte. + */ +template [[nodiscard]] consteval bool register_byte_shuffle_case() noexcept +{ + using register_t = SimdLib::Register; + constexpr auto source = SimdLib::Tests::LogicalShuffle::distinct_lanes(); + constexpr register_t source_register = register_t::from_array(source); + constexpr register_t shuffled = register_byte_shuffle_value(source_register, std::make_index_sequence{}); + constexpr auto actual = std::bit_cast>(shuffled.to_array()); + constexpr auto source_bytes = std::bit_cast>(source); + constexpr auto expected = SimdLib::Tests::LogicalShuffle::logical_shuffle_oracle(source_bytes); + return actual == expected; +} + +/** + * @brief Verifies local and cross-half constant-evaluated Register byte shuffles. + * @tparam element_t Logical lane type retained by the result. + * @tparam bits Register width in bits. + * @return True when every independent scalar-oracle comparison succeeds. + */ +template [[nodiscard]] consteval bool register_byte_shuffle_contract() noexcept +{ + if constexpr (bits == 128) + return register_byte_shuffle_case()>(); + else + return register_byte_shuffle_case()>() && + register_byte_shuffle_case()>(); +} + +/** @brief Constructs a register from an expanded compile-time lane array. */ +template +[[nodiscard]] consteval register_t from_lanes(const std::array &values, + std::index_sequence) noexcept +{ + return register_t::from_lanes(values[indices]...); +} + +/** @brief Verifies all constant-evaluable Register construction and lane operations. */ +template [[nodiscard]] consteval bool register_constexpr_contract() noexcept +{ + using register_type = SimdLib::Register; + std::array values{}; + for (std::size_t index = 0; index < values.size(); ++index) + values[index] = static_cast(index + 1); + std::array broadcast_values{}; + broadcast_values.fill(static_cast(7)); + const std::array zeros{}; +#if SIMDLIB_COMPILER_MSVC + const register_type value{}; + const register_type zero = register_type::zero(); + const register_type broadcast = register_type::broadcast(static_cast(7)); + const register_type array_value = register_type::from_array(values); + const register_type lane_value = from_lanes(values, std::make_index_sequence{}); + const register_type native_value{array_value.native}; + const element_t first_lane = array_value.template lane<0>(); + const register_type changed_value = array_value.template with_lane(static_cast(43)); + (void)value; + (void)zero; + (void)broadcast; + (void)lane_value; + (void)native_value; + return first_lane == values.front() && changed_value.template lane() == static_cast(43); +#else + if (register_type{}.to_array() != zeros || register_type::zero().to_array() != zeros) + return false; + if (register_type::broadcast(static_cast(7)).to_array() != broadcast_values) + return false; + const auto array_value = register_type::from_array(values); + if (array_value.to_array() != values) + return false; + if (from_lanes(values, std::make_index_sequence{}).to_array() != values) + return false; + const register_type native_value{array_value.native}; + if (native_value.to_array() != values || array_value.template lane<0>() != values.front() || + array_value.template lane() != values.back()) + return false; + const auto changed_lanes = array_value.template with_lane(static_cast(43)).to_array(); + return changed_lanes.front() == values.front() && changed_lanes.back() == static_cast(43); +#endif +} + +/** @brief Verifies constant-evaluated mask comparisons, combination, reductions, and selection. */ +template [[nodiscard]] consteval bool register_mask_constexpr_contract() noexcept +{ + using register_type = SimdLib::Register; + using mask_type = typename register_type::mask_type; +#if SIMDLIB_COMPILER_MSVC + const mask_type mask{}; + (void)mask; + return true; +#else + std::array left{}; + std::array right{}; + for (std::size_t index = 0; index < left.size(); ++index) + { + left[index] = static_cast((index % 2) == 0 ? 2 : 0); + right[index] = static_cast(1); + } + typename mask_type::bits_type expected = 0; + for (std::size_t index = 0; index < mask_type::lane_count; index += 2) + expected |= typename mask_type::bits_type{1} << index; + const auto lhs = register_type::from_array(left); + const auto rhs = register_type::from_array(right); + const auto greater = lhs.compare_greater(rhs); + const auto less = lhs.compare_less(rhs); + const mask_type rewrapped{greater.native}; + if (greater.bits() != expected || greater.none() || !greater.any() || greater.all()) + return false; + if (rewrapped.bits() != expected) + return false; + if (!(greater | less).all() || !(greater & less).none() || (greater ^ less).bits() != (greater | less).bits() || !(~(greater | less)).none()) + return false; + const auto selected = greater.select(register_type::broadcast(static_cast(11)), register_type::broadcast(static_cast(22))).to_array(); + for (std::size_t index = 0; index < selected.size(); ++index) + { + if (selected[index] != static_cast((index % 2) == 0 ? 11 : 22)) + return false; + } + return lhs == lhs && lhs != rhs && lhs.compare_equal(lhs).all() && lhs.compare_greater_equal(rhs).bits() == expected && + lhs.compare_less_equal(rhs).bits() == less.bits(); +#endif +} + +/** @brief Verifies constant-evaluated first-minimum and first-maximum position reductions. */ +template + requires std::is_integral_v +[[nodiscard]] consteval bool register_position_constexpr_contract() noexcept +{ + using register_type = SimdLib::Register; + std::array values{}; + values.fill(static_cast(7)); + values[0] = static_cast(1); + values[register_type::lane_count - 1] = static_cast(12); + const auto value = register_type::from_array(values); + return value.min_position() == 0 && value.max_position() == register_type::lane_count - 1; +} + +/** @brief Verifies constant-evaluated bitwise expressions, assignments, and sign reductions. */ +template [[nodiscard]] consteval bool register_bitwise_constexpr_contract() noexcept +{ + using register_type = SimdLib::Register; + const auto value = register_type::broadcast(static_cast(-1)); + const auto zero = register_type::zero(); +#if SIMDLIB_COMPILER_MSVC + const auto intersection = value & value; + const auto combined = value | zero; + const auto toggled = value ^ value; + const auto inverted = ~~value; + const auto excluded = value.andnot(value); + auto reassigned = value; + reassigned = reassigned & value; + reassigned = reassigned | zero; + reassigned = reassigned ^ value; + (void)intersection; + (void)combined; + (void)toggled; + (void)inverted; + (void)excluded; + (void)reassigned; + return true; +#else + if ((value & value).to_array() != value.to_array() || (value | zero).to_array() != value.to_array() || (value ^ value).to_array() != zero.to_array() || + (~~value).to_array() != value.to_array() || value.andnot(value).to_array() != zero.to_array()) + return false; + auto reassigned = value; + reassigned = reassigned & value; + reassigned = reassigned | zero; + reassigned = reassigned ^ value; + return reassigned.to_array() == zero.to_array() && value.lane_sign_bits() != 0 && value.movemask() != 0; +#endif +} + +/** @brief Verifies constant-evaluated per-lane shift boundary semantics. */ +template + requires std::is_integral_v +[[nodiscard]] consteval bool register_lane_shift_constexpr_contract() noexcept +{ + using register_type = SimdLib::Register; + using unsigned_type = std::make_unsigned_t; + constexpr int lane_width = std::numeric_limits::digits; + constexpr unsigned_type high_bit = unsigned_type{1} << (lane_width - 1); + const auto value = register_type::broadcast(std::bit_cast(high_bit)); +#if SIMDLIB_COMPILER_MSVC + const auto left = value << lane_width; + const auto logical = value.logical_shift_right(lane_width - 1); + const auto right = value >> (lane_width + 1); + (void)left; + (void)logical; + (void)right; + return true; +#else + const auto zeros = register_type::zero().to_array(); + if ((value << 0).to_array() != value.to_array() || (value << lane_width).to_array() != zeros || (value << (lane_width + 1)).to_array() != zeros || + value.logical_shift_right(lane_width).to_array() != zeros || value.logical_shift_right(lane_width + 1).to_array() != zeros) + return false; + for (const auto lane : value.logical_shift_right(lane_width - 1).to_array()) + if (lane != element_t{1}) + return false; + if constexpr (std::is_signed_v) + { + for (const auto lane : (value >> lane_width).to_array()) + if (lane != element_t{-1}) + return false; + } + else if ((value >> lane_width).to_array() != zeros) + return false; + auto reassigned = value; + reassigned = reassigned << lane_width; + reassigned = value; + reassigned = reassigned >> (lane_width + 1); + return true; +#endif +} + +/** @brief Verifies constant-evaluated 128-bit byte and static whole-register shifts. */ +[[nodiscard]] consteval bool register_complete_shift_constexpr_contract() noexcept +{ + using register_type = SimdLib::Register; + std::array lanes{}; + for (std::size_t index = 0; index < lanes.size(); ++index) + lanes[index] = static_cast(index + 1); + const auto value = register_type::from_array(lanes); +#if SIMDLIB_COMPILER_MSVC + const auto bytes = value.shift_bytes_left_slow(1); + (void)bytes; + return true; +#else + const auto zeros = register_type::zero().to_array(); + return value.shift_bytes_left_slow(0).to_array() == lanes && value.shift_bytes_left_slow(16).to_array() == zeros && + value.shift_bytes_left_slow(17).to_array() == zeros && value.shift_bytes_right_slow(16).to_array() == zeros && + value.shift_bits_left_slow(128).to_array() == zeros && value.shift_bits_right_slow(128).to_array() == zeros && + value.template shift_bits_left<128>().to_array() == zeros && value.template shift_bits_left<129>().to_array() == zeros && + value.template shift_bits_right<128>().to_array() == zeros && value.template shift_bits_right<129>().to_array() == zeros; +#endif +} + +/** + * @brief Verifies one Register immediate byte shift during constant evaluation. + * @tparam Width SIMD register width in bits. + * @tparam Count Compile-time byte count. + * @return `true` when both directions match a scalar byte oracle. + */ +template [[nodiscard]] consteval bool register_immediate_byte_shift_count_contract() noexcept +{ + using register_type = SimdLib::Register; + std::array source{}; + std::array expected_left{}; + std::array expected_right{}; + for (std::size_t index = 0; index < source.size(); ++index) + source[index] = static_cast(index * 7 + 1); + if constexpr (Count < register_type::byte_count) + { + for (std::size_t index = Count; index < source.size(); ++index) + expected_left[index] = source[index - Count]; + for (std::size_t index = 0; index + Count < source.size(); ++index) + expected_right[index] = source[index + Count]; + } + const auto value = register_type::from_array(source); +#if SIMDLIB_COMPILER_MSVC + const auto shifted_left = value.template shift_bytes_left(Count)>(); + const auto shifted_right = value.template shift_bytes_right(Count)>(); + (void)shifted_left; + (void)shifted_right; + return true; +#else + return value.template shift_bytes_left(Count)>().to_array() == expected_left && + value.template shift_bytes_right(Count)>().to_array() == expected_right; +#endif +} + +/** @brief Verifies every supported bit-cast and numeric-conversion constexpr cell for one source and target type. */ +template [[nodiscard]] consteval bool register_conversion_constexpr_cell() noexcept +{ + using source_register = SimdLib::Register; + const auto source = source_register::broadcast(static_cast(1)); + if constexpr (SimdLib::IRegister::BitCast) + { +#if SIMDLIB_COMPILER_MSVC + (void)source; +#else + const auto round_trip = source.template bit_cast().template bit_cast(); + if (round_trip.template lane<0>() != static_cast(1)) + return false; +#endif + } + if constexpr (SimdLib::IRegister::Convert) + { + const auto converted = source.template convert(); + if (converted.template lane<0>() != static_cast(1)) + return false; + } + return true; +} + +/** @brief Verifies every target type for one source type in the constexpr conversion matrix. */ +template +[[nodiscard]] consteval bool register_conversion_constexpr_targets(register_element_types) noexcept +{ + return (register_conversion_constexpr_cell() && ...); +} + +/** @brief Verifies one supported or rejected low-lane widening constexpr cell. */ +template +[[nodiscard]] consteval bool register_widen_constexpr_cell() noexcept +{ + using source_register = SimdLib::Register; + const auto source = source_register::broadcast(static_cast(1)); + if constexpr (SimdLib::IRegister::WidenLow) + { + const auto widened = source.template widen_low(); + return widened.template lane<0>() == static_cast(1); + } + return true; +} + +/** @brief Verifies both supported destination widths for one widening source and target type. */ +template [[nodiscard]] consteval bool register_widen_constexpr_widths() noexcept +{ + return register_widen_constexpr_cell() && register_widen_constexpr_cell(); +} + +/** @brief Verifies every widening target type for one source type. */ +template +[[nodiscard]] consteval bool register_widen_constexpr_targets(register_element_types) noexcept +{ + return (register_widen_constexpr_widths() && ...); +} + +/** @brief Verifies constant-evaluated rearrangement, reinterpretation, numeric conversion, and widening. */ +template [[nodiscard]] consteval bool register_rearrangement_conversion_constexpr_contract() noexcept +{ +#if SIMDLIB_COMPILER_MSVC + return SimdLib::IRegister::UnpackLow> && + SimdLib::IRegister::ShuffleLow, 0x1B> && SimdLib::IRegister::Blend, 0xA5> && + SimdLib::IRegister::BitCast, float> && + SimdLib::IRegister::Convert, std::int32_t>; +#else + using bytes_t = SimdLib::Register; + using words_t = SimdLib::Register; + using ints_t = SimdLib::Register; + using floats_t = SimdLib::Register; + std::array bytes{}; + std::array words{}; + std::array ints{}; + for (std::size_t lane = 0; lane < bytes.size(); ++lane) + bytes[lane] = static_cast(lane + 1); + for (std::size_t lane = 0; lane < words.size(); ++lane) + words[lane] = static_cast(lane + 1); + for (std::size_t lane = 0; lane < ints.size(); ++lane) + ints[lane] = static_cast(lane + 1); + + const auto byte_value = bytes_t::from_array(bytes); + const auto word_value = words_t::from_array(words); + const auto int_value = ints_t::from_array(ints); + const auto unpacked = int_value.unpack_low(ints_t::broadcast(40)); + const auto unpacked_high = int_value.unpack_high(ints_t::broadcast(40)); + const auto low_shuffle = word_value.template shuffle_low<0x1B>(); + const auto high_shuffle = word_value.template shuffle_high<0x1B>(); + const auto blended = word_value.template blend<0xA5>(words_t::broadcast(70)); + const auto reinterpreted = int_value.template bit_cast().template bit_cast(); + const auto converted = int_value.template convert(); + const auto rounded = floats_t::broadcast(2.5F).template convert(); + const auto widened = + SimdLib::Register::from_lanes(-8, -7, -6, -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, 6, 7, 8).template widen_low(); + if constexpr (bits == 128) + { + const auto shuffled = byte_value.template shuffle<15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0>(); + (void)shuffled; + } + else + { + const auto shuffled = + byte_value.template shuffle<15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16>(); + const auto lower = int_value.lower_half(); + (void)shuffled; + (void)lower; + } + + const auto unpacked_lanes = unpacked.to_array(); + const auto unpacked_high_lanes = unpacked_high.to_array(); + const auto low_lanes = low_shuffle.to_array(); + const auto high_lanes = high_shuffle.to_array(); + const auto blend_lanes = blended.to_array(); + if (unpacked_lanes[0] != 1 || unpacked_lanes[1] != 40 || unpacked_high_lanes[0] != 3 || unpacked_high_lanes[1] != 40 || reinterpreted.to_array() != ints) + return false; + for (std::size_t group = 0; group < words.size(); group += 8) + { + for (std::size_t lane = 0; lane < 4; ++lane) + { + if (low_lanes[group + lane] != words[group + 3 - lane] || low_lanes[group + 4 + lane] != words[group + 4 + lane] || + high_lanes[group + lane] != words[group + lane] || high_lanes[group + 4 + lane] != words[group + 7 - lane]) + return false; + } + } + for (std::size_t lane = 0; lane < blend_lanes.size(); ++lane) + { + const std::int16_t expected = (0xA5u & (1u << (lane % 8))) != 0 ? 70 : words[lane]; + if (blend_lanes[lane] != expected) + return false; + } + for (std::size_t lane = 0; lane < converted.lane_count; ++lane) + { + if (converted.to_array()[lane] != static_cast(ints[lane]) || rounded.to_array()[lane] != 2) + return false; + } + constexpr std::array widen_source{-8, -7, -6, -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, 6, 7, 8}; + const auto widened_lanes = widened.to_array(); + for (std::size_t lane = 0; lane < widened_lanes.size(); ++lane) + { + if (widened_lanes[lane] != widen_source[lane]) + return false; + } + return true; +#endif +} + +#define SIMDLIB_ASSERT_REGISTER_CONSTEXPR(element_type) \ + static_assert(register_constexpr_contract()); \ + static_assert(register_mask_constexpr_contract()); \ + static_assert(register_bitwise_constexpr_contract()) + +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(std::int8_t); +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(std::uint8_t); +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(std::int16_t); +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(std::uint16_t); +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(std::int32_t); +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(std::uint32_t); +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(std::int64_t); +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(std::uint64_t); +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(float); +SIMDLIB_ASSERT_REGISTER_CONSTEXPR(double); + +#undef SIMDLIB_ASSERT_REGISTER_CONSTEXPR + +#define SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR(element_type) static_assert(register_lane_shift_constexpr_contract()) + +SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR(std::int8_t); +SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR(std::uint8_t); +SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR(std::int16_t); +SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR(std::uint16_t); +SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR(std::int32_t); +SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR(std::uint32_t); +SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR(std::int64_t); +SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR(std::uint64_t); + +#undef SIMDLIB_ASSERT_REGISTER_SHIFT_CONSTEXPR + +#define SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(element_type) \ + static_assert(register_logical_shuffle_contract()) + +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(std::int8_t); +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(std::uint8_t); +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(std::int16_t); +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(std::uint16_t); +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(std::int32_t); +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(std::uint32_t); +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(std::int64_t); +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(std::uint64_t); +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(float); +SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR(double); + +#undef SIMDLIB_ASSERT_REGISTER_LOGICAL_SHUFFLE_CONSTEXPR + +#define SIMDLIB_ASSERT_REGISTER_BYTE_SHUFFLE_CONSTEXPR(element_type) static_assert(register_byte_shuffle_contract()) + +SIMDLIB_ASSERT_REGISTER_BYTE_SHUFFLE_CONSTEXPR(std::int32_t); +SIMDLIB_ASSERT_REGISTER_BYTE_SHUFFLE_CONSTEXPR(double); + +#undef SIMDLIB_ASSERT_REGISTER_BYTE_SHUFFLE_CONSTEXPR + +static_assert(register_complete_shift_constexpr_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_immediate_byte_shift_count_contract()); +static_assert(register_rearrangement_conversion_constexpr_contract()); +static_assert(register_position_constexpr_contract()); +static_assert(register_position_constexpr_contract()); +static_assert(register_position_constexpr_contract()); +static_assert(register_position_constexpr_contract()); +static_assert(register_position_constexpr_contract()); +static_assert(register_position_constexpr_contract()); +static_assert(register_position_constexpr_contract()); +static_assert(register_position_constexpr_contract()); +#define SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(source_type) \ + static_assert(register_conversion_constexpr_targets(supported_register_element_types{})); \ + static_assert(register_widen_constexpr_targets(supported_register_element_types{})) + +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(std::int8_t); +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(std::uint8_t); +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(std::int16_t); +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(std::uint16_t); +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(std::int32_t); +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(std::uint32_t); +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(std::int64_t); +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(std::uint64_t); +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(float); +SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR(double); + +#undef SIMDLIB_ASSERT_REGISTER_CONVERSION_CONSTEXPR + +} // namespace diff --git a/tests/constexpr/UInt128Constexpr.tests.cpp b/tests/constexpr/UInt128Constexpr.tests.cpp index 5de5e33..5d8f3a3 100644 --- a/tests/constexpr/UInt128Constexpr.tests.cpp +++ b/tests/constexpr/UInt128Constexpr.tests.cpp @@ -23,28 +23,22 @@ static_assert(popcount(std::numeric_limits::max()) == 128); return false; if ((lhs & rhs) != uint128_t{0x1010'2200'3210'0000ULL, 0x0101'4466'0123'8888ULL} || (lhs | rhs) != uint128_t{0xFFDD'BABA'7777'7654ULL, 0x5577'6767'FFFF'CDEFULL} || - (lhs ^ rhs) != uint128_t{0xEFCD'98BA'4567'7654ULL, 0x5476'2301'FEDC'4567ULL} || - ~lhs != uint128_t{0x0123'4567'89AB'CDEFULL, 0xFEDC'BA98'7654'3210ULL}) + (lhs ^ rhs) != uint128_t{0xEFCD'98BA'4567'7654ULL, 0x5476'2301'FEDC'4567ULL} || ~lhs != uint128_t{0x0123'4567'89AB'CDEFULL, 0xFEDC'BA98'7654'3210ULL}) return false; - if ((uint128_t{1} << 0) != uint128_t{1} || (uint128_t{1} << 63) != uint128_t{std::uint64_t{1} << 63} || - (uint128_t{1} << 64) != uint128_t{0, 1} || (uint128_t{1} << 127) != uint128_t{0, std::uint64_t{1} << 63} || - (uint128_t{1} << 128) != uint128_t{} || (uint128_t{1} << 129) != uint128_t{}) + if ((uint128_t{1} << 0) != uint128_t{1} || (uint128_t{1} << 63) != uint128_t{std::uint64_t{1} << 63} || (uint128_t{1} << 64) != uint128_t{0, 1} || + (uint128_t{1} << 127) != uint128_t{0, std::uint64_t{1} << 63} || (uint128_t{1} << 128) != uint128_t{} || (uint128_t{1} << 129) != uint128_t{}) return false; constexpr uint128_t highBit{0, std::uint64_t{1} << 63}; - if ((highBit >> 0) != highBit || (highBit >> 63) != uint128_t{0, 1} || - (highBit >> 64) != uint128_t{std::uint64_t{1} << 63} || (highBit >> 127) != uint128_t{1} || - (highBit >> 128) != uint128_t{} || (highBit >> 129) != uint128_t{}) + if ((highBit >> 0) != highBit || (highBit >> 63) != uint128_t{0, 1} || (highBit >> 64) != uint128_t{std::uint64_t{1} << 63} || + (highBit >> 127) != uint128_t{1} || (highBit >> 128) != uint128_t{} || (highBit >> 129) != uint128_t{}) return false; if (uint128_t::create_mask(0) != uint128_t{} || uint128_t::create_mask(64) != uint128_t{~std::uint64_t{0}} || - uint128_t::create_mask(65) != uint128_t{~std::uint64_t{0}, 1} || - uint128_t::create_mask(128) != std::numeric_limits::max()) + uint128_t::create_mask(65) != uint128_t{~std::uint64_t{0}, 1} || uint128_t::create_mask(128) != std::numeric_limits::max()) return false; - return popcount(lhs) == std::popcount(lhs.low()) + std::popcount(lhs.high()) && - countr_zero(uint128_t{}) == 128 && countl_zero(uint128_t{}) == 128 && - bit_width(highBit) == 128 && bit_floor(highBit) == highBit && bit_ceil(highBit) == highBit && - has_single_bit(highBit) && Bmi::bextr(lhs, 17, 61) == ((lhs >> 61) & uint128_t::create_mask(17)); + return popcount(lhs) == std::popcount(lhs.low()) + std::popcount(lhs.high()) && countr_zero(uint128_t{}) == 128 && countl_zero(uint128_t{}) == 128 && + bit_width(highBit) == 128 && bit_floor(highBit) == highBit && bit_ceil(highBit) == highBit && has_single_bit(highBit) && + Bmi::bextr(lhs, 17, 61) == ((lhs >> 61) & uint128_t::create_mask(17)); } static_assert(uint128_contract()); } // namespace SimdLib - diff --git a/tests/consumer/CMakeLists.txt b/tests/consumer/CMakeLists.txt index 2a3e849..7f352fd 100644 --- a/tests/consumer/CMakeLists.txt +++ b/tests/consumer/CMakeLists.txt @@ -6,29 +6,100 @@ if(NOT DEFINED SIMDLIB_SOURCE_DIR) get_filename_component(SIMDLIB_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) endif() -set(SIMDLIB_BUILD_SMOKE_TESTS OFF CACHE BOOL "" FORCE) -set(SIMDLIB_BUILD_TESTS OFF CACHE BOOL "" FORCE) -set(SIMDLIB_BUILD_BENCHMARKS OFF CACHE BOOL "" FORCE) -set(SIMDLIB_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) -set(SIMDLIB_BUILD_CONFIGURATION_TESTS OFF CACHE BOOL "" FORCE) -set(SIMDLIB_BUILD_HEADER_TESTS OFF CACHE BOOL "" FORCE) - +get_cmake_property(consumer_cache_before CACHE_VARIABLES) add_subdirectory("${SIMDLIB_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/simdlib" EXCLUDE_FROM_ALL) +get_cmake_property(consumer_cache_after CACHE_VARIABLES) + +if(DEFINED CACHE{BUILD_TESTING}) + message(FATAL_ERROR + "add_subdirectory introduced CTest's BUILD_TESTING cache option") +endif() + +foreach(cache_variable IN LISTS consumer_cache_after) + if(cache_variable MATCHES "^SIMDLIB_" AND + NOT cache_variable IN_LIST consumer_cache_before) + message(FATAL_ERROR + "add_subdirectory introduced development cache option ${cache_variable}") + endif() +endforeach() + +get_property(simdlib_nested_targets + DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/simdlib" PROPERTY BUILDSYSTEM_TARGETS) +list(SORT simdlib_nested_targets) +if(NOT simdlib_nested_targets STREQUAL "SimdLib;SimdLibRegister") + message(FATAL_ERROR + "add_subdirectory introduced unexpected targets: ${simdlib_nested_targets}") +endif() +get_property(simdlib_nested_tests + DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/simdlib" PROPERTY TESTS) +if(simdlib_nested_tests) + message(FATAL_ERROR + "add_subdirectory registered development tests: ${simdlib_nested_tests}") +endif() get_target_property(simdlib_target_type SimdLib TYPE) if(NOT simdlib_target_type STREQUAL "INTERFACE_LIBRARY") message(FATAL_ERROR "SimdLib must remain header-only; target type is ${simdlib_target_type}") endif() -add_executable(SimdLibConsumerSmoke main.cpp) -target_link_libraries(SimdLibConsumerSmoke PRIVATE SimdLib::SimdLib) +get_target_property(simdlib_core_features SimdLib INTERFACE_COMPILE_FEATURES) +if(NOT "cxx_std_20" IN_LIST simdlib_core_features OR "cxx_std_23" IN_LIST simdlib_core_features) + message(FATAL_ERROR "SimdLib::SimdLib must require C++20 without inheriting the Register language requirement") +endif() + +get_target_property(simdlib_register_features SimdLibRegister INTERFACE_COMPILE_FEATURES) +get_target_property(simdlib_register_definitions SimdLibRegister INTERFACE_COMPILE_DEFINITIONS) +get_target_property(simdlib_register_links SimdLibRegister INTERFACE_LINK_LIBRARIES) +if(NOT "cxx_std_23" IN_LIST simdlib_register_features) + message(FATAL_ERROR "SimdLib::Register must request C++23") +endif() +if(NOT "SIMDLIB_REQUIRE_REGISTER_INTERFACE=1" IN_LIST simdlib_register_definitions) + message(FATAL_ERROR "SimdLib::Register must publish the Register requirement signal") +endif() +if(NOT "SimdLib::SimdLib" IN_LIST simdlib_register_links) + message(FATAL_ERROR "SimdLib::Register must link the core SimdLib target") +endif() + +add_executable(CoreConsumerSmoke main.cpp) +target_link_libraries(CoreConsumerSmoke PRIVATE SimdLib::SimdLib) +set_target_properties(CoreConsumerSmoke PROPERTIES + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF) if(MSVC) - target_compile_definitions(SimdLibConsumerSmoke PRIVATE + target_compile_definitions(CoreConsumerSmoke PRIVATE SIMDLIB_HAS_SSE3=1 SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1) - target_compile_options(SimdLibConsumerSmoke PRIVATE /arch:AVX2) + target_compile_options(CoreConsumerSmoke PRIVATE /arch:AVX2) else() - target_compile_options(SimdLibConsumerSmoke PRIVATE -msse4.2) + target_compile_options(CoreConsumerSmoke PRIVATE -msse4.2) endif() enable_testing() -add_test(NAME SimdLib.ConsumerSmoke COMMAND SimdLibConsumerSmoke) +add_test(NAME CoreConsumerSmoke COMMAND CoreConsumerSmoke) + +get_target_property(simdlib_register_compiler_supported SimdLibRegister + SIMDLIB_REGISTER_COMPILER_SUPPORTED) +option(SIMDLIB_BUILD_REGISTER_CONSUMER + "Build the opt-in C++23 Register consumer smoke test" + ${simdlib_register_compiler_supported}) +if(SIMDLIB_BUILD_REGISTER_CONSUMER) + add_executable(RegisterConsumerSmoke + register.cpp + register_api.cpp + register_api.h) + target_link_libraries(RegisterConsumerSmoke PRIVATE SimdLib::Register) + target_compile_definitions(RegisterConsumerSmoke PRIVATE + SIMDLIB_HAS_SSE=1 SIMDLIB_HAS_SSE2=1 SIMDLIB_HAS_SSE3=1 + SIMDLIB_HAS_SSSE3=1 SIMDLIB_HAS_SSE41=1 SIMDLIB_HAS_SSE42=1 + SIMDLIB_HAS_AVX=0 SIMDLIB_HAS_AVX2=0 SIMDLIB_HAS_FMA=0) + if(MSVC) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(RegisterConsumerSmoke PRIVATE + /clang:-msse4.2 /clang:-mno-avx /clang:-mno-avx2 /clang:-mno-fma) + endif() + else() + target_compile_options(RegisterConsumerSmoke PRIVATE + -msse4.2 -mno-avx -mno-avx2 -mno-fma) + endif() + add_test(NAME RegisterConsumerSmoke COMMAND RegisterConsumerSmoke) +endif() diff --git a/tests/consumer/register.cpp b/tests/consumer/register.cpp new file mode 100644 index 0000000..a0e573a --- /dev/null +++ b/tests/consumer/register.cpp @@ -0,0 +1,30 @@ +#include "register_api.h" + +#if !SIMDLIB_REQUIRE_REGISTER_INTERFACE +#error "The Register target must publish its requirement signal to consumers" +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +static_assert(_MSVC_LANG > 202002L); +#else +static_assert(__cplusplus > 202002L); +#endif + +/** + * @brief Verifies cross-translation-unit use of flagged Register and native SIMD boundaries. + * @return Zero when both downstream declaration contracts are satisfied. + */ +int main() +{ + using namespace SimdLibConsumer; + const Register expected = Register::broadcast(4); + const Register actual = increment(Register::broadcast(3)); + const RegisterMask equal = actual.compare_equal(expected); + if (!equal.all() || equal.select(actual, Register::zero()) != expected) + { + return 1; + } + + const auto native = increment_native(_mm_set1_epi32(3)); + return _mm_cvtsi128_si32(native) == 4 ? 0 : 2; +} diff --git a/tests/consumer/register_api.cpp b/tests/consumer/register_api.cpp new file mode 100644 index 0000000..341f8f3 --- /dev/null +++ b/tests/consumer/register_api.cpp @@ -0,0 +1,16 @@ +#include "register_api.h" + +namespace SimdLibConsumer +{ +/** Defines the downstream Register boundary in a separate translation unit. */ +Register SIMD_FLAGS(InOut, RegisterOnly) increment(Register value) noexcept +{ + return value + Register::broadcast(1); +} + +/** Defines the downstream native-SIMD boundary in a separate translation unit. */ +native_type SIMD_FLAGS(InOut, RegisterOnly) increment_native(native_type value) noexcept +{ + return _mm_add_epi32(value, _mm_set1_epi32(1)); +} +} // namespace SimdLibConsumer diff --git a/tests/consumer/register_api.h b/tests/consumer/register_api.h new file mode 100644 index 0000000..65eb593 --- /dev/null +++ b/tests/consumer/register_api.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +#include +#include + +namespace SimdLibConsumer +{ +using Register = SimdLib::Register; +using RegisterMask = Register::mask_type; +using native_type = __m128i; + +/** + * @brief Increments every lane of a downstream Register value. + * @param value Input register. + * @return Input register increased by one in every lane. + */ +[[nodiscard]] Register SIMD_FLAGS(InOut, RegisterOnly) increment(Register value) noexcept; + +/** + * @brief Increments every lane of a downstream native SIMD value. + * @param value Input native register. + * @return Input register increased by one in every lane. + */ +[[nodiscard]] native_type SIMD_FLAGS(InOut, RegisterOnly) increment_native(native_type value) noexcept; +} // namespace SimdLibConsumer diff --git a/tests/format_odr/main.cpp b/tests/format_odr/main.cpp index 2402538..d6e1a63 100644 --- a/tests/format_odr/main.cpp +++ b/tests/format_odr/main.cpp @@ -8,8 +8,5 @@ std::string FormatFromSecondTranslationUnit(); int main() { const SimdLib::SimdVector value{1, 2, 3}; - return std::format("{}", value) == "{1, 2, 3}" && - FormatFromSecondTranslationUnit() == "18446744073709551616" - ? 0 - : 1; + return std::format("{}", value) == "{1, 2, 3}" && FormatFromSecondTranslationUnit() == "18446744073709551616" ? 0 : 1; } diff --git a/tests/headers/AliasesHeaderProbe.cpp b/tests/headers/AliasesHeaderProbe.cpp new file mode 100644 index 0000000..7eb4b04 --- /dev/null +++ b/tests/headers/AliasesHeaderProbe.cpp @@ -0,0 +1,17 @@ +#include + +#include +#include + +static_assert(std::same_as>); +static_assert(std::same_as>); + +static_assert(std::same_as>); +static_assert(std::same_as>); +static_assert(std::same_as>); +static_assert(std::same_as>); + +static_assert(std::same_as>); +static_assert(std::same_as>); +static_assert(std::same_as>); +static_assert(std::same_as>); diff --git a/tests/headers/IApiHeaderProbe.cpp b/tests/headers/IApiHeaderProbe.cpp new file mode 100644 index 0000000..066d8dc --- /dev/null +++ b/tests/headers/IApiHeaderProbe.cpp @@ -0,0 +1,30 @@ +#include + +namespace +{ + +/** @brief Minimal metadata-only type used to verify the standalone API interface header. */ +struct ApiShape +{ + using element_type = int; + using vector_t = int; + constexpr static inline std::size_t register_width = 128; +}; + +static_assert(SimdLib::IApi::Type); +static_assert(SimdLib::IApi::WidenTarget); +static_assert(!SimdLib::IApi::Add); +static_assert(!SimdLib::IApi::LowerHalf); +static_assert(!SimdLib::IApi::UnpackLow); +static_assert(!SimdLib::IApi::UnpackHigh); +static_assert(!SimdLib::IApi::Shuffle); +static_assert(!SimdLib::IApi::ShuffleLow); +static_assert(!SimdLib::IApi::ShuffleHigh); +static_assert(!SimdLib::IApi::Blend); +static_assert(!SimdLib::IApi::BitCast); +static_assert(!SimdLib::IApi::Convert); +static_assert(!SimdLib::IApi::Widen); +static_assert(!SimdLib::ApiAvailable<128, bool>); +static_assert(!SimdLib::NativeApiAvailable); + +} // namespace diff --git a/tests/headers/IImplHeaderProbe.cpp b/tests/headers/IImplHeaderProbe.cpp new file mode 100644 index 0000000..175db84 --- /dev/null +++ b/tests/headers/IImplHeaderProbe.cpp @@ -0,0 +1,19 @@ +#include + +namespace +{ + +/** @brief Minimal metadata-only type used to verify the standalone implementation interface header. */ +struct ImplementationShape +{ + using vector_t = int; +}; + +static_assert(SimdLib::IImpl::Mapping); +static_assert(!SimdLib::IImpl::Add); +static_assert(!SimdLib::IImpl::SetZero); +static_assert(!SimdLib::IImpl::IndexedShuffleLow); +static_assert(!SimdLib::IImpl::IndexedShuffleHigh); +static_assert(!SimdLib::IImpl::IndexedBlend); + +} // namespace diff --git a/tests/headers/IRegisterHeaderProbe.cpp b/tests/headers/IRegisterHeaderProbe.cpp new file mode 100644 index 0000000..6dd5d38 --- /dev/null +++ b/tests/headers/IRegisterHeaderProbe.cpp @@ -0,0 +1,48 @@ +#include + +namespace +{ + +/** @brief Minimal API metadata used by the standalone Register interface probe. */ +struct ApiShape +{ + using mask_t = unsigned int; +}; + +/** @brief Minimal predicate type used by the standalone Register interface probe. */ +struct MaskShape +{ +}; + +/** @brief Minimal aggregate metadata shape used to verify the standalone Register interface header. */ +struct RegisterShape +{ + using element_type = int; + using api_type = ApiShape; + using native_type = int; + using mask_type = MaskShape; + + constexpr static inline std::size_t register_width = 128; + constexpr static inline std::size_t byte_count = 16; + constexpr static inline std::size_t lane_count = 4; + + native_type native{}; +}; + +static_assert(SimdLib::IRegister::Type); +static_assert(SimdLib::IRegister::Shape); +static_assert(!SimdLib::IRegister::Zero); +static_assert(!SimdLib::IRegister::Add); +static_assert(!SimdLib::IRegister::LowerHalf); +static_assert(!SimdLib::IRegister::UnpackLow); +static_assert(!SimdLib::IRegister::UnpackHigh); +static_assert(!SimdLib::IRegister::Shuffle); +static_assert(!SimdLib::IRegister::ShuffleLow); +static_assert(!SimdLib::IRegister::ShuffleBytes); +static_assert(!SimdLib::IRegister::ShuffleHigh); +static_assert(!SimdLib::IRegister::Blend); +static_assert(!SimdLib::IRegister::BitCast); +static_assert(!SimdLib::IRegister::Convert); +static_assert(!SimdLib::IRegister::WidenLow); + +} // namespace diff --git a/tests/headers/IRegisterMaskHeaderProbe.cpp b/tests/headers/IRegisterMaskHeaderProbe.cpp new file mode 100644 index 0000000..44cb3ff --- /dev/null +++ b/tests/headers/IRegisterMaskHeaderProbe.cpp @@ -0,0 +1,101 @@ +#include + +#include +#include + +namespace +{ + +/** @brief Minimal API metadata used by the standalone RegisterMask interface probe. */ +struct ApiShape +{ +}; + +/** @brief Minimal Register result used by the standalone RegisterMask selection probe. */ +struct RegisterShape +{ + int native{}; +}; + +/** @brief Minimal aggregate predicate implementation used to verify the standalone RegisterMask interface header. */ +struct MaskShape +{ + using element_type = int; + using api_type = ApiShape; + using native_type = int; + using register_type = RegisterShape; + using bits_type = std::uint32_t; + + constexpr static inline std::size_t register_width = 128; + constexpr static inline std::size_t byte_count = 16; + constexpr static inline std::size_t lane_count = 4; + + native_type native{}; + + /** @brief Reports whether any predicate lane is active. */ + [[nodiscard]] constexpr bool any() const noexcept + { + return native != 0; + } + + /** @brief Reports whether every predicate lane is active. */ + [[nodiscard]] constexpr bool all() const noexcept + { + return native == -1; + } + + /** @brief Reports whether no predicate lane is active. */ + [[nodiscard]] constexpr bool none() const noexcept + { + return native == 0; + } + + /** @brief Returns one compact bit per logical predicate lane. */ + [[nodiscard]] constexpr bits_type bits() const noexcept + { + return static_cast(native); + } + + /** @brief Selects one Register value according to the predicate. */ + [[nodiscard]] constexpr register_type select(register_type when_true, register_type when_false) const noexcept + { + return native != 0 ? when_true : when_false; + } + + /** @brief Computes predicate intersection. */ + [[maybe_unused, nodiscard]] friend constexpr MaskShape operator&(MaskShape lhs, MaskShape rhs) noexcept + { + return {lhs.native & rhs.native}; + } + + /** @brief Computes predicate union. */ + [[maybe_unused, nodiscard]] friend constexpr MaskShape operator|(MaskShape lhs, MaskShape rhs) noexcept + { + return {lhs.native | rhs.native}; + } + + /** @brief Computes predicate exclusive union. */ + [[maybe_unused, nodiscard]] friend constexpr MaskShape operator^(MaskShape lhs, MaskShape rhs) noexcept + { + return {lhs.native ^ rhs.native}; + } + + /** @brief Computes predicate complement. */ + [[maybe_unused, nodiscard]] friend constexpr MaskShape operator~(MaskShape value) noexcept + { + return {~value.native}; + } +}; + +static_assert(SimdLib::IRegisterMask::Type); +static_assert(SimdLib::IRegisterMask::Any); +static_assert(SimdLib::IRegisterMask::All); +static_assert(SimdLib::IRegisterMask::None); +static_assert(SimdLib::IRegisterMask::Bits); +static_assert(SimdLib::IRegisterMask::Select); +static_assert(SimdLib::IRegisterMask::BitwiseAnd); +static_assert(SimdLib::IRegisterMask::BitwiseOr); +static_assert(SimdLib::IRegisterMask::BitwiseXor); +static_assert(SimdLib::IRegisterMask::BitwiseNot); + +} // namespace diff --git a/tests/headers/InstalledConfigHeaderProbe.cpp b/tests/headers/InstalledConfigHeaderProbe.cpp new file mode 100644 index 0000000..f15371d --- /dev/null +++ b/tests/headers/InstalledConfigHeaderProbe.cpp @@ -0,0 +1,13 @@ +#include + +/** + * @brief Exercises the public method-flags parser from an isolated header image. + * @param value Scalar value returned unchanged. + * @return The supplied scalar value. + */ +int SIMD_FLAGS(Neither, ForceInline) installed_config_identity(const int value) noexcept +{ + return value; +} + +static_assert(SimdLib::Config::version_major == SimdLib::version_major); diff --git a/tests/headers/InstalledDisabledHeaderProbe.cpp b/tests/headers/InstalledDisabledHeaderProbe.cpp new file mode 100644 index 0000000..07fa1e7 --- /dev/null +++ b/tests/headers/InstalledDisabledHeaderProbe.cpp @@ -0,0 +1,14 @@ +#include + +/** + * @brief Exercises method flags when every optional instruction family is disabled. + * @param value Scalar value returned unchanged. + * @return The supplied scalar value. + */ +int SIMD_FLAGS(Neither, RegisterOnly) installed_disabled_identity(const int value) noexcept +{ + return value; +} + +static_assert(!SimdLib::is_api_available_v<128, unsigned>); +static_assert(!SimdLib::is_api_available_v<256, unsigned>); diff --git a/tests/headers/InstalledHeaderOdrConsumer.cpp b/tests/headers/InstalledHeaderOdrConsumer.cpp new file mode 100644 index 0000000..741199a --- /dev/null +++ b/tests/headers/InstalledHeaderOdrConsumer.cpp @@ -0,0 +1,10 @@ +#include "InstalledHeaderOdrFixture.h" + +/** + * @brief Verifies declaration and definition agreement across translation units. + * @return Zero when the copied-header declaration linked and executed correctly. + */ +int main() +{ + return installed_header_odr_value(41) == 42 ? 0 : 1; +} diff --git a/tests/headers/InstalledHeaderOdrDefinition.cpp b/tests/headers/InstalledHeaderOdrDefinition.cpp new file mode 100644 index 0000000..a7411ad --- /dev/null +++ b/tests/headers/InstalledHeaderOdrDefinition.cpp @@ -0,0 +1,11 @@ +#include "InstalledHeaderOdrFixture.h" + +/** + * @brief Defines the copied-header cross-translation-unit fixture. + * @param value Scalar value transformed by the fixture. + * @return The supplied value incremented by one. + */ +int SIMD_FLAGS(Neither) installed_header_odr_value(const int value) noexcept +{ + return value + 1; +} diff --git a/tests/headers/InstalledHeaderOdrFixture.h b/tests/headers/InstalledHeaderOdrFixture.h new file mode 100644 index 0000000..e40cd43 --- /dev/null +++ b/tests/headers/InstalledHeaderOdrFixture.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +/** + * @brief Declares a cross-translation-unit function through the copied headers. + * @param value Scalar value transformed by the definition translation unit. + * @return The transformed scalar value. + */ +int SIMD_FLAGS(Neither) installed_header_odr_value(int value) noexcept; diff --git a/tests/headers/InstalledRegisterHeaderProbe.cpp b/tests/headers/InstalledRegisterHeaderProbe.cpp new file mode 100644 index 0000000..e020cc9 --- /dev/null +++ b/tests/headers/InstalledRegisterHeaderProbe.cpp @@ -0,0 +1,13 @@ +#include + +/** + * @brief Exercises a Register boundary using only the isolated public headers. + * @param value Register value returned unchanged. + * @return The supplied register value. + */ +SimdLib::Register SIMD_FLAGS(InOut, RegisterOnly, ForceInline) installed_register_identity(const SimdLib::Register value) noexcept +{ + return value; +} + +static_assert(sizeof(SimdLib::Register) == 16); diff --git a/tests/headers/InstalledUmbrellaHeaderProbe.cpp b/tests/headers/InstalledUmbrellaHeaderProbe.cpp new file mode 100644 index 0000000..7678605 --- /dev/null +++ b/tests/headers/InstalledUmbrellaHeaderProbe.cpp @@ -0,0 +1,17 @@ +#include + +/** + * @brief Exercises the umbrella header and public declaration macro together. + * @param value Scalar value returned unchanged. + * @return The supplied scalar value. + */ +int SIMD_FLAGS(Neither) installed_umbrella_identity(const int value) noexcept +{ + return value; +} + +#if SIMDLIB_HAS_SSE42 +static_assert(SimdLib::Api<128, unsigned>::byte_count == 16); +#else +static_assert(!SimdLib::is_api_available_v<128, unsigned>); +#endif diff --git a/tests/headers/PublicSurfaceHeaderProbe.cpp b/tests/headers/PublicSurfaceHeaderProbe.cpp index 2f26a58..f63cccd 100644 --- a/tests/headers/PublicSurfaceHeaderProbe.cpp +++ b/tests/headers/PublicSurfaceHeaderProbe.cpp @@ -6,7 +6,6 @@ #include #include -static_assert(std::same_as, SimdLib::uint32x4>); static_assert(std::same_as); static_assert(std::same_as); diff --git a/tests/headers/RegisterHeaderProbe.cpp b/tests/headers/RegisterHeaderProbe.cpp new file mode 100644 index 0000000..c720307 --- /dev/null +++ b/tests/headers/RegisterHeaderProbe.cpp @@ -0,0 +1,4 @@ +#include + +static_assert(SIMDLIB_REGISTER_INTERFACE_AVAILABLE == 1); +static_assert(SIMDLIB_REQUIRE_REGISTER_INTERFACE == 1); diff --git a/tests/headers/RegisterMaskHeaderProbe.cpp b/tests/headers/RegisterMaskHeaderProbe.cpp new file mode 100644 index 0000000..7c20f38 --- /dev/null +++ b/tests/headers/RegisterMaskHeaderProbe.cpp @@ -0,0 +1,4 @@ +#include + +static_assert(SIMDLIB_REGISTER_INTERFACE_AVAILABLE == 1); +static_assert(SIMDLIB_REQUIRE_REGISTER_INTERFACE == 1); diff --git a/tests/headers/SimdLibRegisterHeaderProbe.cpp b/tests/headers/SimdLibRegisterHeaderProbe.cpp new file mode 100644 index 0000000..d7d3b95 --- /dev/null +++ b/tests/headers/SimdLibRegisterHeaderProbe.cpp @@ -0,0 +1,16 @@ +#include + +#include +#include + +static_assert(SIMDLIB_REGISTER_INTERFACE_AVAILABLE == 1); +static_assert(SIMDLIB_REQUIRE_REGISTER_INTERFACE == 1); + +using UmbrellaRegister = SimdLib::Register; +using UmbrellaNativeRegister = SimdLib::NativeRegister; +using UmbrellaRegisterMask = typename UmbrellaRegister::mask_type; + +static_assert(SimdLib::IRegister::Type); +static_assert(std::same_as); +static_assert(SimdLib::IRegister::Type); +static_assert(SimdLib::IRegisterMask::Type); diff --git a/tests/headers/UInt128HeaderProbe.cpp b/tests/headers/UInt128HeaderProbe.cpp index 0844455..c3c36c8 100644 --- a/tests/headers/UInt128HeaderProbe.cpp +++ b/tests/headers/UInt128HeaderProbe.cpp @@ -1,3 +1,5 @@ #include static_assert(SimdLib::version_major == 0); +static_assert(std::numeric_limits::has_denorm == std::denorm_absent); +static_assert(!std::numeric_limits::has_denorm_loss); diff --git a/tests/method_flags/InvalidDuplicate.cpp b/tests/method_flags/InvalidDuplicate.cpp new file mode 100644 index 0000000..1fe2579 --- /dev/null +++ b/tests/method_flags/InvalidDuplicate.cpp @@ -0,0 +1,5 @@ +#define SIMDLIB_PRECONDITION(condition, message) +#include + +/// Declares a function with a duplicate method-flags modifier. +int SIMD_FLAGS(InOut, RegisterOnly, RegisterOnly) invalid_duplicate(); diff --git a/tests/method_flags/InvalidEmpty.cpp b/tests/method_flags/InvalidEmpty.cpp new file mode 100644 index 0000000..f852813 --- /dev/null +++ b/tests/method_flags/InvalidEmpty.cpp @@ -0,0 +1,5 @@ +#define SIMDLIB_PRECONDITION(condition, message) +#include + +/// Declares a function with an invalid empty method-flags invocation. +int SIMD_FLAGS() invalid_empty(); diff --git a/tests/method_flags/InvalidMissingBoundary.cpp b/tests/method_flags/InvalidMissingBoundary.cpp new file mode 100644 index 0000000..0767064 --- /dev/null +++ b/tests/method_flags/InvalidMissingBoundary.cpp @@ -0,0 +1,5 @@ +#define SIMDLIB_PRECONDITION(condition, message) +#include + +/// Declares a function whose invocation omits the required boundary mode. +int SIMD_FLAGS(RegisterOnly) invalid_missing_boundary(); diff --git a/tests/method_flags/InvalidModifierOrder.cpp b/tests/method_flags/InvalidModifierOrder.cpp new file mode 100644 index 0000000..f660656 --- /dev/null +++ b/tests/method_flags/InvalidModifierOrder.cpp @@ -0,0 +1,5 @@ +#define SIMDLIB_PRECONDITION(condition, message) +#include + +/// Declares a function whose modifiers use a noncanonical order. +int SIMD_FLAGS(InOut, Flatten, ForceInline) invalid_modifier_order(); diff --git a/tests/method_flags/InvalidObjectMacroCollision.cpp b/tests/method_flags/InvalidObjectMacroCollision.cpp new file mode 100644 index 0000000..4784b98 --- /dev/null +++ b/tests/method_flags/InvalidObjectMacroCollision.cpp @@ -0,0 +1,7 @@ +#define SIMDLIB_PRECONDITION(condition, message) +#include + +#define In downstream_object_macro + +/// Declares a function whose boundary mode collides with an object-like macro. +int SIMD_FLAGS(In) invalid_object_macro_collision(); diff --git a/tests/method_flags/InvalidTooMany.cpp b/tests/method_flags/InvalidTooMany.cpp new file mode 100644 index 0000000..ad19eed --- /dev/null +++ b/tests/method_flags/InvalidTooMany.cpp @@ -0,0 +1,5 @@ +#define SIMDLIB_PRECONDITION(condition, message) +#include + +/// Declares a function with too many method-flags arguments. +int SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten, Extra) invalid_too_many(); diff --git a/tests/method_flags/InvalidUnknown.cpp b/tests/method_flags/InvalidUnknown.cpp new file mode 100644 index 0000000..ce505a4 --- /dev/null +++ b/tests/method_flags/InvalidUnknown.cpp @@ -0,0 +1,5 @@ +#define SIMDLIB_PRECONDITION(condition, message) +#include + +/// Declares a function with an unknown method-flags modifier. +int SIMD_FLAGS(InOut, Unknown) invalid_unknown(); diff --git a/tests/method_flags/MethodFlagsContractPass.cpp b/tests/method_flags/MethodFlagsContractPass.cpp new file mode 100644 index 0000000..2360298 --- /dev/null +++ b/tests/method_flags/MethodFlagsContractPass.cpp @@ -0,0 +1,102 @@ +#include + +#include + +namespace SimdLibMethodFlagsContract +{ +/// Declares the Neither boundary with no modifiers. +[[nodiscard]] int SIMD_FLAGS(Neither) contract_neither_plain(int value) noexcept; + +/// Declares the Neither boundary with RegisterOnly. +[[nodiscard]] int SIMD_FLAGS(Neither, RegisterOnly) contract_neither_registeronly(int value) noexcept; + +/// Declares the Neither boundary with ForceInline. +[[nodiscard]] int SIMD_FLAGS(Neither, ForceInline) contract_neither_forceinline(int value) noexcept; + +/// Declares the Neither boundary with Flatten. +[[nodiscard]] int SIMD_FLAGS(Neither, Flatten) contract_neither_flatten(int value) noexcept; + +/// Declares the Neither boundary with RegisterOnly, ForceInline. +[[nodiscard]] int SIMD_FLAGS(Neither, RegisterOnly, ForceInline) contract_neither_registeronly_forceinline(int value) noexcept; + +/// Declares the Neither boundary with RegisterOnly, Flatten. +[[nodiscard]] int SIMD_FLAGS(Neither, RegisterOnly, Flatten) contract_neither_registeronly_flatten(int value) noexcept; + +/// Declares the Neither boundary with ForceInline, Flatten. +[[nodiscard]] int SIMD_FLAGS(Neither, ForceInline, Flatten) contract_neither_forceinline_flatten(int value) noexcept; + +/// Declares the Neither boundary with RegisterOnly, ForceInline, Flatten. +[[nodiscard]] int SIMD_FLAGS(Neither, RegisterOnly, ForceInline, Flatten) contract_neither_registeronly_forceinline_flatten(int value) noexcept; + +/// Declares the In boundary with no modifiers. +[[nodiscard]] int SIMD_FLAGS(In) contract_in_plain(__m128 value) noexcept; + +/// Declares the In boundary with RegisterOnly. +[[nodiscard]] int SIMD_FLAGS(In, RegisterOnly) contract_in_registeronly(__m128 value) noexcept; + +/// Declares the In boundary with ForceInline. +[[nodiscard]] int SIMD_FLAGS(In, ForceInline) contract_in_forceinline(__m128 value) noexcept; + +/// Declares the In boundary with Flatten. +[[nodiscard]] int SIMD_FLAGS(In, Flatten) contract_in_flatten(__m128 value) noexcept; + +/// Declares the In boundary with RegisterOnly, ForceInline. +[[nodiscard]] int SIMD_FLAGS(In, RegisterOnly, ForceInline) contract_in_registeronly_forceinline(__m128 value) noexcept; + +/// Declares the In boundary with RegisterOnly, Flatten. +[[nodiscard]] int SIMD_FLAGS(In, RegisterOnly, Flatten) contract_in_registeronly_flatten(__m128 value) noexcept; + +/// Declares the In boundary with ForceInline, Flatten. +[[nodiscard]] int SIMD_FLAGS(In, ForceInline, Flatten) contract_in_forceinline_flatten(__m128 value) noexcept; + +/// Declares the In boundary with RegisterOnly, ForceInline, Flatten. +[[nodiscard]] int SIMD_FLAGS(In, RegisterOnly, ForceInline, Flatten) contract_in_registeronly_forceinline_flatten(__m128 value) noexcept; + +/// Declares the Out boundary with no modifiers. +[[nodiscard]] __m128 SIMD_FLAGS(Out) contract_out_plain(int value) noexcept; + +/// Declares the Out boundary with RegisterOnly. +[[nodiscard]] __m128 SIMD_FLAGS(Out, RegisterOnly) contract_out_registeronly(int value) noexcept; + +/// Declares the Out boundary with ForceInline. +[[nodiscard]] __m128 SIMD_FLAGS(Out, ForceInline) contract_out_forceinline(int value) noexcept; + +/// Declares the Out boundary with Flatten. +[[nodiscard]] __m128 SIMD_FLAGS(Out, Flatten) contract_out_flatten(int value) noexcept; + +/// Declares the Out boundary with RegisterOnly, ForceInline. +[[nodiscard]] __m128 SIMD_FLAGS(Out, RegisterOnly, ForceInline) contract_out_registeronly_forceinline(int value) noexcept; + +/// Declares the Out boundary with RegisterOnly, Flatten. +[[nodiscard]] __m128 SIMD_FLAGS(Out, RegisterOnly, Flatten) contract_out_registeronly_flatten(int value) noexcept; + +/// Declares the Out boundary with ForceInline, Flatten. +[[nodiscard]] __m128 SIMD_FLAGS(Out, ForceInline, Flatten) contract_out_forceinline_flatten(int value) noexcept; + +/// Declares the Out boundary with RegisterOnly, ForceInline, Flatten. +[[nodiscard]] __m128 SIMD_FLAGS(Out, RegisterOnly, ForceInline, Flatten) contract_out_registeronly_forceinline_flatten(int value) noexcept; + +/// Declares the InOut boundary with no modifiers. +[[nodiscard]] __m128 SIMD_FLAGS(InOut) contract_inout_plain(__m128 value) noexcept; + +/// Declares the InOut boundary with RegisterOnly. +[[nodiscard]] __m128 SIMD_FLAGS(InOut, RegisterOnly) contract_inout_registeronly(__m128 value) noexcept; + +/// Declares the InOut boundary with ForceInline. +[[nodiscard]] __m128 SIMD_FLAGS(InOut, ForceInline) contract_inout_forceinline(__m128 value) noexcept; + +/// Declares the InOut boundary with Flatten. +[[nodiscard]] __m128 SIMD_FLAGS(InOut, Flatten) contract_inout_flatten(__m128 value) noexcept; + +/// Declares the InOut boundary with RegisterOnly, ForceInline. +[[nodiscard]] __m128 SIMD_FLAGS(InOut, RegisterOnly, ForceInline) contract_inout_registeronly_forceinline(__m128 value) noexcept; + +/// Declares the InOut boundary with RegisterOnly, Flatten. +[[nodiscard]] __m128 SIMD_FLAGS(InOut, RegisterOnly, Flatten) contract_inout_registeronly_flatten(__m128 value) noexcept; + +/// Declares the InOut boundary with ForceInline, Flatten. +[[nodiscard]] __m128 SIMD_FLAGS(InOut, ForceInline, Flatten) contract_inout_forceinline_flatten(__m128 value) noexcept; + +/// Declares the InOut boundary with RegisterOnly, ForceInline, Flatten. +[[nodiscard]] __m128 SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) contract_inout_registeronly_forceinline_flatten(__m128 value) noexcept; +} // namespace SimdLibMethodFlagsContract diff --git a/tests/method_flags/MethodFlagsPrototype.h b/tests/method_flags/MethodFlagsPrototype.h new file mode 100644 index 0000000..90e0f0c --- /dev/null +++ b/tests/method_flags/MethodFlagsPrototype.h @@ -0,0 +1,61 @@ +#pragma once + +// Dependency-free preprocessing prototype. Production integration belongs in the +// public configuration boundary after declaration placement is qualified. + +#ifndef SIMDLIB_DETAIL_FLAGS_VECTORCALL +#define SIMDLIB_DETAIL_FLAGS_VECTORCALL +#endif +#ifndef SIMDLIB_DETAIL_FLAGS_REGISTER_ONLY +#define SIMDLIB_DETAIL_FLAGS_REGISTER_ONLY +#endif +#ifndef SIMDLIB_DETAIL_FLAGS_FORCE_INLINE +#define SIMDLIB_DETAIL_FLAGS_FORCE_INLINE +#endif +#ifndef SIMDLIB_DETAIL_FLAGS_FLATTEN +#define SIMDLIB_DETAIL_FLAGS_FLATTEN +#endif + +#define SIMDLIB_DETAIL_FLAGS_CAT_RAW(left, right) left##right +#define SIMDLIB_DETAIL_FLAGS_CAT(left, right) SIMDLIB_DETAIL_FLAGS_CAT_RAW(left, right) + +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_ static_assert(false, "SIMDLIB_FLAGS_ERROR_EMPTY"); +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_Neither +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_In SIMDLIB_DETAIL_FLAGS_VECTORCALL +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_Out SIMDLIB_DETAIL_FLAGS_VECTORCALL +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_InOut SIMDLIB_DETAIL_FLAGS_VECTORCALL + +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_RegisterOnly SIMDLIB_DETAIL_FLAGS_REGISTER_ONLY +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_ForceInline SIMDLIB_DETAIL_FLAGS_FORCE_INLINE +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_Flatten SIMDLIB_DETAIL_FLAGS_FLATTEN +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_RegisterOnly_ForceInline SIMDLIB_DETAIL_FLAGS_FORCE_INLINE SIMDLIB_DETAIL_FLAGS_REGISTER_ONLY +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_RegisterOnly_Flatten SIMDLIB_DETAIL_FLAGS_FLATTEN SIMDLIB_DETAIL_FLAGS_REGISTER_ONLY +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_ForceInline_Flatten SIMDLIB_DETAIL_FLAGS_FLATTEN SIMDLIB_DETAIL_FLAGS_FORCE_INLINE +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_3_RegisterOnly_ForceInline_Flatten \ + SIMDLIB_DETAIL_FLAGS_FLATTEN SIMDLIB_DETAIL_FLAGS_FORCE_INLINE SIMDLIB_DETAIL_FLAGS_REGISTER_ONLY + +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY_RAW(mode) SIMDLIB_DETAIL_FLAGS_BOUNDARY_##mode +#define SIMDLIB_DETAIL_FLAGS_BOUNDARY(mode) SIMDLIB_DETAIL_FLAGS_BOUNDARY_RAW(mode) +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_RAW(a) SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_##a +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_1(a) SIMDLIB_DETAIL_FLAGS_MODIFIERS_1_RAW(a) +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_RAW(a, b) SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_##a##_##b +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_2(a, b) SIMDLIB_DETAIL_FLAGS_MODIFIERS_2_RAW(a, b) +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_3_RAW(a, b, c) SIMDLIB_DETAIL_FLAGS_MODIFIERS_3_##a##_##b##_##c +#define SIMDLIB_DETAIL_FLAGS_MODIFIERS_3(a, b, c) SIMDLIB_DETAIL_FLAGS_MODIFIERS_3_RAW(a, b, c) + +#define SIMDLIB_DETAIL_FLAGS_1(boundary) SIMDLIB_DETAIL_FLAGS_BOUNDARY(boundary) +#define SIMDLIB_DETAIL_FLAGS_2(boundary, a) SIMDLIB_DETAIL_FLAGS_MODIFIERS_1(a) SIMDLIB_DETAIL_FLAGS_BOUNDARY(boundary) +#define SIMDLIB_DETAIL_FLAGS_3(boundary, a, b) SIMDLIB_DETAIL_FLAGS_MODIFIERS_2(a, b) SIMDLIB_DETAIL_FLAGS_BOUNDARY(boundary) +#define SIMDLIB_DETAIL_FLAGS_4(boundary, a, b, c) SIMDLIB_DETAIL_FLAGS_MODIFIERS_3(a, b, c) SIMDLIB_DETAIL_FLAGS_BOUNDARY(boundary) +#define SIMDLIB_DETAIL_FLAGS_5(...) static_assert(false, "SIMDLIB_FLAGS_ERROR_TOO_MANY"); + +// Arity one deliberately includes an empty invocation. The empty boundary +// mapping diagnoses that case without __VA_OPT__ or a compiler extension. +#define SIMDLIB_DETAIL_FLAGS_ARITY_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, count, ...) count +#define SIMDLIB_DETAIL_FLAGS_ARITY_EXPAND(arguments) SIMDLIB_DETAIL_FLAGS_ARITY_IMPL arguments +#define SIMDLIB_DETAIL_FLAGS_ARITY(...) SIMDLIB_DETAIL_FLAGS_ARITY_EXPAND((__VA_ARGS__, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 3, 2, 1)) + +#define SIMDLIB_DETAIL_FLAGS_DISPATCH(count) SIMDLIB_DETAIL_FLAGS_CAT(SIMDLIB_DETAIL_FLAGS_, count) +#define SIMDLIB_DETAIL_FLAGS_EXPAND(...) __VA_ARGS__ + +#define SIMD_FLAGS(...) SIMDLIB_DETAIL_FLAGS_EXPAND(SIMDLIB_DETAIL_FLAGS_DISPATCH(SIMDLIB_DETAIL_FLAGS_ARITY(__VA_ARGS__))(__VA_ARGS__)) diff --git a/tests/method_flags/codegen/MethodFlagsFlagged.cpp b/tests/method_flags/codegen/MethodFlagsFlagged.cpp new file mode 100644 index 0000000..8f362bf --- /dev/null +++ b/tests/method_flags/codegen/MethodFlagsFlagged.cpp @@ -0,0 +1,78 @@ +#include + +#include + +#if defined(_MSC_VER) +#define SIMDLIB_METHOD_FLAGS_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_METHOD_FLAGS_NOINLINE __attribute__((noinline)) +#endif + +namespace SimdLibMethodFlagsCodegen +{ +/** Returns the square root of every input lane. */ +SIMDLIB_METHOD_FLAGS_NOINLINE __m128 SIMD_FLAGS(InOut, RegisterOnly) simdlib_method_flags_codegen_unary(__m128 value) noexcept +{ + return _mm_sqrt_ps(value); +} + +/** Adds corresponding lanes from two input registers. */ +SIMDLIB_METHOD_FLAGS_NOINLINE __m128 SIMD_FLAGS(InOut, RegisterOnly) simdlib_method_flags_codegen_binary(__m128 lhs, __m128 rhs) noexcept +{ + return _mm_add_ps(lhs, rhs); +} + +/** Multiplies two registers and adds a third register. */ +SIMDLIB_METHOD_FLAGS_NOINLINE __m128 SIMD_FLAGS(InOut, RegisterOnly) simdlib_method_flags_codegen_ternary(__m128 lhs, __m128 rhs, __m128 addend) noexcept +{ + return _mm_add_ps(_mm_mul_ps(lhs, rhs), addend); +} + +/** Extracts the low scalar lane from a register. */ +SIMDLIB_METHOD_FLAGS_NOINLINE float SIMD_FLAGS(In, RegisterOnly) simdlib_method_flags_codegen_scalar_result(__m128 value) noexcept +{ + return _mm_cvtss_f32(value); +} + +/** Broadcasts a scalar into a native register result. */ +SIMDLIB_METHOD_FLAGS_NOINLINE __m128 SIMD_FLAGS(Out, RegisterOnly) simdlib_method_flags_codegen_register_result(float value) noexcept +{ + return _mm_set1_ps(value); +} + +/** Loads an unaligned native register without writing through the source pointer. */ +SIMDLIB_METHOD_FLAGS_NOINLINE __m128 SIMD_FLAGS(Out, RegisterOnly) simdlib_method_flags_codegen_load(const float *source) noexcept +{ + return _mm_loadu_ps(source); +} + +/** Stores a native register through a caller-owned pointer. */ +SIMDLIB_METHOD_FLAGS_NOINLINE void SIMD_FLAGS(In) simdlib_method_flags_codegen_store(float *destination, __m128 value) noexcept +{ + _mm_storeu_ps(destination, value); +} + +/** Provides a small leaf for the force-inline-only fixture. */ +__m128 SIMD_FLAGS(InOut, RegisterOnly, ForceInline) simdlib_method_flags_force_leaf(__m128 value) noexcept +{ + return _mm_add_ps(value, _mm_set1_ps(1.0F)); +} + +/** Exercises ForceInline independently of Flatten. */ +SIMDLIB_METHOD_FLAGS_NOINLINE __m128 SIMD_FLAGS(InOut, RegisterOnly) simdlib_method_flags_codegen_forceinline(__m128 value) noexcept +{ + return simdlib_method_flags_force_leaf(value); +} + +/** Provides a small leaf for the flatten-only fixture. */ +inline __m128 SIMD_FLAGS(InOut, RegisterOnly) simdlib_method_flags_flatten_leaf(__m128 value) noexcept +{ + return _mm_mul_ps(value, value); +} + +/** Exercises Flatten independently of ForceInline. */ +SIMDLIB_METHOD_FLAGS_NOINLINE __m128 SIMD_FLAGS(InOut, RegisterOnly, Flatten) simdlib_method_flags_codegen_flatten(__m128 value) noexcept +{ + return simdlib_method_flags_flatten_leaf(simdlib_method_flags_flatten_leaf(value)); +} +} // namespace SimdLibMethodFlagsCodegen diff --git a/tests/method_flags/codegen/MethodFlagsRaw.cpp b/tests/method_flags/codegen/MethodFlagsRaw.cpp new file mode 100644 index 0000000..f6e0076 --- /dev/null +++ b/tests/method_flags/codegen/MethodFlagsRaw.cpp @@ -0,0 +1,85 @@ +#include + +#include + +#if defined(_MSC_VER) +#define SIMDLIB_METHOD_FLAGS_NOINLINE __declspec(noinline) +#else +#define SIMDLIB_METHOD_FLAGS_NOINLINE __attribute__((noinline)) +#endif + +namespace SimdLibMethodFlagsCodegen +{ +/** Returns the square root of every input lane. */ +SIMDLIB_METHOD_FLAGS_NOINLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS __m128 SIMDLIB_METHOD_FLAGS_VECTORCALL simdlib_method_flags_codegen_unary(__m128 value) noexcept +{ + return _mm_sqrt_ps(value); +} + +/** Adds corresponding lanes from two input registers. */ +SIMDLIB_METHOD_FLAGS_NOINLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS __m128 SIMDLIB_METHOD_FLAGS_VECTORCALL simdlib_method_flags_codegen_binary(__m128 lhs, + __m128 rhs) noexcept +{ + return _mm_add_ps(lhs, rhs); +} + +/** Multiplies two registers and adds a third register. */ +SIMDLIB_METHOD_FLAGS_NOINLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS __m128 SIMDLIB_METHOD_FLAGS_VECTORCALL +simdlib_method_flags_codegen_ternary(__m128 lhs, __m128 rhs, __m128 addend) noexcept +{ + return _mm_add_ps(_mm_mul_ps(lhs, rhs), addend); +} + +/** Extracts the low scalar lane from a register. */ +SIMDLIB_METHOD_FLAGS_NOINLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS float SIMDLIB_METHOD_FLAGS_VECTORCALL +simdlib_method_flags_codegen_scalar_result(__m128 value) noexcept +{ + return _mm_cvtss_f32(value); +} + +/** Broadcasts a scalar into a native register result. */ +SIMDLIB_METHOD_FLAGS_NOINLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS __m128 SIMDLIB_METHOD_FLAGS_VECTORCALL +simdlib_method_flags_codegen_register_result(float value) noexcept +{ + return _mm_set1_ps(value); +} + +/** Loads an unaligned native register without writing through the source pointer. */ +SIMDLIB_METHOD_FLAGS_NOINLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS __m128 SIMDLIB_METHOD_FLAGS_VECTORCALL +simdlib_method_flags_codegen_load(const float *source) noexcept +{ + return _mm_loadu_ps(source); +} + +/** Stores a native register through a caller-owned pointer. */ +SIMDLIB_METHOD_FLAGS_NOINLINE void SIMDLIB_METHOD_FLAGS_VECTORCALL simdlib_method_flags_codegen_store(float *destination, __m128 value) noexcept +{ + _mm_storeu_ps(destination, value); +} + +/** Provides a small leaf for the force-inline-only fixture. */ +SIMDLIB_METHOD_FLAGS_FORCE_INLINE __m128 SIMDLIB_METHOD_FLAGS_VECTORCALL simdlib_method_flags_force_leaf(__m128 value) noexcept +{ + return _mm_add_ps(value, _mm_set1_ps(1.0F)); +} + +/** Exercises the raw ForceInline mapping independently of Flatten. */ +SIMDLIB_METHOD_FLAGS_NOINLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS __m128 SIMDLIB_METHOD_FLAGS_VECTORCALL +simdlib_method_flags_codegen_forceinline(__m128 value) noexcept +{ + return simdlib_method_flags_force_leaf(value); +} + +/** Provides a small leaf for the flatten-only fixture. */ +inline __m128 SIMDLIB_METHOD_FLAGS_VECTORCALL simdlib_method_flags_flatten_leaf(__m128 value) noexcept +{ + return _mm_mul_ps(value, value); +} + +/** Exercises the raw Flatten mapping independently of ForceInline. */ +SIMDLIB_METHOD_FLAGS_FLATTEN SIMDLIB_METHOD_FLAGS_NOINLINE SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS __m128 SIMDLIB_METHOD_FLAGS_VECTORCALL +simdlib_method_flags_codegen_flatten(__m128 value) noexcept +{ + return simdlib_method_flags_flatten_leaf(simdlib_method_flags_flatten_leaf(value)); +} +} // namespace SimdLibMethodFlagsCodegen diff --git a/tests/method_flags/placement/CMakeLists.txt b/tests/method_flags/placement/CMakeLists.txt new file mode 100644 index 0000000..b3d3987 --- /dev/null +++ b/tests/method_flags/placement/CMakeLists.txt @@ -0,0 +1,90 @@ +cmake_minimum_required(VERSION 3.31) + +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(SimdLibMethodFlagsPlacement LANGUAGES CXX) + set(CMAKE_CXX_SCAN_FOR_MODULES OFF) + include(CTest) + set(SIMDLIB_METHOD_FLAGS_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../..") + add_library(SimdLibMethodFlagsHeaders INTERFACE) + target_include_directories(SimdLibMethodFlagsHeaders INTERFACE + "${SIMDLIB_METHOD_FLAGS_ROOT}/include" + "${SIMDLIB_METHOD_FLAGS_ROOT}/tests/method_flags") + set(simdlib_method_flags_dependency SimdLibMethodFlagsHeaders) + set(simdlib_method_flags_enable_cxx23 ON) +else() + set(SIMDLIB_METHOD_FLAGS_ROOT "${CMAKE_SOURCE_DIR}") + set(simdlib_method_flags_dependency SimdLib::SimdLib) + set(simdlib_method_flags_enable_cxx23 ${SIMDLIB_REGISTER_COMPILER_SUPPORTED}) +endif() + +# Adds strict warnings to one compiler-placement target. +function(simdlib_configure_method_flags_target target) + if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") + target_compile_options(${target} PRIVATE /W4 /WX /permissive-) + else() + target_compile_options(${target} PRIVATE + -Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion -Werror) + endif() +endfunction() + +add_library(MethodFlagsPlacementCxx20 OBJECT MethodFlagsPlacementCxx20.cpp) +if(COMMAND simdlib_register_development_target) + simdlib_register_development_target(MethodFlagsPlacementCxx20 + COMPILER_CONTRACT) +endif() +target_link_libraries(MethodFlagsPlacementCxx20 PRIVATE + ${simdlib_method_flags_dependency}) +target_compile_features(MethodFlagsPlacementCxx20 PRIVATE cxx_std_20) +set_target_properties(MethodFlagsPlacementCxx20 PROPERTIES + CXX_EXTENSIONS OFF) +simdlib_configure_method_flags_target(MethodFlagsPlacementCxx20) + +if(simdlib_method_flags_enable_cxx23) + add_library(MethodFlagsPlacementCxx23 OBJECT MethodFlagsPlacementCxx23.cpp) + if(COMMAND simdlib_register_development_target) + simdlib_register_development_target(MethodFlagsPlacementCxx23 + COMPILER_CONTRACT) + endif() + target_link_libraries(MethodFlagsPlacementCxx23 PRIVATE + ${simdlib_method_flags_dependency}) + target_compile_features(MethodFlagsPlacementCxx23 PRIVATE cxx_std_23) + set_target_properties(MethodFlagsPlacementCxx23 PROPERTIES + CXX_EXTENSIONS OFF) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(MethodFlagsPlacementCxx23 PRIVATE /std:c++latest) + endif() + simdlib_configure_method_flags_target(MethodFlagsPlacementCxx23) +endif() + +add_executable(MethodFlagsPlacementAbi + MethodFlagsPlacementAbiDefinition.cpp + MethodFlagsPlacementAbiConsumer.cpp) +if(COMMAND simdlib_register_development_target) + simdlib_register_development_target(MethodFlagsPlacementAbi + COMPILER_CONTRACT) +endif() +target_link_libraries(MethodFlagsPlacementAbi PRIVATE + ${simdlib_method_flags_dependency}) +target_compile_features(MethodFlagsPlacementAbi PRIVATE cxx_std_20) +set_target_properties(MethodFlagsPlacementAbi PROPERTIES + CXX_EXTENSIONS OFF) +simdlib_configure_method_flags_target(MethodFlagsPlacementAbi) + +add_custom_target(MethodFlagsPlacement) +if(COMMAND simdlib_register_development_target) + simdlib_register_development_target(MethodFlagsPlacement COMPILER_CONTRACT) +endif() +add_dependencies(MethodFlagsPlacement + MethodFlagsPlacementCxx20 + MethodFlagsPlacementAbi) +if(TARGET MethodFlagsPlacementCxx23) + add_dependencies(MethodFlagsPlacement MethodFlagsPlacementCxx23) +endif() + +if(BUILD_TESTING) + add_test(NAME MethodFlagsPlacementAbi + COMMAND MethodFlagsPlacementAbi) + set_tests_properties(MethodFlagsPlacementAbi PROPERTIES + LABELS "CONFIGURATION;METHOD_FLAGS;ABI") + simdlib_register_development_test(MethodFlagsPlacementAbi COMPILER_CONTRACT) +endif() diff --git a/tests/method_flags/placement/MethodFlagsPlacementAbiConsumer.cpp b/tests/method_flags/placement/MethodFlagsPlacementAbiConsumer.cpp new file mode 100644 index 0000000..89ba22b --- /dev/null +++ b/tests/method_flags/placement/MethodFlagsPlacementAbiConsumer.cpp @@ -0,0 +1,14 @@ +#include "MethodFlagsPlacementFixture.h" + +/// Links and executes both declaration spellings through their derived types. +int main() +{ + const auto input = _mm_set1_ps(7.0F); + const auto flagged = SimdLibMethodFlagsPlacement::compatible_flagged_address(input); + const auto legacy = SimdLibMethodFlagsPlacement::legacy_address(input); + const auto flagged_in = SimdLibMethodFlagsPlacement::compatible_flagged_in_address(input); + const auto legacy_in = SimdLibMethodFlagsPlacement::legacy_in_abi(input); + const auto flagged_out = SimdLibMethodFlagsPlacement::compatible_flagged_out_address(7.0F); + const auto legacy_out = SimdLibMethodFlagsPlacement::legacy_out_abi(7.0F); + return _mm_cvtss_f32(flagged) == _mm_cvtss_f32(legacy) && flagged_in == legacy_in && _mm_cvtss_f32(flagged_out) == _mm_cvtss_f32(legacy_out) ? 0 : 1; +} diff --git a/tests/method_flags/placement/MethodFlagsPlacementAbiDefinition.cpp b/tests/method_flags/placement/MethodFlagsPlacementAbiDefinition.cpp new file mode 100644 index 0000000..c89214e --- /dev/null +++ b/tests/method_flags/placement/MethodFlagsPlacementAbiDefinition.cpp @@ -0,0 +1,40 @@ +#include "MethodFlagsPlacementFixture.h" + +namespace SimdLibMethodFlagsPlacement +{ +/// Defines a flagged declaration with the legacy spelling in another translation unit. +SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS vector_type SIMDLIB_METHOD_FLAGS_VECTORCALL flagged_abi(vector_type value) noexcept +{ + return value; +} + +/// Defines a legacy declaration with the flagged pre-name spelling. +vector_type SIMD_FLAGS(InOut, RegisterOnly) legacy_abi(vector_type value) noexcept +{ + return value; +} + +/// Defines a flagged In declaration with the legacy spelling. +SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS int SIMDLIB_METHOD_FLAGS_VECTORCALL flagged_in_abi(vector_type value) noexcept +{ + return static_cast(_mm_cvtss_f32(value)); +} + +/// Defines a legacy In declaration with the flagged pre-name spelling. +int SIMD_FLAGS(In, RegisterOnly) legacy_in_abi(vector_type value) noexcept +{ + return static_cast(_mm_cvtss_f32(value)); +} + +/// Defines a flagged Out declaration with the legacy spelling. +SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS vector_type SIMDLIB_METHOD_FLAGS_VECTORCALL flagged_out_abi(float value) noexcept +{ + return _mm_set1_ps(value); +} + +/// Defines a legacy Out declaration with the flagged pre-name spelling. +vector_type SIMD_FLAGS(Out, RegisterOnly) legacy_out_abi(float value) noexcept +{ + return _mm_set1_ps(value); +} +} // namespace SimdLibMethodFlagsPlacement diff --git a/tests/method_flags/placement/MethodFlagsPlacementCxx20.cpp b/tests/method_flags/placement/MethodFlagsPlacementCxx20.cpp new file mode 100644 index 0000000..42c6340 --- /dev/null +++ b/tests/method_flags/placement/MethodFlagsPlacementCxx20.cpp @@ -0,0 +1,18 @@ +#include "MethodFlagsPlacementFixture.h" + +namespace SimdLibMethodFlagsPlacement +{ +static_assert(inline_increment(1) == 2); +static_assert(constrained_increment(2) == 3); +static_assert(trailing_increment(3) == 4); + +/// Instantiates every supported C++20 declaration shape. +[[nodiscard]] vector_type SIMD_FLAGS(InOut, RegisterOnly) exercise_cxx20(vector_type value) noexcept +{ + MemberShapes members; + const auto member_result = members.member_transform(value); + const auto static_result = MemberShapes::static_transform(member_result); + const auto boxed_result = VectorBox{static_result} + VectorBox{value}; + return free_transform(boxed_result.value); +} +} // namespace SimdLibMethodFlagsPlacement diff --git a/tests/method_flags/placement/MethodFlagsPlacementCxx23.cpp b/tests/method_flags/placement/MethodFlagsPlacementCxx23.cpp new file mode 100644 index 0000000..7c0a701 --- /dev/null +++ b/tests/method_flags/placement/MethodFlagsPlacementCxx23.cpp @@ -0,0 +1,31 @@ +#include "MethodFlagsPlacementFixture.h" + +namespace SimdLibMethodFlagsPlacement +{ +/// Provides explicit-object member and operator declaration shapes. +struct ExplicitObject final +{ + vector_type value; + + /// Returns a native value through a by-value explicit object parameter. + [[nodiscard]] vector_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) transform(this ExplicitObject self, vector_type rhs) noexcept + { + (void)rhs; + return leaf_transform(self.value); + } + + /// Adds an explicit-object operator declaration shape. + [[nodiscard]] ExplicitObject SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator+(this ExplicitObject lhs, ExplicitObject rhs) noexcept + { + (void)rhs; + return lhs; + } +}; + +/// Instantiates the supported explicit-object declarations. +[[nodiscard]] vector_type SIMD_FLAGS(InOut, RegisterOnly) exercise_cxx23(vector_type value) noexcept +{ + const auto object = ExplicitObject{value} + ExplicitObject{value}; + return object.transform(value); +} +} // namespace SimdLibMethodFlagsPlacement diff --git a/tests/method_flags/placement/MethodFlagsPlacementFixture.h b/tests/method_flags/placement/MethodFlagsPlacementFixture.h new file mode 100644 index 0000000..1a8eb2d --- /dev/null +++ b/tests/method_flags/placement/MethodFlagsPlacementFixture.h @@ -0,0 +1,110 @@ +#pragma once + +#include + +#include +#include +#include + +namespace SimdLibMethodFlagsPlacement +{ +using vector_type = __m128; + +/// Returns a native SIMD value through the fully composed declaration macro. +[[nodiscard]] vector_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline) leaf_transform(vector_type value) noexcept +{ + return value; +} + +/// Calls another flagged function so the flatten attribute has a real callee. +[[nodiscard]] vector_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) free_transform(vector_type value) noexcept +{ + return leaf_transform(value); +} + +/// Exercises an ordinary inline specifier independently of ForceInline. +[[nodiscard]] inline constexpr int SIMD_FLAGS(Neither, RegisterOnly) inline_increment(int value) noexcept +{ + return value + 1; +} + +/// Exercises constexpr, ForceInline, Flatten, and a leading requires clause. +template + requires std::integral +[[nodiscard]] constexpr value_type SIMD_FLAGS(Neither, RegisterOnly, ForceInline, Flatten) constrained_increment(value_type value) noexcept +{ + return static_cast(value + 1); +} + +/// Exercises an independently selected trailing return type. +template +[[nodiscard]] constexpr auto SIMD_FLAGS(Neither, RegisterOnly, ForceInline, Flatten) trailing_increment(value_type value) noexcept -> value_type + requires std::integral +{ + return static_cast(value + 1); +} + +/// Provides static and non-static member declaration shapes. +class MemberShapes final +{ + public: + /// Returns a native value from a static member. + [[nodiscard]] static vector_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) static_transform(vector_type value) noexcept + { + return leaf_transform(value); + } + + /// Returns a native value from a non-static member. + [[nodiscard]] vector_type SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) member_transform(vector_type value) const noexcept + { + return leaf_transform(value); + } +}; + +/// Wraps a native SIMD value for friend-definition and operator coverage. +struct VectorBox final +{ + vector_type value; + + /// Selects the left operand through a friend operator definition. + [[nodiscard]] friend VectorBox SIMD_FLAGS(InOut, RegisterOnly, ForceInline, Flatten) operator+(VectorBox lhs, VectorBox rhs) noexcept + { + (void)rhs; + return lhs; + } +}; + +/// Declares the canonical InOut spelling for cross-TU ABI verification. +[[nodiscard]] vector_type SIMD_FLAGS(InOut, RegisterOnly) flagged_abi(vector_type value) noexcept; + +/// Declares the legacy InOut calling-convention position for type comparison. +[[nodiscard]] SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS vector_type SIMDLIB_METHOD_FLAGS_VECTORCALL legacy_abi(vector_type value) noexcept; + +/// Declares the canonical In spelling for cross-TU ABI verification. +[[nodiscard]] int SIMD_FLAGS(In, RegisterOnly) flagged_in_abi(vector_type value) noexcept; + +/// Declares the legacy In calling-convention position for type comparison. +[[nodiscard]] SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS int SIMDLIB_METHOD_FLAGS_VECTORCALL legacy_in_abi(vector_type value) noexcept; + +/// Declares the canonical Out spelling for cross-TU ABI verification. +[[nodiscard]] vector_type SIMD_FLAGS(Out, RegisterOnly) flagged_out_abi(float value) noexcept; + +/// Declares the legacy Out calling-convention position for type comparison. +[[nodiscard]] SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS vector_type SIMDLIB_METHOD_FLAGS_VECTORCALL legacy_out_abi(float value) noexcept; + +using flagged_callback = decltype(&flagged_abi); +using legacy_callback = decltype(&legacy_abi); +using flagged_in_callback = decltype(&flagged_in_abi); +using legacy_in_callback = decltype(&legacy_in_abi); +using flagged_out_callback = decltype(&flagged_out_abi); +using legacy_out_callback = decltype(&legacy_out_abi); + +inline constexpr flagged_callback flagged_address = &flagged_abi; +inline constexpr legacy_callback legacy_address = &legacy_abi; +/// Proves the flagged declaration is directly assignable to the legacy callback type. +inline constexpr legacy_callback compatible_flagged_address = flagged_address; +/// Proves the flagged In declaration is directly assignable to the legacy callback type. +inline constexpr legacy_in_callback compatible_flagged_in_address = &flagged_in_abi; +/// Proves the flagged Out declaration is directly assignable to the legacy callback type. +inline constexpr legacy_out_callback compatible_flagged_out_address = &flagged_out_abi; +} // namespace SimdLibMethodFlagsPlacement diff --git a/tests/parent/CMakeLists.txt b/tests/parent/CMakeLists.txt new file mode 100644 index 0000000..004fe1f --- /dev/null +++ b/tests/parent/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required(VERSION 3.20) + +project(SimdLibParentSmoke LANGUAGES CXX) + +if(NOT DEFINED SIMDLIB_SOURCE_DIR) + get_filename_component(SIMDLIB_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) +endif() + +include(CTest) +if(NOT BUILD_TESTING) + message(FATAL_ERROR "The parent-consumer fixture requires its own tests to be enabled") +endif() +add_test(NAME ParentConsumer.OwnTest COMMAND "${CMAKE_COMMAND}" -E true) + +get_cmake_property(parent_cache_before CACHE_VARIABLES) +add_subdirectory("${SIMDLIB_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/simdlib" EXCLUDE_FROM_ALL) +get_cmake_property(parent_cache_after CACHE_VARIABLES) + +foreach(cache_variable IN LISTS parent_cache_after) + if(cache_variable MATCHES "^SIMDLIB_" AND NOT cache_variable IN_LIST parent_cache_before) + message(FATAL_ERROR + "add_subdirectory introduced development cache option ${cache_variable}") + endif() +endforeach() + +get_property(simdlib_nested_tests + DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/simdlib" PROPERTY TESTS) +if(simdlib_nested_tests) + message(FATAL_ERROR + "add_subdirectory registered development tests: ${simdlib_nested_tests}") +endif() +get_property(parent_tests DIRECTORY PROPERTY TESTS) +if(NOT parent_tests STREQUAL "ParentConsumer.OwnTest") + message(FATAL_ERROR + "SimdLib altered the parent CTest inventory: ${parent_tests}") +endif() + +foreach(required_target IN ITEMS SimdLib SimdLib::SimdLib SimdLibRegister SimdLib::Register) + if(NOT TARGET ${required_target}) + message(FATAL_ERROR "Required production target is missing: ${required_target}") + endif() +endforeach() +foreach(forbidden_target IN ITEMS + ExhaustiveArtifacts BenchmarkArtifacts Benchmarks DevelopmentWarnings + ApiExamples RegisterExamples CoverageReset CoverageReport Catch2 Catch2WithMain) + if(TARGET ${forbidden_target}) + message(FATAL_ERROR + "add_subdirectory introduced development target ${forbidden_target}") + endif() +endforeach() diff --git a/tests/register/RegisterRepresentation.tests.cpp b/tests/register/RegisterRepresentation.tests.cpp new file mode 100644 index 0000000..4233947 --- /dev/null +++ b/tests/register/RegisterRepresentation.tests.cpp @@ -0,0 +1,128 @@ +#include +#include +#include + +#include +#include +#include + +namespace +{ + +/** @brief Reports whether any intentionally unsupported scalar arithmetic expression is available. */ +template +concept has_scalar_arithmetic = requires(value_t value, typename value_t::element_type scalar) { + value + scalar; + value - scalar; + value * scalar; + value / scalar; +}; + +/** @brief Verifies that the intentionally disabled compound-assignment surface remains unavailable. */ +template consteval bool has_no_compound_assignments() +{ + return !requires(value_t lhs, value_t rhs) { lhs += rhs; } && !requires(value_t lhs, value_t rhs) { lhs -= rhs; } && + !requires(value_t lhs, value_t rhs) { lhs *= rhs; } && !requires(value_t lhs, value_t rhs) { lhs /= rhs; } && + !requires(value_t lhs, value_t rhs) { lhs %= rhs; } && !requires(value_t lhs, value_t rhs) { lhs &= rhs; } && + !requires(value_t lhs, value_t rhs) { lhs |= rhs; } && !requires(value_t lhs, value_t rhs) { lhs ^= rhs; } && + !requires(value_t lhs) { lhs <<= 1; } && !requires(value_t lhs) { lhs >>= 1; }; +} + +/** @brief Checks the aggregate predicate construction and conversion contract. */ +template consteval bool has_mask_construction_contract() +{ + return std::is_constructible_v && !std::is_constructible_v && + !std::is_constructible_v && !std::is_convertible_v; +} + +/** @brief Checks the complete public RegisterMask interface for one predicate type. */ +template consteval bool has_complete_register_mask_surface() +{ + return SimdLib::IRegisterMask::Type && SimdLib::IRegisterMask::Any && SimdLib::IRegisterMask::All && + SimdLib::IRegisterMask::None && SimdLib::IRegisterMask::Bits && SimdLib::IRegisterMask::Select && + SimdLib::IRegisterMask::BitwiseAnd && SimdLib::IRegisterMask::BitwiseOr && SimdLib::IRegisterMask::BitwiseXor && + SimdLib::IRegisterMask::BitwiseNot; +} + +/** @brief Checks the required object-model traits for one register-shaped value type. */ +template consteval bool has_complete_register_value_traits() +{ + using native_type = typename value_t::native_type; + return sizeof(value_t) == sizeof(native_type) && alignof(value_t) == alignof(native_type) && std::is_standard_layout_v && + std::is_trivially_copy_constructible_v && !std::is_trivially_default_constructible_v && + std::is_trivially_move_constructible_v && std::is_trivially_copy_assignable_v && std::is_trivially_move_assignable_v && + std::is_trivially_destructible_v && std::is_trivially_copyable_v; +} + +/** @brief Reports whether a Register accepts one complete homogeneous logical lane list. */ +template consteval bool has_complete_lane_construction(std::index_sequence) +{ + using element_t = typename value_t::element_type; + return SimdLib::IRegister::FromLanes(indices), element_t{}))...>; +} +/** @brief Checks Register and RegisterMask shape invariants for one element type and width. */ +template consteval bool has_complete_register_shapes() +{ + using register_type = SimdLib::Register; + using mask_type = SimdLib::RegisterMask; + static_assert(std::is_aggregate_v); + static_assert(std::is_aggregate_v); + return SimdLib::RegisterAvailable && SimdLib::is_register_available_v && SimdLib::IRegister::Type && + SimdLib::IRegister::Zero && SimdLib::IRegister::Broadcast && + has_complete_lane_construction(std::make_index_sequence{}) && + SimdLib::IRegister::FromArray && SimdLib::IRegister::Load && SimdLib::IRegister::LoadAligned && + SimdLib::IRegister::LoadBytes && SimdLib::IRegister::Store && SimdLib::IRegister::StoreAligned && + SimdLib::IRegister::StoreBytes && SimdLib::IRegister::ToArray && SimdLib::IRegister::Lane && + SimdLib::IRegister::WithLane && SimdLib::IRegister::BitwiseAnd && SimdLib::IRegister::BitwiseOr && + SimdLib::IRegister::BitwiseXor && SimdLib::IRegister::BitwiseNot && SimdLib::IRegister::BitwiseAndNot && + SimdLib::IRegister::Movemask && SimdLib::IRegister::LaneSignBits && SimdLib::IRegister::CompareEqual && + SimdLib::IRegister::CompareGreater && SimdLib::IRegister::CompareGreaterEqual && + SimdLib::IRegister::CompareLess && SimdLib::IRegister::CompareLessEqual && SimdLib::IRegister::Equal && + SimdLib::IRegister::NotEqual && has_complete_register_mask_surface() && + has_complete_register_value_traits() && has_complete_register_value_traits() && + has_mask_construction_contract() && !SimdLib::IRegister::Lane && + !SimdLib::IRegister::WithLane && has_no_compound_assignments() && + has_no_compound_assignments() && register_type::register_width == bits && register_type::byte_count == bits / 8 && + register_type::lane_count == bits / (sizeof(element_t) * 8) && mask_type::register_width == bits && + mask_type::lane_count == register_type::lane_count && std::same_as; +} + +/** @brief Checks the exact operator surface for one element type and width. */ +template consteval bool has_exact_operation_constraints() +{ + using register_type = SimdLib::Register; + constexpr bool integral = std::is_integral_v; + return !has_scalar_arithmetic && SimdLib::IRegister::Modulus == integral && + SimdLib::IRegister::ShiftLeft == integral && SimdLib::IRegister::LogicalShiftRight == integral && + SimdLib::IRegister::ShiftRight == integral && SimdLib::IRegister::ShiftBytesLeftSlow == (integral && bits == 128) && + SimdLib::IRegister::ShiftBytesRightSlow == (integral && bits == 128) && + SimdLib::IRegister::ShiftBytesLeft == integral && SimdLib::IRegister::ShiftBytesRight == integral && + !SimdLib::IRegister::ShiftBytesLeft && !SimdLib::IRegister::ShiftBytesRight && + SimdLib::IRegister::ShiftBitsLeftSlow == (integral && bits == 128) && + SimdLib::IRegister::ShiftBitsRightSlow == (integral && bits == 128) && + SimdLib::IRegister::ShiftBitsLeft == (integral && bits == 128) && + SimdLib::IRegister::ShiftBitsRight == (integral && bits == 128) && !SimdLib::IRegister::ShiftBitsLeft && + !SimdLib::IRegister::ShiftBitsRight; +} + +#define SIMDLIB_ASSERT_REGISTER_SHAPES(element_type, width) \ + static_assert(has_complete_register_shapes()); \ + static_assert(has_exact_operation_constraints()) + +SIMDLIB_ASSERT_REGISTER_SHAPES(std::int8_t, SIMDLIB_REGISTER_TEST_WIDTH); +SIMDLIB_ASSERT_REGISTER_SHAPES(std::uint8_t, SIMDLIB_REGISTER_TEST_WIDTH); +SIMDLIB_ASSERT_REGISTER_SHAPES(std::int16_t, SIMDLIB_REGISTER_TEST_WIDTH); +SIMDLIB_ASSERT_REGISTER_SHAPES(std::uint16_t, SIMDLIB_REGISTER_TEST_WIDTH); +SIMDLIB_ASSERT_REGISTER_SHAPES(std::int32_t, SIMDLIB_REGISTER_TEST_WIDTH); +SIMDLIB_ASSERT_REGISTER_SHAPES(std::uint32_t, SIMDLIB_REGISTER_TEST_WIDTH); +SIMDLIB_ASSERT_REGISTER_SHAPES(std::int64_t, SIMDLIB_REGISTER_TEST_WIDTH); +SIMDLIB_ASSERT_REGISTER_SHAPES(std::uint64_t, SIMDLIB_REGISTER_TEST_WIDTH); +SIMDLIB_ASSERT_REGISTER_SHAPES(float, SIMDLIB_REGISTER_TEST_WIDTH); +SIMDLIB_ASSERT_REGISTER_SHAPES(double, SIMDLIB_REGISTER_TEST_WIDTH); + +#undef SIMDLIB_ASSERT_REGISTER_SHAPES + +using native_register_type = SimdLib::NativeRegister; +static_assert(native_register_type::register_width == (SimdLib::is_register_available_v ? 256 : 128)); + +} // namespace diff --git a/tests/register_odr/main.cpp b/tests/register_odr/main.cpp new file mode 100644 index 0000000..435ae0a --- /dev/null +++ b/tests/register_odr/main.cpp @@ -0,0 +1,37 @@ +#include + +#include + +namespace +{ +using Register = SimdLib::Register; +using RegisterMask = Register::mask_type; +} // namespace + +/** + * @brief Adds two complete registers in a second translation unit. + * @param lhs Left operand. + * @param rhs Right operand. + * @return Lane-wise sum. + */ +Register SIMD_FLAGS(InOut) second_translation_unit_add(Register lhs, Register rhs) noexcept; + +/** + * @brief Compares two complete registers in a second translation unit. + * @param lhs Left operand. + * @param rhs Right operand. + * @return Per-lane equality predicate. + */ +RegisterMask SIMD_FLAGS(InOut) second_translation_unit_equal(Register lhs, Register rhs) noexcept; + +/** + * @brief Verifies umbrella exposure and inline Register definitions across translation units. + * @return Zero when the cross-translation-unit results are correct. + */ +int main() +{ + const Register expected = Register::broadcast(5); + const Register actual = second_translation_unit_add(Register::broadcast(2), Register::broadcast(3)); + const RegisterMask equal = second_translation_unit_equal(actual, expected); + return equal.all() && equal.select(actual, Register::zero()) == expected ? 0 : 1; +} diff --git a/tests/register_odr/second_translation_unit.cpp b/tests/register_odr/second_translation_unit.cpp new file mode 100644 index 0000000..eed2645 --- /dev/null +++ b/tests/register_odr/second_translation_unit.cpp @@ -0,0 +1,31 @@ +#include + +#include + +namespace +{ +using Register = SimdLib::Register; +using RegisterMask = Register::mask_type; +} // namespace + +/** + * @brief Adds two complete registers through the public umbrella header. + * @param lhs Left operand. + * @param rhs Right operand. + * @return Lane-wise sum. + */ +Register SIMD_FLAGS(InOut) second_translation_unit_add(Register lhs, Register rhs) noexcept +{ + return lhs + rhs; +} + +/** + * @brief Compares two complete registers through the public umbrella header. + * @param lhs Left operand. + * @param rhs Right operand. + * @return Per-lane equality predicate. + */ +RegisterMask SIMD_FLAGS(InOut) second_translation_unit_equal(Register lhs, Register rhs) noexcept +{ + return lhs.compare_equal(rhs); +} diff --git a/tests/smoke/main.cpp b/tests/smoke/main.cpp index e2b98c7..a289cf7 100644 --- a/tests/smoke/main.cpp +++ b/tests/smoke/main.cpp @@ -21,12 +21,12 @@ std::uint32_t first_translation_unit_resample() noexcept SimdLib::SimdResample::ExpandBitsToBytesBy8(any, expanded); return any[0] + all[0] + parity[0] + expanded[1]; } -} +} // namespace int main() { - return SimdLib::version_major + SimdLib::version_minor + SimdLib::version_patch == second_translation_unit_version() - && first_translation_unit_resample() == second_translation_unit_resample() - ? 0 - : 1; + return SimdLib::version_major + SimdLib::version_minor + SimdLib::version_patch == second_translation_unit_version() && + first_translation_unit_resample() == second_translation_unit_resample() + ? 0 + : 1; } diff --git a/tools/Audit-ValidationMatrix.ps1 b/tools/Audit-ValidationMatrix.ps1 new file mode 100644 index 0000000..388c8d8 --- /dev/null +++ b/tools/Audit-ValidationMatrix.ps1 @@ -0,0 +1,66 @@ +<# +.SYNOPSIS +Audits one generated validation cell against the canonical matrix contract. +.DESCRIPTION +The command reports duplicate targets or tests, missing ownership, and profile +membership that differs from tools/validation-matrix.json. +.PARAMETER Cell +Canonical cell identifier from tools/validation-matrix.json. +.PARAMETER BuildDirectory +Configured CMake build tree containing generated ownership inventories. +.PARAMETER Configuration +Optional multi-config CTest configuration. +.PARAMETER ResultPath +Optional machine-readable audit result path. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)][string]$Cell, + [Parameter(Mandatory)][string]$BuildDirectory, + [string]$Configuration = '', + [string]$ResultPath = '' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force + +$repositoryRoot = Get-PipelineRepositoryRoot +$cmake = (Get-Command cmake -ErrorAction Stop).Source +$ctest = (Get-Command ctest -ErrorAction Stop).Source +$BuildDirectory = [System.IO.Path]::GetFullPath($BuildDirectory) +if (-not $ResultPath) { + $ResultPath = Join-Path $BuildDirectory 'validation-inventory.audit.json' +} +$ResultPath = [System.IO.Path]::GetFullPath($ResultPath) +$arguments = @( + "-DMATRIX_FILE=$(Join-Path $PSScriptRoot 'validation-matrix.json')", + "-DCELL_ID=$Cell", + "-DBUILD_DIRECTORY=$BuildDirectory", + "-DCMAKE_CTEST_COMMAND=$ctest", + "-DRESULT_FILE=$ResultPath" +) +if ($Configuration) { + $arguments += "-DCONFIGURATION=$Configuration" +} +$arguments += @( + '-P', + (Join-Path $repositoryRoot 'cmake/AuditValidationInventory.cmake') +) + +& $cmake @arguments | Out-Host +if ($LASTEXITCODE -ne 0) { + throw "Validation matrix inventory audit failed for $Cell" +} +if (-not (Test-Path -LiteralPath $ResultPath -PathType Leaf)) { + throw "Validation matrix inventory audit did not produce $ResultPath" +} + +$result = Get-Content -LiteralPath $ResultPath -Raw | ConvertFrom-Json +Write-Host ( + "Matrix audit passed: cell={0} profile={1} targets={2} selected={3} tests={4}" -f + $result.cell, + $result.profile, + $result.targets, + $result.selectedTargets, + $result.tests) diff --git a/tools/Build-Benchmarks.ps1 b/tools/Build-Benchmarks.ps1 new file mode 100644 index 0000000..68f0630 --- /dev/null +++ b/tools/Build-Benchmarks.ps1 @@ -0,0 +1,79 @@ +<# +.SYNOPSIS +Builds benchmark artifacts in validated Release trees. +.DESCRIPTION +This operation never creates a benchmark-specific configure tree. Native and +container benchmark targets reuse the matching Release validation fingerprints. +#> +[CmdletBinding()] +param( + [ValidateSet('All', 'Native', 'Containers')] + [string]$Scope = 'All', + [ValidateSet('All', 'Msvc', 'ClangCl', 'ClangCoverage', 'Gcc13', 'Gcc14', 'Clang22')] + [string[]]$Compiler = @('All') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force + +<# +.SYNOPSIS +Expands and validates compiler filters for the requested platform scope. +#> +function Resolve-BenchmarkCompilerSelection { + $nativeNames = @(Get-PipelineValidationCompilers -Platform native) + $containerNames = @(Get-PipelineValidationCompilers -Platform container) + $benchmarkCells = @(Get-PipelineValidationOperationCells -Operation benchmarks) + $nativeBenchmarkOwners = @($benchmarkCells | + Where-Object platform -eq native | ForEach-Object compiler) + if ('All' -in $Compiler -and $Compiler.Count -ne 1) { throw 'Compiler All cannot be combined with another compiler filter.' } + $selected = if ($Compiler -contains 'All') { + switch ($Scope) { + 'Native' { $nativeNames } + 'Containers' { $containerNames } + default { $nativeNames + $containerNames } + } + } else { @($Compiler | Select-Object -Unique) } + if ($Scope -eq 'Native' -and @($selected | Where-Object { $_ -in $containerNames }).Count) { throw 'Container compiler filters are invalid for Native scope.' } + if ($Scope -eq 'Containers' -and @($selected | Where-Object { $_ -in $nativeNames }).Count) { throw 'Native compiler filters are invalid for Containers scope.' } + [pscustomobject]@{ + Native = if ($Scope -in @('All', 'Native')) { @($selected | Where-Object { $_ -in $nativeBenchmarkOwners }) } else { @() } + Containers = if ($Scope -in @('All', 'Containers')) { @($selected | Where-Object { $_ -in $containerNames }) } else { @() } + } +} + +$selection = Resolve-BenchmarkCompilerSelection +$operations = [System.Collections.Generic.List[object]]::new() +$nativeSelection = @($selection.Native) +$containerSelection = @($selection.Containers) +if ($nativeSelection.Count) { + $nativeFilter = if ($nativeSelection.Count -eq 2) { 'All' } else { $nativeSelection[0] } + $operations.Add([pscustomobject]@{ + Id = 'native-benchmarks'; Script = Join-Path $PSScriptRoot 'Run-NativeMatrix.ps1' + Arguments = @('-Action', 'BuildBenchmarks', '-Compiler', $nativeFilter, '-Cell', 'Release') + }) +} +if ($containerSelection.Count) { + $containerFilter = if ($containerSelection.Count -eq 3) { 'All' } else { $null } + if ($containerFilter) { + $operations.Add([pscustomobject]@{ + Id = 'container-benchmarks'; Script = Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1' + Arguments = @('-Action', 'BuildBenchmarks', '-Compiler', 'All', '-Cell', 'Release') + }) + } else { + foreach ($name in $containerSelection) { + $operations.Add([pscustomobject]@{ + Id = "container-$($name.ToLowerInvariant())-benchmarks"; Script = Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1' + Arguments = @('-Action', 'BuildBenchmarks', '-Compiler', $name, '-Cell', 'Release') + }) + } + } +} +if (-not $operations.Count) { + Write-Host 'No selected compiler owns benchmark artifacts.' + exit 0 +} +$logDirectory = Join-Path (Get-PipelineRepositoryRoot) "out/pipeline/logs/$(Get-Date -Format 'yyyyMMdd-HHmmssfff')-build-benchmarks-$PID" +Invoke-PipelineChildOperations -Operations $operations.ToArray() -LogDirectory $logDirectory +Write-Host "Benchmark artifacts built. Logs: $logDirectory" diff --git a/tools/Build.ps1 b/tools/Build.ps1 new file mode 100644 index 0000000..cb5b11e --- /dev/null +++ b/tools/Build.ps1 @@ -0,0 +1,178 @@ +<# +.SYNOPSIS +Builds the requested complete SimdLib validation artifact matrix. +.DESCRIPTION +Scope must be explicit so a host cannot silently omit required native or +container cells. The command builds validation artifacts and records an exact +manifest receipt consumed by Run-Tests.ps1. Benchmark compilation is owned +exclusively by Build-Benchmarks.ps1. +#> +[CmdletBinding()] +param( + [ValidateSet('', 'All', 'Native', 'Containers')] + [string]$Scope = '', + [ValidateSet('All', 'Msvc', 'ClangCl', 'ClangCoverage', 'Gcc13', 'Gcc14', 'Clang22')] + [string[]]$Compiler = @('All') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force +$repositoryRoot = Get-PipelineRepositoryRoot +$pipelineRoot = Join-Path $repositoryRoot 'out/pipeline' + +<# +.SYNOPSIS +Expands compiler filters and enforces their platform scope. +#> +function Resolve-BuildSelection { + $nativeNames = @(Get-PipelineValidationCompilers -Platform native) + $containerNames = @(Get-PipelineValidationCompilers -Platform container) + if (-not $Scope) { throw 'Build scope is required. Use -Scope All, -Scope Native, or -Scope Containers.' } + if ('All' -in $Compiler -and $Compiler.Count -ne 1) { throw 'Compiler All cannot be combined with another compiler filter.' } + if ($Compiler -contains 'All') { + $selected = switch ($Scope) { + 'Native' { $nativeNames } + 'Containers' { $containerNames } + default { $nativeNames + $containerNames } + } + } else { + $selected = @($Compiler | Select-Object -Unique) + } + if ($Scope -eq 'Native' -and @($selected | Where-Object { $_ -in $containerNames }).Count) { throw 'Container compiler filters are invalid for Native scope.' } + if ($Scope -eq 'Containers' -and @($selected | Where-Object { $_ -in $nativeNames }).Count) { throw 'Native compiler filters are invalid for Containers scope.' } + return @($selected) +} + +<# +.SYNOPSIS +Records the exact completed validation manifests produced by this build. +.PARAMETER SelectedCompilers +Canonical compiler selection. +.PARAMETER PipelineValidationPath +Machine-readable pipeline-tooling validation result for the current tooling digest. +#> +function Write-BuildReceipt { + param( + [Parameter(Mandatory)][string[]]$SelectedCompilers, + [Parameter(Mandatory)][string]$PipelineValidationPath + ) + $currentSourceDigest = Get-PipelineSourceDigest -RepositoryRoot $repositoryRoot + $toolingDigest = Get-PipelineToolingDigest -RepositoryRoot $repositoryRoot + $pipelineValidationEntry = New-PipelineValidationEntry ` + -RepositoryRoot $repositoryRoot ` + -ResultPath $PipelineValidationPath ` + -ExpectedToolingDigest $toolingDigest + $expectedPresets = @(Get-PipelineDefaultValidationPresets -SelectedCompilers $SelectedCompilers) + $manifestFiles = @(Get-ChildItem -LiteralPath $pipelineRoot -Filter 'validation-build.manifest' -File -Recurse -ErrorAction SilentlyContinue) + $entries = [System.Collections.Generic.List[object]]::new() + foreach ($preset in $expectedPresets) { + $matches = @($manifestFiles | Where-Object { + try { (Read-PipelineManifest -Path $_.FullName).preset -eq $preset } catch { $false } + } | Sort-Object LastWriteTimeUtc -Descending) + if ($matches.Count -eq 0) { throw "Build completed without the required manifest for preset $preset" } + $manifest = Read-PipelineManifest -Path $matches[0].FullName + if ($manifest.operation -ne 'build-validation' -or $manifest.status -ne 'complete') { throw "Incomplete validation manifest for preset $preset" } + if ($manifest.source_digest -ne $currentSourceDigest) { throw "Validation manifest has a stale source digest for preset $preset" } + if ($manifest.aggregate -ne 'ExhaustiveArtifacts') { throw "Default validation manifest has an unexpected scoped aggregate for preset $preset" } + foreach ($requiredManifestField in @( + 'target_inventory_sha256', + 'main_test_inventory_sha256', + 'matrix_cell', + 'matrix_contract_sha256', + 'validation_inventory_audit_sha256', + 'build_profile', + 'sanitizer', + 'instrumentation', + 'codegen_mode', + 'consumer_scope' + )) { + if (-not $manifest.ContainsKey($requiredManifestField) -or [string]::IsNullOrWhiteSpace($manifest[$requiredManifestField])) { + throw "Validation manifest omits required provenance $requiredManifestField for preset $preset" + } + } + foreach ($requiredInventoryField in @( + 'target_inventory_sha256', + 'main_test_inventory_sha256', + 'matrix_cell', + 'matrix_contract_sha256', + 'validation_inventory_audit_sha256' + )) { + if ($manifest[$requiredInventoryField] -eq 'none') { + throw "Validation manifest has no required $requiredInventoryField for preset $preset" + } + } + $inventoryAuditPath = Resolve-PipelineArtifactPath ` + -RepositoryRoot $repositoryRoot ` + -Path ([string]$manifest.validation_inventory_audit) + if (-not (Test-Path -LiteralPath $inventoryAuditPath -PathType Leaf)) { + throw "Validation manifest inventory audit is missing for preset $preset`: $inventoryAuditPath" + } + $inventoryAuditHash = ( + Get-FileHash -LiteralPath $inventoryAuditPath -Algorithm SHA256 + ).Hash.ToLowerInvariant() + if ($inventoryAuditHash -ne $manifest.validation_inventory_audit_sha256) { + throw "Validation manifest inventory audit changed for preset $preset`: $inventoryAuditPath" + } + $entries.Add([ordered]@{ + preset = $preset + path = [System.IO.Path]::GetRelativePath($repositoryRoot, $matches[0].FullName).Replace('\', '/') + sha256 = (Get-FileHash -LiteralPath $matches[0].FullName -Algorithm SHA256).Hash.ToLowerInvariant() + fingerprint = $manifest.fingerprint_sha256 + sourceDigest = $manifest.source_digest + aggregate = $manifest.aggregate + matrixCell = $manifest.matrix_cell + targetInventorySha256 = $manifest.target_inventory_sha256 + testInventorySha256 = $manifest.main_test_inventory_sha256 + matrixContractSha256 = $manifest.matrix_contract_sha256 + inventoryAuditSha256 = $manifest.validation_inventory_audit_sha256 + configuration = $manifest.build_profile + instrumentation = $manifest.instrumentation + generatedCodeMode = $manifest.codegen_mode + consumerScope = $manifest.consumer_scope + }) + } + $selectionText = "$Scope|$($SelectedCompilers -join ',')" + $selectionId = (Get-PipelineTextDigest -Text $selectionText).Substring(0, 16) + $receiptPath = Join-Path $pipelineRoot "provenance/build-$selectionId.json" + $document = [ordered]@{ + schema = 'simdlib.unified-build-receipt.v5'; status = 'complete'; scope = $Scope + compilers = @($SelectedCompilers); sourceDigest = $currentSourceDigest + sourceRevision = Get-PipelineRevision -RepositoryRoot $repositoryRoot + pipelineValidation = $pipelineValidationEntry + manifests = $entries.ToArray() + } + Set-PipelineTextFile -Path $receiptPath -Content ($document | ConvertTo-Json -Depth 6) + Set-PipelineTextFile -Path (Join-Path $pipelineRoot 'provenance/latest-build-receipt.txt') -Content ([System.IO.Path]::GetRelativePath($repositoryRoot, $receiptPath).Replace('\', '/')) + return $receiptPath +} + +$selectedCompilers = @(Resolve-BuildSelection) +if ($Scope -in @('All', 'Native') -and -not $IsWindows) { throw 'Native scope requires a Windows x64 host with Visual Studio C++ tools and LLVM 20 or newer.' } +$toolingDigest = Get-PipelineToolingDigest -RepositoryRoot $repositoryRoot +$pipelineValidationPath = Join-Path $pipelineRoot ( + "provenance/pipeline-validation-$($toolingDigest.Substring(0, 16)).json") +& (Join-Path $PSScriptRoot 'Validate-PipelineTooling.ps1') ` + -ResultPath $pipelineValidationPath +& (Join-Path $PSScriptRoot 'Test-PublicConsumerBoundary.ps1') + +$operations = [System.Collections.Generic.List[object]]::new() +foreach ($name in @($selectedCompilers | Where-Object { $_ -in (Get-PipelineValidationCompilers -Platform native) })) { + $operations.Add([pscustomobject]@{ + Id = "native-$($name.ToLowerInvariant())"; Script = Join-Path $PSScriptRoot 'Run-NativeMatrix.ps1' + Arguments = @('-Action', 'Build', '-Compiler', $name, '-Cell', 'All') + }) +} +$containerCompilers = @($selectedCompilers | Where-Object { $_ -in (Get-PipelineValidationCompilers -Platform container) }) +if ($containerCompilers.Count -eq 3) { + $operations.Add([pscustomobject]@{ Id = 'containers'; Script = Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1'; Arguments = @('-Action', 'Build', '-Compiler', 'All', '-Cell', 'All') }) +} else { + foreach ($name in $containerCompilers) { + $operations.Add([pscustomobject]@{ Id = "container-$($name.ToLowerInvariant())"; Script = Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1'; Arguments = @('-Action', 'Build', '-Compiler', $name, '-Cell', 'All') }) + } +} +$logDirectory = Join-Path $pipelineRoot "logs/$(Get-Date -Format 'yyyyMMdd-HHmmssfff')-build-$PID" +Invoke-PipelineChildOperations -Operations $operations.ToArray() -LogDirectory $logDirectory + +$receipt = Write-BuildReceipt -SelectedCompilers $selectedCompilers -PipelineValidationPath $pipelineValidationPath +Write-Host "Unified build passed. Receipt: $receipt" diff --git a/tools/Pipeline.Common.psm1 b/tools/Pipeline.Common.psm1 new file mode 100644 index 0000000..965c531 --- /dev/null +++ b/tools/Pipeline.Common.psm1 @@ -0,0 +1,603 @@ +Set-StrictMode -Version Latest + +$script:Utf8NoBom = [System.Text.UTF8Encoding]::new($false) + +<# +.SYNOPSIS +Returns the repository root owned by the pipeline tools. +#> +function Get-PipelineRepositoryRoot { + return Split-Path -Parent $PSScriptRoot +} + +<# +.SYNOPSIS +Reads the canonical validation matrix. +.PARAMETER RepositoryRoot +Absolute SimdLib source tree. +#> +function Get-PipelineValidationMatrix { + param([string]$RepositoryRoot = (Get-PipelineRepositoryRoot)) + + $path = Join-Path $RepositoryRoot 'tools/validation-matrix.json' + $matrix = Get-Content -LiteralPath $path -Raw | ConvertFrom-Json + if ($matrix.schema -ne 'simdlib.validation-matrix.v1') { + throw "Unsupported validation matrix schema in $path" + } + return $matrix +} + +<# +.SYNOPSIS +Returns compiler names in matrix-owned deterministic order for one platform. +.PARAMETER Platform +Validation runner platform. +#> +function Get-PipelineValidationCompilers { + param([Parameter(Mandatory)][ValidateSet('native', 'container')][string]$Platform) + + $matrix = Get-PipelineValidationMatrix + $available = @($matrix.cells.PSObject.Properties | + Where-Object { $_.Value.platform -eq $Platform } | + ForEach-Object { $_.Value.compiler } | Select-Object -Unique) + return @($matrix.compilerOrder | Where-Object { $_ -in $available }) +} +<# +.SYNOPSIS +Returns the ordered cells assigned to one canonical matrix operation. +.PARAMETER Operation +Operation name from the validation matrix. +.PARAMETER RepositoryRoot +Absolute SimdLib source tree. +#> +function Get-PipelineValidationOperationCells { + param( + [Parameter(Mandatory)][string]$Operation, + [string]$RepositoryRoot = (Get-PipelineRepositoryRoot) + ) + + $matrix = Get-PipelineValidationMatrix -RepositoryRoot $RepositoryRoot + $operationProperty = $matrix.operations.PSObject.Properties[$Operation] + if (-not $operationProperty) { + throw "Validation matrix does not define operation $Operation" + } + $seen = [System.Collections.Generic.HashSet[string]]::new( + [System.StringComparer]::Ordinal) + return @( + foreach ($cellId in @($operationProperty.Value)) { + if (-not $seen.Add([string]$cellId)) { + throw "Validation matrix operation $Operation duplicates cell $cellId" + } + $cellProperty = $matrix.cells.PSObject.Properties[[string]$cellId] + if (-not $cellProperty) { + throw "Validation matrix operation $Operation references unknown cell $cellId" + } + $cell = $cellProperty.Value.PSObject.Copy() + Add-Member -InputObject $cell -NotePropertyName MatrixCell ` + -NotePropertyValue ([string]$cellId) -Force + $cell + } + ) +} + +<# +.SYNOPSIS +Resolves runner-facing cells from canonical matrix operations and filters. +.PARAMETER Platform +Runner platform to select. +.PARAMETER CompilerNames +Canonical user-facing compiler names. +.PARAMETER CellScope +Requested configuration or instrumentation scope. +.PARAMETER Operation +Runner operation name. +#> +function Resolve-PipelineValidationCells { + param( + [Parameter(Mandatory)][ValidateSet('native', 'container')][string]$Platform, + [Parameter(Mandatory)][string[]]$CompilerNames, + [Parameter(Mandatory)][string]$CellScope, + [Parameter(Mandatory)][string]$Operation + ) + + $operationName = switch ($Operation) { + { $_ -in @('BuildCompilerContracts', 'TestCompilerContracts') } { 'compilerContracts'; break } + 'RecordCodegen' { 'optionalDiagnostics'; break } + { $_ -in @('BuildBenchmarks', 'RunBenchmarks') } { 'benchmarks'; break } + 'Test' { 'defaultTests'; break } + default { 'defaultBuild' } + } + $matrix = Get-PipelineValidationMatrix + $candidateIds = [System.Collections.Generic.List[string]]::new() + foreach ($name in @($operationName, 'optionalDebug', 'coverage', 'sanitizer')) { + $property = $matrix.operations.PSObject.Properties[$name] + if ($property) { + foreach ($cellId in @($property.Value)) { + if (-not $candidateIds.Contains([string]$cellId)) { + $candidateIds.Add([string]$cellId) + } + } + } + } + + $operationIds = @($matrix.operations.PSObject.Properties[$operationName].Value) + return @( + foreach ($cellId in $candidateIds) { + $cell = $matrix.cells.PSObject.Properties[$cellId].Value + if ($cell.platform -ne $Platform -or $cell.compiler -notin $CompilerNames) { + continue + } + $scopeMatches = switch ($CellScope) { + 'All' { $cellId -in $operationIds } + 'Release' { $cell.profile -in @('RELEASE', 'COMPILER_CONTRACTS') } + 'Debug' { + $cell.profile -in @('DEBUG', 'CODEGEN_DIAGNOSTIC') -and + $cell.instrumentation -eq 'none' + } + 'Coverage' { $cell.profile -eq 'COVERAGE' } + 'AsanUbsan' { $cell.instrumentation -eq 'asan-ubsan' } + default { $false } + } + if (-not $scopeMatches) { continue } + if ($Operation -in @('BuildBenchmarks', 'RunBenchmarks', + 'BuildCompilerContracts', 'TestCompilerContracts', + 'RecordCodegen') -and $cellId -notin $operationIds) { + continue + } + + $runnerCompiler = switch ($cell.compiler) { + 'Msvc' { 'msvc' } + 'ClangCl' { 'clangcl' } + 'ClangCoverage' { 'clang-coverage' } + default { ([string]$cell.compiler).ToLowerInvariant() } + } + $definition = [ordered]@{ + MatrixCell = [string]$cellId + Key = [string]$cell.artifactKey + Preset = [string]$cell.preset + BuildProfile = [string]$cell.configuration + Consumer = [bool]$cell.consumer + Coverage = $cell.instrumentation -eq 'coverage' + Instrumentation = [string]$cell.instrumentation + Sanitizer = if ($cell.instrumentation -eq 'asan-ubsan') { 'asan-ubsan' } else { 'none' } + CodegenMode = [string]$cell.codegenMode + Aggregate = [string]$cell.aggregate + } + if ($Platform -eq 'native') { + $definition.Compiler = $runnerCompiler + $definition.Generator = [string]$cell.generator + } else { + $definition.Service = $runnerCompiler + } + [pscustomobject]$definition + } + ) +} +<# +.SYNOPSIS +Returns the exact configure presets owned by the unified default validation matrix. +.PARAMETER SelectedCompilers +Canonical user-facing compiler names selected by the caller. +#> +function Get-PipelineDefaultValidationPresets { + param([Parameter(Mandatory)][string[]]$SelectedCompilers) + + return @( + Get-PipelineValidationOperationCells -Operation defaultBuild | + Where-Object compiler -in $SelectedCompilers | + ForEach-Object preset + ) +} + +<# +.SYNOPSIS +Reports whether a configure preset belongs to the unified default validation matrix. +.PARAMETER Preset +Configure preset name to classify. +#> +function Test-PipelineDefaultValidationPreset { + param([Parameter(Mandatory)][string]$Preset) + + $allDefaultPresets = Get-PipelineValidationOperationCells ` + -Operation defaultBuild | ForEach-Object preset + return $Preset -in $allDefaultPresets +} + +<# +.SYNOPSIS +Computes the canonical digest of source inputs that affect build artifacts. +.PARAMETER RepositoryRoot +Absolute path to the SimdLib source tree. +#> +function Get-PipelineSourceDigest { + param([Parameter(Mandatory)][string]$RepositoryRoot) + $root = [System.IO.Path]::GetFullPath($RepositoryRoot) + $files = [System.Collections.Generic.List[string]]::new() + foreach ($name in @('CMakeLists.txt', 'CMakePresets.json', 'compose.yml', '.clang-format')) { + $path = Join-Path $root $name + if (Test-Path -LiteralPath $path -PathType Leaf) { $files.Add($path) } + } + foreach ($directory in @('include', 'cmake', 'tests', 'examples', 'benchmarks', 'containers', 'tools')) { + $path = Join-Path $root $directory + if (Test-Path -LiteralPath $path -PathType Container) { + foreach ($file in Get-ChildItem -LiteralPath $path -File -Recurse) { $files.Add($file.FullName) } + } + } + $stream = [System.IO.MemoryStream]::new() + try { + $relativeFiles = @($files | ForEach-Object { + [System.IO.Path]::GetRelativePath($root, $_).Replace('\', '/') + }) + [Array]::Sort($relativeFiles, [System.StringComparer]::Ordinal) + foreach ($relative in $relativeFiles) { + $file = Join-Path $root $relative.Replace('/', [System.IO.Path]::DirectorySeparatorChar) + $relativeBytes = $script:Utf8NoBom.GetBytes($relative) + $stream.Write($relativeBytes, 0, $relativeBytes.Length) + $stream.WriteByte(0) + $hash = (Get-FileHash -LiteralPath $file -Algorithm SHA256).Hash.ToLowerInvariant() + $hashBytes = $script:Utf8NoBom.GetBytes($hash) + $stream.Write($hashBytes, 0, $hashBytes.Length) + $stream.WriteByte(10) + } + return [Convert]::ToHexString( + [System.Security.Cryptography.SHA256]::HashData($stream.ToArray())).ToLowerInvariant() + } finally { + $stream.Dispose() + } +} + +<# +.SYNOPSIS +Returns the reviewed files that own pipeline-tooling validation. +.PARAMETER RepositoryRoot +Absolute SimdLib source tree. +#> +function Get-PipelineToolingInputs { + param([Parameter(Mandatory)][string]$RepositoryRoot) + + $root = [System.IO.Path]::GetFullPath($RepositoryRoot) + $matrix = Get-PipelineValidationMatrix -RepositoryRoot $root + $classes = @($matrix.toolingValidation.inputClasses.PSObject.Properties) + if ($classes.Count -eq 0) { throw 'Validation matrix defines no tooling-input classes.' } + $owned = [System.Collections.Generic.List[object]]::new() + $relativeOwners = @{} + foreach ($class in $classes) { + foreach ($declaredPath in @($class.Value)) { + $path = Join-Path $root ([string]$declaredPath) + if (Test-Path -LiteralPath $path -PathType Container) { + $files = @(Get-ChildItem -LiteralPath $path -File -Recurse | Sort-Object FullName) + } elseif (Test-Path -LiteralPath $path -PathType Leaf) { + $files = @((Get-Item -LiteralPath $path)) + } else { + throw "Pipeline-tooling input is missing: $declaredPath" + } + foreach ($file in $files) { + $relative = [System.IO.Path]::GetRelativePath($root, $file.FullName).Replace('\', '/') + if ($relativeOwners.ContainsKey($relative)) { + throw "Pipeline-tooling input $relative belongs to both $($relativeOwners[$relative]) and $($class.Name)" + } + $relativeOwners[$relative] = $class.Name + $owned.Add([pscustomobject]@{ + Class = [string]$class.Name + RelativePath = $relative + FullName = $file.FullName + }) + } + } + } + return @($owned | Sort-Object Class, RelativePath) +} + +<# +.SYNOPSIS +Computes the digest of the reviewed pipeline-tooling input set. +.PARAMETER RepositoryRoot +Absolute SimdLib source tree. +#> +function Get-PipelineToolingDigest { + param([Parameter(Mandatory)][string]$RepositoryRoot) + + $stream = [System.IO.MemoryStream]::new() + try { + foreach ($input in Get-PipelineToolingInputs -RepositoryRoot $RepositoryRoot) { + $record = "$($input.Class)`0$($input.RelativePath)`0$((Get-FileHash -LiteralPath $input.FullName -Algorithm SHA256).Hash.ToLowerInvariant())`n" + $bytes = $script:Utf8NoBom.GetBytes($record) + $stream.Write($bytes, 0, $bytes.Length) + } + return [Convert]::ToHexString( + [System.Security.Cryptography.SHA256]::HashData($stream.ToArray())).ToLowerInvariant() + } finally { + $stream.Dispose() + } +} +<# +.SYNOPSIS +Computes the lowercase SHA-256 digest of a UTF-8 string. +.PARAMETER Text +Text to hash. +#> +function Get-PipelineTextDigest { + param([Parameter(Mandatory)][string]$Text) + $bytes = $script:Utf8NoBom.GetBytes($Text) + return [Convert]::ToHexString( + [System.Security.Cryptography.SHA256]::HashData($bytes)).ToLowerInvariant() +} + +<# +.SYNOPSIS +Writes UTF-8 text atomically. +.PARAMETER Path +Destination file. +.PARAMETER Content +Text to write. +#> +function Set-PipelineTextFile { + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][AllowEmptyString()][string]$Content + ) + $directory = Split-Path -Parent $Path + if ($directory) { New-Item -ItemType Directory -Path $directory -Force | Out-Null } + $temporary = "$Path.tmp-$PID" + [System.IO.File]::WriteAllText($temporary, $Content, $script:Utf8NoBom) + Move-Item -LiteralPath $temporary -Destination $Path -Force +} + +<# +.SYNOPSIS +Reads a key-value build manifest. +.PARAMETER Path +Manifest path. +#> +function Read-PipelineManifest { + param([Parameter(Mandatory)][string]$Path) + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "Required build manifest is missing: $Path" } + $values = @{} + foreach ($line in Get-Content -LiteralPath $Path) { + $separator = $line.IndexOf('=') + if ($separator -gt 0) { $values[$line.Substring(0, $separator)] = $line.Substring($separator + 1) } + } + return $values +} + +<# +.SYNOPSIS +Resolves a manifest artifact path into the host repository. +.PARAMETER RepositoryRoot +Absolute SimdLib source tree. +.PARAMETER Path +Host, repository-relative, or canonical `/workspace` container path. +#> +function Resolve-PipelineArtifactPath { + param( + [Parameter(Mandatory)][string]$RepositoryRoot, + [Parameter(Mandatory)][string]$Path + ) + + $root = [System.IO.Path]::GetFullPath($RepositoryRoot) + if (Test-Path -LiteralPath $Path) { + $candidate = $Path + } elseif ($Path -match '^/workspace/out/(?.+)$') { + $candidate = Join-Path ( + Join-Path $root 'out/pipeline') $Matches.relative + } elseif (-not [System.IO.Path]::IsPathRooted($Path)) { + $candidate = Join-Path $root $Path + } else { + throw "Manifest artifact path is not host-accessible: $Path" + } + $resolved = [System.IO.Path]::GetFullPath($candidate) + if (-not $resolved.StartsWith( + $root + [System.IO.Path]::DirectorySeparatorChar, + [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Manifest artifact path escapes the repository: $Path" + } + return $resolved +} + +<# +.SYNOPSIS +Creates the unified-receipt entry for completed pipeline-tooling validation. +.PARAMETER RepositoryRoot +Absolute SimdLib source tree. +.PARAMETER ResultPath +Machine-readable pipeline-tooling validation result. +.PARAMETER ExpectedToolingDigest +Canonical tooling digest the result must own. +#> +function New-PipelineValidationEntry { + param( + [Parameter(Mandatory)][string]$RepositoryRoot, + [Parameter(Mandatory)][string]$ResultPath, + [Parameter(Mandatory)][string]$ExpectedToolingDigest + ) + if (-not (Test-Path -LiteralPath $ResultPath -PathType Leaf)) { + throw "Pipeline-tooling validation result is missing: $ResultPath" + } + $result = Get-Content -LiteralPath $ResultPath -Raw | ConvertFrom-Json + if ($result.schema -ne 'simdlib.pipeline-tooling-validation.v1' -or + $result.status -ne 'complete' -or + $result.toolingDigest -ne $ExpectedToolingDigest) { + throw "Pipeline-tooling validation result is stale or incompatible: $ResultPath" + } + return [ordered]@{ + path = [System.IO.Path]::GetRelativePath($RepositoryRoot, $ResultPath).Replace('\', '/') + sha256 = (Get-FileHash -LiteralPath $ResultPath -Algorithm SHA256).Hash.ToLowerInvariant() + status = [string]$result.status + schema = [string]$result.schema + toolingDigest = [string]$result.toolingDigest + } +} + +<# +.SYNOPSIS +Validates the pipeline-tooling entry bound into a unified build receipt. +.PARAMETER RepositoryRoot +Absolute SimdLib source tree. +.PARAMETER Entry +Receipt entry containing result identity and tooling digest. +.PARAMETER ExpectedToolingDigest +Canonical tooling digest required by the consuming operation. +#> +function Assert-PipelineValidationEntry { + param( + [Parameter(Mandatory)][string]$RepositoryRoot, + [Parameter(Mandatory)][AllowNull()][object]$Entry, + [Parameter(Mandatory)][string]$ExpectedToolingDigest + ) + if (-not $Entry -or + $Entry.schema -ne 'simdlib.pipeline-tooling-validation.v1' -or + $Entry.status -ne 'complete' -or + $Entry.toolingDigest -ne $ExpectedToolingDigest) { + throw 'Unified build receipt does not contain current pipeline-tooling validation.' + } + $resultPath = Join-Path $RepositoryRoot ([string]$Entry.path) + if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) { + throw "Receipt pipeline-tooling validation is missing: $resultPath" + } + $resultHash = (Get-FileHash -LiteralPath $resultPath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($resultHash -ne $Entry.sha256) { + throw "Receipt pipeline-tooling validation changed after the unified build: $resultPath" + } + $result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json + if ($result.schema -ne $Entry.schema -or + $result.status -ne $Entry.status -or + $result.toolingDigest -ne $ExpectedToolingDigest) { + throw "Receipt pipeline-tooling validation is incomplete or stale: $resultPath" + } + return $resultPath +} +<# +.SYNOPSIS +Invokes a command, records its combined output, and preserves its exit code. +.PARAMETER FilePath +Executable to invoke. +.PARAMETER ArgumentList +Arguments passed without shell reinterpretation. +.PARAMETER LogPath +File that receives combined output. +#> +function Invoke-PipelineCommand { + param( + [Parameter(Mandatory)][string]$FilePath, + [Parameter(Mandatory)][string[]]$ArgumentList, + [Parameter(Mandatory)][string]$LogPath + ) + $directory = Split-Path -Parent $LogPath + if ($directory) { New-Item -ItemType Directory -Path $directory -Force | Out-Null } + & $FilePath @ArgumentList 2>&1 | Tee-Object -FilePath $LogPath + if ($LASTEXITCODE -ne 0) { throw "$FilePath failed with exit code $LASTEXITCODE. Log: $LogPath" } +} + +<# +.SYNOPSIS +Imports the installed Visual Studio x64 developer environment. +#> +function Initialize-PipelineVisualStudioEnvironment { + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' + if (-not (Test-Path -LiteralPath $vswhere)) { throw "Visual Studio locator is missing: $vswhere" } + $installation = (& $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath).Trim() + if ($LASTEXITCODE -ne 0 -or -not $installation) { throw 'A Visual Studio installation with the x64 C++ tools is required.' } + $developerCommand = Join-Path $installation 'Common7\Tools\VsDevCmd.bat' + $environmentLines = & cmd.exe /s /c "`"$developerCommand`" -no_logo -arch=x64 -host_arch=x64 && set" + if ($LASTEXITCODE -ne 0) { throw 'Unable to initialize the Visual Studio x64 developer environment.' } + foreach ($line in $environmentLines) { + $separator = $line.IndexOf('=') + if ($separator -gt 0) { [Environment]::SetEnvironmentVariable($line.Substring(0, $separator), $line.Substring($separator + 1), 'Process') } + } + return $installation +} + +<# +.SYNOPSIS +Returns the current Git revision or a stable unknown marker. +.PARAMETER RepositoryRoot +Absolute source-tree path. +#> +function Get-PipelineRevision { + param([Parameter(Mandatory)][string]$RepositoryRoot) + $revision = (& git -C $RepositoryRoot rev-parse HEAD 2>$null).Trim() + if ($LASTEXITCODE -ne 0 -or -not $revision) { return 'unknown' } + return $revision +} + +<# +.SYNOPSIS +Runs independent PowerShell pipeline operations concurrently and aggregates failures. +.PARAMETER Operations +Objects with Id, Script, and Arguments properties. +.PARAMETER LogDirectory +Invocation-owned directory for child stdout and stderr logs. +#> +function Invoke-PipelineChildOperations { + param( + [Parameter(Mandatory)][object[]]$Operations, + [Parameter(Mandatory)][string]$LogDirectory + ) + if ($Operations.Count -eq 0) { return } + New-Item -ItemType Directory -Path $LogDirectory -Force | Out-Null + $pwsh = (Get-Command pwsh -ErrorAction Stop).Source + $runs = [System.Collections.Generic.List[object]]::new() + try { + foreach ($operation in $Operations) { + $startInfo = [System.Diagnostics.ProcessStartInfo]::new() + $startInfo.FileName = $pwsh + $startInfo.UseShellExecute = $false + $startInfo.RedirectStandardOutput = $true + $startInfo.RedirectStandardError = $true + foreach ($argument in @('-NoProfile', '-File', $operation.Script) + @($operation.Arguments)) { + $startInfo.ArgumentList.Add([string]$argument) + } + $process = [System.Diagnostics.Process]::new() + $process.StartInfo = $startInfo + if (-not $process.Start()) { throw "Unable to start pipeline operation $($operation.Id)" } + $runs.Add([pscustomobject]@{ + Id = $operation.Id; Process = $process + StandardOutput = $process.StandardOutput.ReadToEndAsync() + StandardError = $process.StandardError.ReadToEndAsync() + }) + Write-Host "Started pipeline operation: $($operation.Id)" + } + $failures = [System.Collections.Generic.List[string]]::new() + foreach ($run in $runs) { + $run.Process.WaitForExit() + $stdout = $run.StandardOutput.GetAwaiter().GetResult() + $stderr = $run.StandardError.GetAwaiter().GetResult() + Set-PipelineTextFile -Path (Join-Path $LogDirectory "$($run.Id).stdout.log") -Content $stdout + Set-PipelineTextFile -Path (Join-Path $LogDirectory "$($run.Id).stderr.log") -Content $stderr + if ($stdout) { Write-Host $stdout.TrimEnd() } + if ($stderr) { [Console]::Error.WriteLine($stderr.TrimEnd()) } + if ($run.Process.ExitCode -ne 0) { $failures.Add("$($run.Id)=$($run.Process.ExitCode)") } + } + if ($failures.Count) { throw "Pipeline operations failed: $($failures -join ', '). Logs: $LogDirectory" } + } finally { + foreach ($run in $runs) { + if (-not $run.Process.HasExited) { + try { $run.Process.Kill($true); $run.Process.WaitForExit() } catch { Write-Warning "Unable to stop $($run.Id): $_" } + } + $run.Process.Dispose() + } + } +} + +Export-ModuleMember -Function @( + 'Get-PipelineRepositoryRoot', + 'Get-PipelineValidationMatrix', + 'Get-PipelineValidationCompilers', + 'Get-PipelineValidationOperationCells', + 'Resolve-PipelineValidationCells', + 'Get-PipelineDefaultValidationPresets', + 'Test-PipelineDefaultValidationPreset', + 'Get-PipelineSourceDigest', + 'Get-PipelineToolingInputs', + 'Get-PipelineToolingDigest', + 'Get-PipelineTextDigest', + 'Set-PipelineTextFile', + 'Read-PipelineManifest', + 'Resolve-PipelineArtifactPath', + 'New-PipelineValidationEntry', + 'Assert-PipelineValidationEntry', + 'Invoke-PipelineCommand', + 'Initialize-PipelineVisualStudioEnvironment', + 'Get-PipelineRevision', + 'Invoke-PipelineChildOperations' +) diff --git a/tools/Record-Codegen.ps1 b/tools/Record-Codegen.ps1 new file mode 100644 index 0000000..52a54d2 --- /dev/null +++ b/tools/Record-Codegen.ps1 @@ -0,0 +1,52 @@ +<# +.SYNOPSIS +Records one explicitly selected Register generated-code diagnostic. +.DESCRIPTION +The command configures a diagnostic-only fingerprint, compiles only paired +Register fixtures, records wrapper/raw disassembly, and writes dedicated +provenance. Its record-only output cannot satisfy a Release generated-code gate. +#> +[CmdletBinding()] +param( + [ValidateSet('', 'Native', 'Containers')] + [string]$Scope = '', + [ValidateSet('', 'Msvc', 'ClangCl', 'Gcc14', 'Clang22')] + [string]$Compiler = '', + [ValidateSet('', 'Debug', 'AsanUbsan')] + [string]$Cell = '', + [switch]$SkipImageBuild +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not $Scope -or -not $Compiler -or -not $Cell) { + throw 'Record-Codegen requires explicit -Scope, -Compiler, and -Cell selections.' +} +if ($Scope -eq 'Native') { + if ($Compiler -notin @('Msvc', 'ClangCl')) { + throw 'Native codegen diagnostics support Msvc or ClangCl.' + } + if ($Cell -ne 'Debug') { + throw 'Native codegen diagnostics support the Debug cell.' + } + if ($SkipImageBuild) { + throw '-SkipImageBuild is available only for container diagnostics.' + } + & (Join-Path $PSScriptRoot 'Run-NativeMatrix.ps1') ` + -Action RecordCodegen -Compiler $Compiler -Cell $Cell +} else { + if ($Compiler -notin @('Gcc14', 'Clang22')) { + throw 'Container codegen diagnostics support Gcc14 or Clang22.' + } + if ($Cell -eq 'AsanUbsan' -and $Compiler -ne 'Clang22') { + throw 'The sanitizer-instrumented codegen diagnostic is owned by Clang22.' + } + if ($SkipImageBuild) { + & (Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1') ` + -Action RecordCodegen -Compiler $Compiler -Cell $Cell -SkipImageBuild + } else { + & (Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1') ` + -Action RecordCodegen -Compiler $Compiler -Cell $Cell + } +} diff --git a/tools/Run-Benchmarks.ps1 b/tools/Run-Benchmarks.ps1 new file mode 100644 index 0000000..8626d8f --- /dev/null +++ b/tools/Run-Benchmarks.ps1 @@ -0,0 +1,64 @@ +<# +.SYNOPSIS +Runs benchmarks from completed benchmark-build manifests. +.DESCRIPTION +The command delegates only to benchmark execution operations. Those operations +reject missing, stale, or incompatible manifests and never configure or build. +#> +[CmdletBinding()] +param( + [ValidateSet('All', 'Native', 'Containers')] + [string]$Scope = 'All', + [ValidateSet('All', 'Msvc', 'ClangCl', 'ClangCoverage', 'Gcc13', 'Gcc14', 'Clang22')] + [string[]]$Compiler = @('All') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force + +<# +.SYNOPSIS +Expands benchmark execution filters into native and container owners. +#> +function Resolve-BenchmarkExecutionSelection { + $nativeNames = @(Get-PipelineValidationCompilers -Platform native) + $containerNames = @(Get-PipelineValidationCompilers -Platform container) + $benchmarkCells = @(Get-PipelineValidationOperationCells -Operation benchmarks) + $nativeBenchmarkOwners = @($benchmarkCells | + Where-Object platform -eq native | ForEach-Object compiler) + if ('All' -in $Compiler -and $Compiler.Count -ne 1) { throw 'Compiler All cannot be combined with another compiler filter.' } + $selected = if ($Compiler -contains 'All') { + switch ($Scope) { + 'Native' { $nativeNames } + 'Containers' { $containerNames } + default { $nativeNames + $containerNames } + } + } else { @($Compiler | Select-Object -Unique) } + if ($Scope -eq 'Native' -and @($selected | Where-Object { $_ -in $containerNames }).Count) { throw 'Container compiler filters are invalid for Native scope.' } + if ($Scope -eq 'Containers' -and @($selected | Where-Object { $_ -in $nativeNames }).Count) { throw 'Native compiler filters are invalid for Containers scope.' } + [pscustomobject]@{ + Native = if ($Scope -in @('All', 'Native')) { @($selected | Where-Object { $_ -in $nativeBenchmarkOwners }) } else { @() } + Containers = if ($Scope -in @('All', 'Containers')) { @($selected | Where-Object { $_ -in $containerNames }) } else { @() } + } +} + +$selection = Resolve-BenchmarkExecutionSelection +$operations = [System.Collections.Generic.List[object]]::new() +$nativeSelection = @($selection.Native) +$containerSelection = @($selection.Containers) +if ($nativeSelection.Count) { + $nativeFilter = if ($nativeSelection.Count -eq 2) { 'All' } else { $nativeSelection[0] } + $operations.Add([pscustomobject]@{ Id = 'native-benchmarks'; Script = Join-Path $PSScriptRoot 'Run-NativeMatrix.ps1'; Arguments = @('-Action', 'RunBenchmarks', '-Compiler', $nativeFilter, '-Cell', 'Release') }) +} +if ($containerSelection.Count -eq 3) { + $operations.Add([pscustomobject]@{ Id = 'container-benchmarks'; Script = Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1'; Arguments = @('-Action', 'RunBenchmarks', '-Compiler', 'All', '-Cell', 'Release') }) +} else { + foreach ($name in $containerSelection) { + $operations.Add([pscustomobject]@{ Id = "container-$($name.ToLowerInvariant())-benchmarks"; Script = Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1'; Arguments = @('-Action', 'RunBenchmarks', '-Compiler', $name, '-Cell', 'Release') }) + } +} +if (-not $operations.Count) { Write-Host 'No selected compiler owns benchmark execution.'; exit 0 } +$logDirectory = Join-Path (Get-PipelineRepositoryRoot) "out/pipeline/logs/$(Get-Date -Format 'yyyyMMdd-HHmmssfff')-run-benchmarks-$PID" +Invoke-PipelineChildOperations -Operations $operations.ToArray() -LogDirectory $logDirectory +Write-Host "Benchmarks passed. Logs: $logDirectory" diff --git a/tools/Run-ContainerMatrix.ps1 b/tools/Run-ContainerMatrix.ps1 new file mode 100644 index 0000000..36ba181 --- /dev/null +++ b/tools/Run-ContainerMatrix.ps1 @@ -0,0 +1,548 @@ +<# +.SYNOPSIS +Builds or consumes fingerprinted Linux compiler cells. +.DESCRIPTION +Each invocation owns one action. Build creates all selected validation +artifacts, Test consumes them without compilation, benchmark actions share the +Release trees, RecordCodegen creates an isolated diagnostic fingerprint, and +InspectEnvironment performs no project build. +#> +[CmdletBinding()] +param( + [ValidateSet('Build', 'Test', 'BuildCompilerContracts', 'TestCompilerContracts', 'RecordCodegen', 'BuildBenchmarks', 'RunBenchmarks', 'InspectEnvironment', 'Clean')] + [string]$Action = 'Build', + [ValidateSet('All', 'Release', 'Debug', 'AsanUbsan')] + [string]$Cell = 'All', + [ValidateSet('All', 'Gcc13', 'Gcc14', 'Clang22')] + [string]$Compiler = 'All', + [ValidateRange(1, 32)] + [int]$MaxParallel = 3, + [switch]$SkipImageBuild, + [switch]$NoImageCache, + [string]$TestRegex = '', + [string]$TestLabel = '', + [string[]]$InjectFailure = @('None'), + [ValidateRange(0, 86400)] + [int]$CancelAfterSeconds = 0 +) + +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force +$repositoryRoot = Split-Path -Parent $PSScriptRoot +$composeFile = Join-Path $repositoryRoot 'compose.yml' +$pipelineRoot = Join-Path $repositoryRoot 'out/pipeline' +$utf8NoBom = [System.Text.UTF8Encoding]::new($false) + +if (-not (Get-Command docker -CommandType Application -ErrorAction SilentlyContinue)) { + throw 'Docker CLI is required to run the container compiler matrix, but docker was not found on PATH.' +} + +if (-not $env:SIMDLIB_BUILD_REVISION) { + $env:SIMDLIB_BUILD_REVISION = (& git -C $repositoryRoot rev-parse HEAD).Trim() + if ($LASTEXITCODE -ne 0) { + throw 'Unable to determine the SimdLib revision for operation provenance.' + } +} +if ($IsLinux -or $IsMacOS) { + $env:SIMDLIB_HOST_UID = (& id -u).Trim() + $env:SIMDLIB_HOST_GID = (& id -g).Trim() +} + +<# +.SYNOPSIS +Invokes Docker and rejects a nonzero exit code. +.PARAMETER Arguments +Arguments passed directly to Docker. +#> +function Invoke-DockerChecked { + param([Parameter(Mandatory)][string[]]$Arguments) + & docker @Arguments + if ($LASTEXITCODE -ne 0) { + throw "docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" + } +} + +<# +.SYNOPSIS +Rejects Docker Compose versions that cannot control build provenance. +#> +function Assert-DockerComposeBuildVersion { + $versionText = [string](& docker compose version 2>&1) + if ($LASTEXITCODE -ne 0) { + throw "Unable to determine the Docker Compose version: $versionText" + } + if ($versionText -notmatch '\bv?(?\d+\.\d+\.\d+)\b') { + throw "Unable to parse the Docker Compose version from: $versionText" + } + $minimumVersion = [version]'2.39.0' + $selectedVersion = [version]$Matches.version + if ($selectedVersion -lt $minimumVersion) { + throw "Docker Compose $minimumVersion or newer is required to disable build provenance, but PATH selected $selectedVersion." + } + Write-Host "Docker Compose version: $selectedVersion" +} + +<# +.SYNOPSIS +Returns the selected compiler service names. +.PARAMETER CompilerName +User-facing compiler selection. +#> +function Resolve-Services { + param([Parameter(Mandatory)][string]$CompilerName) + switch ($CompilerName) { + 'Gcc13' { @('gcc13') } + 'Gcc14' { @('gcc14') } + 'Clang22' { @('clang22') } + default { @(Get-PipelineValidationCompilers -Platform container | + ForEach-Object { $_.ToLowerInvariant() }) } + } +} + +<# +.SYNOPSIS +Returns every build cell owned by the selected compilers and scope. +.PARAMETER Services +Selected Compose services. +.PARAMETER CellScope +Requested configuration scope. +.PARAMETER Operation +Requested pipeline operation. +#> +function Resolve-Cells { + param( + [Parameter(Mandatory)][string[]]$Services, + [Parameter(Mandatory)][string]$CellScope, + [Parameter(Mandatory)][string]$Operation + ) + + $compilerNames = @($Services | ForEach-Object { + switch ($_) { + 'gcc13' { 'Gcc13' } + 'gcc14' { 'Gcc14' } + 'clang22' { 'Clang22' } + default { throw "Unknown container compiler service: $_" } + } + }) + return @(Resolve-PipelineValidationCells -Platform container ` + -CompilerNames $compilerNames -CellScope $CellScope -Operation $Operation) +} +<# +.SYNOPSIS +Returns the canonical validation-matrix cell identifier for one container cell. +.PARAMETER BuildCell +Resolved container cell definition. +#> +function Get-ContainerValidationCellId { + param([Parameter(Mandatory)]$BuildCell) + + return [string]$BuildCell.MatrixCell +} + +<# +.SYNOPSIS +Reads immutable identity and labels from one local compiler image. +.PARAMETER Service +Compose service whose image is inspected. +#> +function Get-ImageMetadata { + param([Parameter(Mandatory)][string]$Service) + $imageName = "simdlib/${Service}:local" + $raw = & docker image inspect $imageName + if ($LASTEXITCODE -ne 0) { + throw "Unable to inspect required image $imageName. Build it first." + } + $inspection = ($raw | ConvertFrom-Json)[0] + $stableLabels = [ordered]@{} + foreach ($property in @($inspection.Config.Labels.PSObject.Properties | Sort-Object Name)) { + if ($property.Name -notlike 'com.docker.compose.*') { + $stableLabels[$property.Name] = $property.Value + } + } + $contentDocument = [ordered]@{ + architecture = $inspection.Architecture + os = $inspection.Os + layers = @($inspection.RootFS.Layers) + config = [ordered]@{ + user = $inspection.Config.User + environment = @($inspection.Config.Env) + entrypoint = @($inspection.Config.Entrypoint) + command = @($inspection.Config.Cmd) + workingDirectory = $inspection.Config.WorkingDir + labels = $stableLabels + } + } + $contentJson = $contentDocument | ConvertTo-Json -Depth 8 -Compress + $contentBytes = $utf8NoBom.GetBytes($contentJson) + $contentIdentity = [Convert]::ToHexString( + [System.Security.Cryptography.SHA256]::HashData($contentBytes)).ToLowerInvariant() + [pscustomobject]@{ + Name = $imageName + Id = $inspection.Id + ContentIdentity = "sha256:$contentIdentity" + BaseDigest = $inspection.Config.Labels.'org.simdlib.base.digest' + ToolchainVersion = $inspection.Config.Labels.'org.opencontainers.image.version' + CMakeSha256 = $inspection.Config.Labels.'org.simdlib.cmake.sha256' + Catch2Commit = $inspection.Config.Labels.'org.simdlib.catch2.commit' + } +} + +<# +.SYNOPSIS +Creates the canonical fingerprint document for one build cell. +.PARAMETER BuildCell +Compiler/configuration cell being identified. +.PARAMETER Image +Immutable local image metadata. +#> +function New-FingerprintDocument { + param([Parameter(Mandatory)]$BuildCell, [Parameter(Mandatory)]$Image) + $requiredFlags = if ($BuildCell.Service -eq 'clang22') { + [ordered]@{ cxx = '-stdlib=libc++'; linker = '-fuse-ld=lld --rtlib=compiler-rt --unwindlib=libunwind' } + } else { + [ordered]@{ cxx = ''; linker = '' } + } + [ordered]@{ + schema = 'simdlib.build-cell-fingerprint.v1' + platform = 'linux-x64' + compiler = $BuildCell.Service + image = [ordered]@{ identity = $Image.ContentIdentity; name = $Image.Name; baseDigest = $Image.BaseDigest; toolchainVersion = $Image.ToolchainVersion } + configuration = [ordered]@{ + key = $BuildCell.Key + preset = $BuildCell.Preset + buildProfile = $BuildCell.BuildProfile + sanitizer = $BuildCell.Sanitizer + instrumentation = $BuildCell.Instrumentation + codegenMode = $BuildCell.CodegenMode + aggregate = $BuildCell.Aggregate + consumerScope = if ($BuildCell.Consumer) { 'compiler-release' } else { 'none' } + generator = 'Ninja' + cxxStandard = 20 + cxxFlags = $requiredFlags.cxx + linkerFlags = $requiredFlags.linker + } + dependencies = [ordered]@{ cmakeVersion = '4.4.0'; cmakeSha256 = $Image.CMakeSha256; catch2Commit = $Image.Catch2Commit } + requiredCpuFeatures = @('sse4_2', 'avx2', 'fma', 'bmi1', 'bmi2') + } +} + +<# +.SYNOPSIS +Materializes and returns the fingerprinted artifact location for one cell. +.PARAMETER BuildCell +Compiler/configuration cell being located. +.PARAMETER Image +Immutable local image metadata. +#> +function Initialize-CellArtifact { + param([Parameter(Mandatory)]$BuildCell, [Parameter(Mandatory)]$Image) + $fingerprint = New-FingerprintDocument -BuildCell $BuildCell -Image $Image + $json = $fingerprint | ConvertTo-Json -Depth 8 -Compress + $bytes = $utf8NoBom.GetBytes($json) + $digest = [Convert]::ToHexString([System.Security.Cryptography.SHA256]::HashData($bytes)).ToLowerInvariant() + $compilerDirectoryName = "linux-$($BuildCell.Service)" + $cellDirectoryName = "$($BuildCell.Key)-$($digest.Substring(0, 16))" + $hostRoot = Join-Path $pipelineRoot "$compilerDirectoryName/$cellDirectoryName" + $provenanceDirectory = Join-Path $hostRoot 'provenance' + New-Item -ItemType Directory -Path $provenanceDirectory -Force | Out-Null + [System.IO.File]::WriteAllText((Join-Path $provenanceDirectory 'fingerprint.json'), $json, $utf8NoBom) + [pscustomobject]@{ + Service = $BuildCell.Service + Key = $BuildCell.Key + Id = "$($BuildCell.Service)-$($BuildCell.Key)" + Preset = $BuildCell.Preset + BuildProfile = $BuildCell.BuildProfile + Sanitizer = $BuildCell.Sanitizer + Instrumentation = $BuildCell.Instrumentation + CodegenMode = $BuildCell.CodegenMode + Aggregate = $BuildCell.Aggregate + MatrixCell = Get-ContainerValidationCellId -BuildCell $BuildCell + Consumer = $BuildCell.Consumer + Fingerprint = $digest + HostRoot = $hostRoot + ContainerRoot = "/workspace/out/$compilerDirectoryName/$cellDirectoryName" + } +} + +<# +.SYNOPSIS +Starts one isolated Compose operation with independent output logs. +.PARAMETER CellArtifact +Resolved fingerprinted cell artifact. +.PARAMETER Operation +Entrypoint operation to execute. +.PARAMETER ProjectName +Unique Compose project for this invocation. +.PARAMETER LogDirectory +Invocation-owned log directory. +.PARAMETER FailIntentionally +Whether this operation is an aggregate-failure probe. +#> +function Start-CellOperation { + param( + [Parameter(Mandatory)]$CellArtifact, + [Parameter(Mandatory)][string]$Operation, + [Parameter(Mandatory)][string]$ProjectName, + [Parameter(Mandatory)][string]$LogDirectory, + [Parameter(Mandatory)][bool]$FailIntentionally + ) + $arguments = [System.Collections.Generic.List[string]]::new() + foreach ($argument in @('compose', '--file', $composeFile, '--project-name', $ProjectName, '--profile', 'compilers', 'run', '--rm', '--no-deps')) { + $arguments.Add($argument) + } + if ($FailIntentionally) { + foreach ($argument in @('--entrypoint', '/bin/sh', $CellArtifact.Service, '-c', 'echo SIMDLIB_INTENTIONAL_MATRIX_FAILURE >&2; exit 23')) { + $arguments.Add($argument) + } + } else { + $arguments.Add($CellArtifact.Service) + foreach ($argument in @( + '--operation', $Operation, + '--preset', $CellArtifact.Preset, + '--build-profile', $CellArtifact.BuildProfile, + '--sanitizer', $CellArtifact.Sanitizer, + '--instrumentation', $CellArtifact.Instrumentation, + '--codegen-mode', $CellArtifact.CodegenMode, + '--aggregate', $CellArtifact.Aggregate, + '--matrix-cell', $CellArtifact.MatrixCell, + '--consumer-scope', $(if ($CellArtifact.Consumer) { 'compiler-release' } else { 'none' }), + '--artifact-root', $CellArtifact.ContainerRoot, + '--fingerprint-sha256', $CellArtifact.Fingerprint + )) { + $arguments.Add($argument) + } + if ($Operation -eq 'test' -and $TestRegex) { + $arguments.Add('--test-regex'); $arguments.Add($TestRegex) + } + if ($Operation -eq 'test' -and $TestLabel) { + $arguments.Add('--test-label'); $arguments.Add($TestLabel) + } + } + $processInfo = [System.Diagnostics.ProcessStartInfo]::new() + $processInfo.FileName = 'docker' + $processInfo.UseShellExecute = $false + $processInfo.RedirectStandardOutput = $true + $processInfo.RedirectStandardError = $true + foreach ($argument in $arguments) { $processInfo.ArgumentList.Add($argument) } + $process = [System.Diagnostics.Process]::new() + $process.StartInfo = $processInfo + if (-not $process.Start()) { throw "Failed to start $($CellArtifact.Id) operation $Operation" } + [pscustomobject]@{ + Cell = $CellArtifact + Operation = $Operation + Process = $process + StandardOutput = $process.StandardOutput.ReadToEndAsync() + StandardError = $process.StandardError.ReadToEndAsync() + StandardOutputPath = Join-Path $LogDirectory "$($CellArtifact.Id).$Operation.stdout.log" + StandardErrorPath = Join-Path $LogDirectory "$($CellArtifact.Id).$Operation.stderr.log" + Captured = $false + } +} + +<# +.SYNOPSIS +Completes one child process, writes its logs, and returns its exit code. +.PARAMETER Run +Running cell operation to complete. +#> +function Complete-CellOperation { + param([Parameter(Mandatory)]$Run) + if (-not $Run.Process.HasExited) { $Run.Process.WaitForExit() } + if (-not $Run.Captured) { + [System.IO.File]::WriteAllText($Run.StandardOutputPath, $Run.StandardOutput.GetAwaiter().GetResult(), $utf8NoBom) + [System.IO.File]::WriteAllText($Run.StandardErrorPath, $Run.StandardError.GetAwaiter().GetResult(), $utf8NoBom) + $Run.Captured = $true + } + return $Run.Process.ExitCode +} + +<# +.SYNOPSIS +Runs cell operations with bounded concurrency and aggregate failure reporting. +.PARAMETER CellArtifacts +Resolved cells to execute. +.PARAMETER Operation +Entrypoint operation shared by the cells. +.PARAMETER ProjectName +Unique Compose project for this invocation. +.PARAMETER LogDirectory +Invocation-owned log directory. +#> +function Invoke-CellOperations { + param( + [Parameter(Mandatory)][object[]]$CellArtifacts, + [Parameter(Mandatory)][string]$Operation, + [Parameter(Mandatory)][string]$ProjectName, + [Parameter(Mandatory)][string]$LogDirectory + ) + $pending = [System.Collections.Generic.Queue[object]]::new() + foreach ($cellArtifact in $CellArtifacts) { $pending.Enqueue($cellArtifact) } + $running = [System.Collections.Generic.List[object]]::new() + $allRuns = [System.Collections.Generic.List[object]]::new() + $failed = [System.Collections.Generic.List[string]]::new() + $deadline = if ($CancelAfterSeconds -gt 0) { (Get-Date).AddSeconds($CancelAfterSeconds) } else { $null } + $cancelled = $false + try { + while ($pending.Count -gt 0 -or $running.Count -gt 0) { + while ($pending.Count -gt 0 -and $running.Count -lt $MaxParallel) { + $cellArtifact = $pending.Dequeue() + $fail = $InjectFailure -contains 'All' -or $InjectFailure -contains $cellArtifact.Id + $run = Start-CellOperation -CellArtifact $cellArtifact -Operation $Operation -ProjectName $ProjectName -LogDirectory $LogDirectory -FailIntentionally $fail + $running.Add($run); $allRuns.Add($run) + Write-Host "Started $($cellArtifact.Id) operation=$Operation" + } + if ($deadline -and (Get-Date) -ge $deadline) { $cancelled = $true; break } + $completed = @($running | Where-Object { $_.Process.HasExited }) + if ($completed.Count -eq 0) { Start-Sleep -Milliseconds 100; continue } + foreach ($run in $completed) { + $exitCode = Complete-CellOperation -Run $run + [void]$running.Remove($run) + Write-Host "$($run.Cell.Id): operation=$Operation exit=$exitCode logs=$LogDirectory" + if ($exitCode -ne 0) { $failed.Add($run.Cell.Id) } + } + } + } finally { + if ($cancelled) { + foreach ($run in $running) { + try { $run.Process.Kill($true); $run.Process.WaitForExit() } + catch { Write-Warning "Process cancellation failed for $($run.Cell.Id): $_" } + } + } + foreach ($run in $allRuns) { + try { [void](Complete-CellOperation -Run $run) } + catch { Write-Warning "Log capture failed for $($run.Cell.Id): $_" } + $run.Process.Dispose() + } + } + if ($cancelled) { + throw [System.OperationCanceledException]::new("Container operation cancelled after $CancelAfterSeconds seconds. Logs: $LogDirectory") + } + if ($failed.Count -ne 0) { + throw "Container operation $Operation failed: $($failed -join ', '). Logs: $LogDirectory" + } +} + +<# +.SYNOPSIS +Removes selected pipeline artifacts, images, and abandoned Compose resources. +.PARAMETER Services +Compiler services selected for cleanup. +#> +function Remove-PipelineState { + param([Parameter(Mandatory)][string[]]$Services) + $resolvedPipelineRoot = [System.IO.Path]::GetFullPath($pipelineRoot) + $resolvedRepositoryRoot = [System.IO.Path]::GetFullPath($repositoryRoot) + if (-not $resolvedPipelineRoot.StartsWith($resolvedRepositoryRoot + [System.IO.Path]::DirectorySeparatorChar, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Refusing to clean outside the repository: $resolvedPipelineRoot" + } + $containerIds = @(& docker ps --all --quiet --filter 'name=simdlib-container-') + if ($LASTEXITCODE -ne 0) { throw 'Unable to enumerate SimdLib containers for cleanup.' } + if ($containerIds.Count -ne 0) { Invoke-DockerChecked (@('container', 'rm', '--force') + $containerIds) } + $networkIds = @(& docker network ls --quiet --filter 'name=simdlib-container-') + if ($LASTEXITCODE -ne 0) { throw 'Unable to enumerate SimdLib networks for cleanup.' } + if ($networkIds.Count -ne 0) { Invoke-DockerChecked (@('network', 'rm') + $networkIds) } + foreach ($service in $Services) { + $compilerRoot = [System.IO.Path]::GetFullPath((Join-Path $pipelineRoot "linux-$service")) + if (-not $compilerRoot.StartsWith($resolvedPipelineRoot + [System.IO.Path]::DirectorySeparatorChar, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Refusing to clean unexpected compiler artifacts: $compilerRoot" + } + if (Test-Path -LiteralPath $compilerRoot) { Remove-Item -LiteralPath $compilerRoot -Recurse -Force } + $image = "simdlib/${service}:local" + & docker image inspect $image *> $null + if ($LASTEXITCODE -eq 0) { Invoke-DockerChecked @('image', 'rm', $image) } + } + if ($Compiler -eq 'All') { + $logsRoot = Join-Path $pipelineRoot 'logs' + if (Test-Path -LiteralPath $logsRoot) { Remove-Item -LiteralPath $logsRoot -Recurse -Force } + } + Write-Host 'Removed selected pipeline artifacts, images, containers, and networks.' +} + +$services = @(Resolve-Services -CompilerName $Compiler) +if ($Action -eq 'Clean') { + Remove-PipelineState -Services $services + exit 0 +} +if ($NoImageCache -and ($SkipImageBuild -or $Action -notin @('Build', 'BuildCompilerContracts', 'RecordCodegen', 'InspectEnvironment'))) { + throw '-NoImageCache is only valid when Build, RecordCodegen, or InspectEnvironment owns the image build.' +} +if ($SkipImageBuild -and $Action -notin @('Build', 'BuildCompilerContracts', 'RecordCodegen', 'InspectEnvironment')) { + throw '-SkipImageBuild is only valid for Build, RecordCodegen, or InspectEnvironment.' +} +if (($TestRegex -or $TestLabel) -and $Action -notin @('Test', 'TestCompilerContracts')) { + throw '-TestRegex and -TestLabel are valid only for Test and TestCompilerContracts.' +} +if ($Cell -eq 'AsanUbsan' -and 'clang22' -notin $services) { + throw 'The ASan+UBSan cell is owned by Clang 22.' +} +if ($Action -in @('BuildCompilerContracts', 'TestCompilerContracts') -and $Cell -notin @('All', 'Release')) { + throw 'Focused compiler-contract operations use Release compiler identities.' +} +if ($Action -eq 'RecordCodegen') { + if ($Cell -notin @('All', 'Debug', 'AsanUbsan')) { + throw 'Container codegen diagnostics use Debug or AsanUbsan cells only.' + } + if ($Compiler -eq 'Gcc13') { + throw 'GCC 13 is core-only and owns no Register codegen diagnostic.' + } +} + +$selectedCellScope = if ($Action -in @('BuildCompilerContracts', 'TestCompilerContracts')) { + 'Release' +} elseif ($Action -eq 'InspectEnvironment') { + if ($Cell -notin @('All', 'Release')) { throw 'Environment inspection is compiler-scoped and uses one Release identity per compiler.' } + 'Release' +} elseif ($Action -in @('BuildBenchmarks', 'RunBenchmarks')) { + if ($Cell -notin @('All', 'Release')) { throw 'Benchmark operations only use Release cells.' } + 'Release' +} else { + $Cell +} +$cells = @(Resolve-Cells -Services $services -CellScope $selectedCellScope -Operation $Action) +if ($cells.Count -eq 0) { throw 'The compiler and cell selections do not identify any operation cells.' } + +$runId = "{0}-{1}-{2}" -f (Get-Date -Format 'yyyyMMdd-HHmmssfff'), $Action.ToLowerInvariant(), $PID +$projectName = "simdlib-container-$runId".ToLowerInvariant() +$imageBuildProjectName = 'simdlib-container-images' +$logDirectory = Join-Path $pipelineRoot "logs/$runId" +New-Item -ItemType Directory -Path $logDirectory -Force | Out-Null +Write-Host "Container operation: action=$Action cells=$($cells.Count) maxParallel=$MaxParallel" + +if ($Action -in @('Build', 'BuildCompilerContracts', 'RecordCodegen', 'InspectEnvironment') -and -not $SkipImageBuild) { + Assert-DockerComposeBuildVersion + $buildArguments = @( + 'compose', '--file', $composeFile, '--project-name', $imageBuildProjectName, + '--profile', 'compilers', 'build', '--provenance=false' + ) + if ($NoImageCache) { $buildArguments += '--no-cache' } + $buildArguments += $services + Invoke-DockerChecked $buildArguments +} + +$imageMetadata = @{} +foreach ($service in $services) { + $imageMetadata[$service] = Get-ImageMetadata -Service $service + Write-Host "$service image: config=$($imageMetadata[$service].Id) content=$($imageMetadata[$service].ContentIdentity)" +} +$cellArtifacts = @( + foreach ($cellDefinition in $cells) { + Initialize-CellArtifact -BuildCell $cellDefinition -Image $imageMetadata[$cellDefinition.Service] + } +) +$operation = switch ($Action) { + 'Build' { 'build-validation' } + 'BuildCompilerContracts' { 'build-validation' } + 'TestCompilerContracts' { 'test-compiler-contracts' } + 'Test' { 'test' } + 'RecordCodegen' { 'record-codegen' } + 'BuildBenchmarks' { 'build-benchmarks' } + 'RunBenchmarks' { 'run-benchmarks' } + 'InspectEnvironment' { 'inspect-environment' } +} +try { + Invoke-CellOperations -CellArtifacts $cellArtifacts -Operation $operation -ProjectName $projectName -LogDirectory $logDirectory +} finally { + & docker compose --file $composeFile --project-name $projectName --profile compilers down --remove-orphans 2>$null | Out-Null + if ($LASTEXITCODE -ne 0) { Write-Warning "Compose cleanup failed for project $projectName." } +} +Write-Host "Container operation passed. Logs: $logDirectory" diff --git a/tools/Run-NativeMatrix.ps1 b/tools/Run-NativeMatrix.ps1 new file mode 100644 index 0000000..6dbe5b8 --- /dev/null +++ b/tools/Run-NativeMatrix.ps1 @@ -0,0 +1,842 @@ +<# +.SYNOPSIS +Builds or consumes fingerprinted native compiler cells. +.DESCRIPTION +Build creates validation artifacts and manifests. Test validates those manifests +and runs CTest without configuring or building. RecordCodegen creates an +independent record-only diagnostic fingerprint. Benchmark operations reuse only +the existing Release trees. Coverage is an independent Clang Debug cell. +#> +[CmdletBinding()] +param( + [ValidateSet('Build', 'Test', 'BuildCompilerContracts', 'TestCompilerContracts', 'RecordCodegen', 'BuildBenchmarks', 'RunBenchmarks')] + [string]$Action = 'Build', + [ValidateSet('All', 'Release', 'Debug', 'Coverage')] + [string]$Cell = 'All', + [ValidateSet('All', 'Msvc', 'ClangCl', 'ClangCoverage')] + [string]$Compiler = 'All', + [string]$TestRegex = '', + [string]$TestLabel = '', + [string[]]$InjectFailure = @('None') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force + +<# +.SYNOPSIS +Resolves and validates the Clang commands selected by the caller's PATH. +.PARAMETER CompilerName +Requested native compiler scope. +#> +function Resolve-RequestedClangCommands { + param([Parameter(Mandatory)][string]$CompilerName) + + $commandNames = @( + if ($CompilerName -in @('All', 'ClangCl')) { 'clang-cl.exe' } + if ($CompilerName -in @('All', 'ClangCoverage')) { 'clang++.exe' } + ) + $commands = [ordered]@{} + if ($commandNames.Count -eq 0) { return ,$commands } + + foreach ($commandName in $commandNames) { + $command = @(Get-Command $commandName -CommandType Application -ErrorAction Stop)[0] + $versionLine = [string](@(& $command.Source --version 2>&1)[0]) + if ($versionLine -notmatch '\bclang version (?\d+)(?:\.\d+)*') { + throw "Unable to determine the Clang version selected for $commandName at $($command.Source): $versionLine" + } + if ([int]$Matches.major -lt 20) { + throw "Clang 20 or newer is required for $commandName, but PATH selected $versionLine at $($command.Source)." + } + $commands[$commandName] = [pscustomobject]@{ + Name = $commandName + Source = $command.Source + Directory = Split-Path -Parent $command.Source + Version = $versionLine.Trim() + } + } + + $directories = @($commands.Values.Directory | Select-Object -Unique) + if ($directories.Count -gt 1) { + throw "clang-cl and clang++ must come from one LLVM installation, but PATH selected: $($directories -join ', ')" + } + return $commands +} + +<# +.SYNOPSIS +Restores the caller-selected LLVM directory after Visual Studio environment setup. +.PARAMETER Commands +Validated Clang commands captured before Visual Studio initialization. +#> +function Restore-RequestedClangCommands { + param([Parameter(Mandatory)][System.Collections.IDictionary]$Commands) + + if ($Commands.Count -eq 0) { return } + $selectedDirectory = [string]@($Commands.Values.Directory)[0] + $pathSeparator = [System.IO.Path]::PathSeparator + $remainingEntries = @($env:PATH -split [regex]::Escape([string]$pathSeparator) | Where-Object { + $_ -and -not [string]::Equals( + $_.TrimEnd('\', '/'), $selectedDirectory.TrimEnd('\', '/'), + [System.StringComparison]::OrdinalIgnoreCase) + }) + $env:PATH = (@($selectedDirectory) + $remainingEntries) -join $pathSeparator + + foreach ($entry in $Commands.GetEnumerator()) { + $resolved = @(Get-Command $entry.Key -CommandType Application -ErrorAction Stop)[0] + if (-not [string]::Equals( + $resolved.Source, $entry.Value.Source, + [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Unable to restore caller-selected $($entry.Key): expected $($entry.Value.Source), resolved $($resolved.Source)." + } + } +} + +$repositoryRoot = Get-PipelineRepositoryRoot +$pipelineRoot = Join-Path $repositoryRoot 'out/pipeline' +$cmake = (Get-Command cmake -ErrorAction Stop).Source +$ctest = (Get-Command ctest -ErrorAction Stop).Source +$requestedClangCommands = Resolve-RequestedClangCommands -CompilerName $Compiler +$visualStudio = Initialize-PipelineVisualStudioEnvironment +Restore-RequestedClangCommands -Commands $requestedClangCommands +$ninja = Join-Path $visualStudio 'Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe' +if (-not (Test-Path -LiteralPath $ninja -PathType Leaf)) { + throw "Visual Studio's bundled Ninja executable is missing: $ninja" +} +$env:SIMDLIB_NINJA = $ninja + +<# +.SYNOPSIS +Returns native cells selected by compiler, configuration, and operation. +.PARAMETER CompilerName +Requested native compiler. +.PARAMETER CellScope +Requested configuration scope. +.PARAMETER Operation +Requested pipeline operation. +#> +function Resolve-NativeCells { + param( + [Parameter(Mandatory)][string]$CompilerName, + [Parameter(Mandatory)][string]$CellScope, + [Parameter(Mandatory)][string]$Operation + ) + + $compilerNames = if ($CompilerName -eq 'All') { + @(Get-PipelineValidationCompilers -Platform native) + } else { + @($CompilerName) + } + return @(Resolve-PipelineValidationCells -Platform native ` + -CompilerNames $compilerNames -CellScope $CellScope -Operation $Operation) +} +<# +.SYNOPSIS +Returns immutable compiler identity for one cell. +.PARAMETER BuildCell +Native cell definition. +#> +function Get-NativeCompilerIdentity { + param([Parameter(Mandatory)]$BuildCell) + $commandName = if ($BuildCell.Compiler -eq 'msvc') { 'cl.exe' } elseif ($BuildCell.Compiler -eq 'clangcl') { 'clang-cl.exe' } else { 'clang++.exe' } + $command = Get-Command $commandName -ErrorAction Stop + $version = if ($BuildCell.Compiler -eq 'msvc') { + $command.FileVersionInfo.ProductVersion + } else { + (& $command.Source --version | Select-Object -First 1).Trim() + } + return [ordered]@{ id = $BuildCell.Compiler; path = $command.Source; version = $version } +} + +<# +.SYNOPSIS +Creates and validates one native fingerprint artifact location. +.PARAMETER BuildCell +Native cell definition. +#> +function Initialize-NativeArtifact { + param([Parameter(Mandatory)]$BuildCell) + $compilerIdentity = Get-NativeCompilerIdentity -BuildCell $BuildCell + $fingerprint = [ordered]@{ + schema = 'simdlib.build-cell-fingerprint.v1' + platform = 'windows-x64' + compiler = $compilerIdentity + configuration = [ordered]@{ + key = $BuildCell.Key; preset = $BuildCell.Preset; buildProfile = $BuildCell.BuildProfile + sanitizer = $BuildCell.Sanitizer; instrumentation = $BuildCell.Instrumentation + coverage = $BuildCell.Coverage; generator = $BuildCell.Generator + codegenMode = $BuildCell.CodegenMode + aggregate = $BuildCell.Aggregate + consumerScope = if ($BuildCell.Consumer) { 'compiler-release' } else { 'none' } + cxxStandard = '20-and-23-register' + } + dependencies = [ordered]@{ + cmakeVersion = (& $cmake --version | Select-Object -First 1).Trim() + ninjaPath = if ($BuildCell.Generator -eq 'Ninja') { $ninja } else { '' } + visualStudio = $visualStudio + catch2Commit = '2b60af89e23d28eefc081bc930831ee9d45ea58b' + } + requiredCpuFeatures = @('sse4.2', 'avx2', 'fma', 'bmi1', 'bmi2') + } + $json = $fingerprint | ConvertTo-Json -Depth 8 -Compress + $digest = Get-PipelineTextDigest -Text $json + $root = Join-Path $pipelineRoot "windows-$($BuildCell.Compiler)/$($BuildCell.Key)-$($digest.Substring(0, 16))" + $provenance = Join-Path $root 'provenance' + $fingerprintPath = Join-Path $provenance 'fingerprint.json' + New-Item -ItemType Directory -Path $provenance -Force | Out-Null + if (Test-Path -LiteralPath $fingerprintPath) { + $existing = Get-Content -LiteralPath $fingerprintPath -Raw + if ($existing -ne $json) { throw "Fingerprint collision at $root" } + } else { + Set-PipelineTextFile -Path $fingerprintPath -Content $json + } + return [pscustomobject]@{ + Id = "$($BuildCell.Compiler)-$($BuildCell.Key)"; Definition = $BuildCell; Fingerprint = $digest + Root = $root; Build = Join-Path $root 'build'; Consumer = Join-Path $root 'consumer' + Reports = Join-Path $root 'reports'; Provenance = $provenance; FingerprintPath = $fingerprintPath + CompilerIdentity = $compilerIdentity + } +} + +<# +.SYNOPSIS +Returns whether any supported CI indicator is nonempty. +#> +function Test-CiEnvironment { + foreach ($name in @('CI', 'GITHUB_ACTIONS', 'GITLAB_CI', 'TF_BUILD', 'BUILDKITE', 'CIRCLECI', 'JENKINS_URL', 'TEAMCITY_VERSION')) { + if ([Environment]::GetEnvironmentVariable($name)) { return $true } + } + return $false +} + +<# +.SYNOPSIS +Runs the CMake test-inventory recorder or validator. +.PARAMETER Mode +RECORD or VALIDATE. +.PARAMETER TestDirectory +CTest tree. +.PARAMETER InventoryPath +Owned inventory file. +.PARAMETER Configuration +Optional multi-config configuration. +#> +function Invoke-TestInventory { + param( + [Parameter(Mandatory)][ValidateSet('RECORD', 'VALIDATE')][string]$Mode, + [Parameter(Mandatory)][string]$TestDirectory, + [Parameter(Mandatory)][string]$InventoryPath, + [string]$Configuration = '' + ) + $arguments = @("-DMODE=$Mode", "-DTEST_DIRECTORY=$TestDirectory", "-DINVENTORY_FILE=$InventoryPath", "-DCMAKE_CTEST_COMMAND=$ctest") + if ($Configuration) { $arguments += "-DCONFIGURATION=$Configuration" } + $arguments += @('-P', (Join-Path $repositoryRoot 'cmake/RecordTestInventory.cmake')) + & $cmake @arguments + if ($LASTEXITCODE -ne 0) { throw "CTest inventory $Mode failed for $TestDirectory" } +} + +<# +.SYNOPSIS +Verifies that mandatory runtime-test labels and families exist in one native CTest tree. +.PARAMETER Artifact +Resolved native build-cell artifact. +#> +function Invoke-RuntimeTestInventoryAudit { + param([Parameter(Mandatory)]$Artifact) + $arguments = @( + "-DTEST_DIRECTORY=$($Artifact.Build)", + "-DCMAKE_CTEST_COMMAND=$ctest", + "-DAUDIT_FILE=$(Join-Path $Artifact.Reports 'runtime-test-inventory.audit.txt')", + '-DREGISTER_REQUIRED=ON' + ) + if ($Artifact.Definition.Compiler -eq 'msvc') { + $arguments += "-DCONFIGURATION=$($Artifact.Definition.BuildProfile)" + } + $arguments += @('-P', (Join-Path $repositoryRoot 'cmake/VerifyRuntimeTestInventory.cmake')) + & $cmake @arguments + if ($LASTEXITCODE -ne 0) { + throw "Mandatory runtime-test inventory audit failed for $($Artifact.Id)" + } +} + + +<# +.SYNOPSIS +Returns the canonical validation-matrix cell identifier for one native artifact. +.PARAMETER Artifact +Resolved native build-cell artifact. +#> +function Get-NativeValidationCellId { + param([Parameter(Mandatory)]$Artifact) + + return [string]$Artifact.Definition.MatrixCell +} +<# +.SYNOPSIS +Audits generated target and CTest ownership for one native build cell. +.PARAMETER Artifact +Resolved native build-cell artifact. +#> +function Invoke-NativeValidationInventoryAudit { + param([Parameter(Mandatory)]$Artifact) + + $auditParameters = @{ + Cell = Get-NativeValidationCellId -Artifact $Artifact + BuildDirectory = $Artifact.Build + ResultPath = Join-Path $Artifact.Reports 'validation-inventory.audit.json' + } + if ($Artifact.Definition.Compiler -eq 'msvc') { + $auditParameters.Configuration = $Artifact.Definition.BuildProfile + } + & (Join-Path $PSScriptRoot 'Audit-ValidationMatrix.ps1') @auditParameters + if ($LASTEXITCODE -ne 0) { + throw "Validation inventory audit failed for $($Artifact.Id)" + } +} +<# +.SYNOPSIS +Returns a file hash or the manifest marker for an absent optional file. +.PARAMETER Path +File to hash. +#> +function Get-OptionalFileHash { + param([Parameter(Mandatory)][string]$Path) + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return 'none' } + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +<# +.SYNOPSIS +Writes the aggregate generated-code record index from CMake-owned validation indexes. +.PARAMETER BuildDirectory +Configured build tree containing the owner indexes. +.PARAMETER OutputPath +Pipeline record index to write. +.PARAMETER AllowEmpty +Allows profiles that own no generated-code work to emit an empty index. +#> +function Write-CodegenRecordIndex { + param( + [Parameter(Mandatory)][string]$BuildDirectory, + [Parameter(Mandatory)][string]$OutputPath, + [switch]$AllowEmpty + ) + $ownerIndexes = @( + (Join-Path $BuildDirectory 'method-flags-codegen/all-records.txt'), + (Join-Path $BuildDirectory 'register-codegen/sse42/128/all-records.txt'), + (Join-Path $BuildDirectory 'register-codegen/avx2/128/all-records.txt'), + (Join-Path $BuildDirectory 'register-codegen/avx2/256/all-records.txt') + ) + $records = @( + foreach ($ownerIndex in $ownerIndexes) { + if (Test-Path -LiteralPath $ownerIndex -PathType Leaf) { + Get-Content -LiteralPath $ownerIndex | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + } + } + ) + $records = @($records | Sort-Object -Unique) + if (-not $records.Count -and -not $AllowEmpty) { + throw "No CMake-owned generated-code records were found under $BuildDirectory" + } + $content = if ($records.Count) { + ($records -join [Environment]::NewLine) + [Environment]::NewLine + } else { + '' + } + Set-PipelineTextFile -Path $OutputPath -Content $content +} + +<# +.SYNOPSIS +Returns and validates the external-consumer scope owned by one native cell. +.PARAMETER Artifact +Resolved cell whose configured capability inventory is inspected. +#> +function Get-NativeConsumerScope { + param([Parameter(Mandatory)]$Artifact) + if (-not $Artifact.Definition.Consumer) { return 'none' } + + $capabilityPath = Join-Path $Artifact.Build 'external-consumer-targets.txt' + if (-not (Test-Path -LiteralPath $capabilityPath -PathType Leaf)) { + throw "External-consumer capability inventory is missing: $capabilityPath" + } + $targets = @( + Get-Content -LiteralPath $capabilityPath | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | + Sort-Object -Unique + ) + $targetSequence = $targets -join '|' + if ($targetSequence -eq 'CoreConsumerSmoke') { return 'core' } + if ($targetSequence -eq 'CoreConsumerSmoke|RegisterConsumerSmoke') { + return 'core-register' + } + throw "Unsupported external-consumer capability inventory: $targetSequence" +} + +<# +.SYNOPSIS +Writes an atomic completed-operation manifest for one native cell. +.PARAMETER Artifact +Resolved cell artifact. +.PARAMETER Operation +Completed operation identity. +#> +function Write-NativeManifest { + param([Parameter(Mandatory)]$Artifact, [Parameter(Mandatory)][string]$Operation) + $mainInventory = Join-Path $Artifact.Provenance 'main-test-artifacts.inventory' + $consumerInventory = Join-Path $Artifact.Provenance 'consumer-test-artifacts.inventory' + $codegenIndex = Join-Path $Artifact.Provenance 'codegen-records.index' + $targetInventory = Join-Path $Artifact.Build 'development-profile-targets.txt' + $ownershipAudit = Join-Path $Artifact.Reports 'validation-inventory.audit.json' + $matrixContract = Join-Path $PSScriptRoot 'validation-matrix.json' + $consumerScope = Get-NativeConsumerScope -Artifact $Artifact + $aggregate = if ($Operation -eq 'build-benchmarks') { 'BenchmarkArtifacts' } else { $Artifact.Definition.Aggregate } + $mainMetadata = Join-Path $Artifact.Build 'CTestTestfile.cmake' + $consumerMetadata = Join-Path $Artifact.Consumer 'CTestTestfile.cmake' + $manifestName = if ($Operation -eq 'build-benchmarks') { 'benchmark-build.manifest' } else { 'validation-build.manifest' } + $manifestPath = Join-Path $Artifact.Provenance $manifestName + $lines = @( + 'schema=simdlib.build-manifest.v1', "operation=$Operation", 'status=complete', + "source_revision=$(Get-PipelineRevision -RepositoryRoot $repositoryRoot)", + "source_digest=$(Get-PipelineSourceDigest -RepositoryRoot $repositoryRoot)", + "fingerprint_sha256=$($Artifact.Fingerprint)", "fingerprint_document=$($Artifact.FingerprintPath)", + "compiler_id=$($Artifact.Definition.Compiler)", "compiler=$($Artifact.CompilerIdentity.version)", 'base_image=none', + "preset=$($Artifact.Definition.Preset)", "build_profile=$($Artifact.Definition.BuildProfile)", + "sanitizer=$($Artifact.Definition.Sanitizer)", "instrumentation=$($Artifact.Definition.Instrumentation)", + "codegen_mode=$($Artifact.Definition.CodegenMode)", + "aggregate=$aggregate", "matrix_cell=$(Get-NativeValidationCellId -Artifact $Artifact)", "consumer_scope=$consumerScope", + "target_inventory=$targetInventory", "target_inventory_sha256=$(Get-OptionalFileHash -Path $targetInventory)", + "matrix_contract_sha256=$(Get-OptionalFileHash -Path $matrixContract)", + "validation_inventory_audit=$ownershipAudit", + "validation_inventory_audit_sha256=$(Get-OptionalFileHash -Path $ownershipAudit)", + "build_directory=$($Artifact.Build)", "consumer_directory=$($Artifact.Consumer)", + "cmake_cache_sha256=$(Get-OptionalFileHash -Path (Join-Path $Artifact.Build 'CMakeCache.txt'))", + 'required_cpu_features=sse4.2,avx2,fma,bmi1,bmi2', + "main_test_inventory=$mainInventory", "main_test_inventory_sha256=$(Get-OptionalFileHash -Path $mainInventory)", + "main_ctest_metadata_sha256=$(Get-OptionalFileHash -Path $mainMetadata)", + "consumer_test_inventory=$consumerInventory", "consumer_test_inventory_sha256=$(Get-OptionalFileHash -Path $consumerInventory)", + "consumer_ctest_metadata_sha256=$(Get-OptionalFileHash -Path $consumerMetadata)", + "codegen_record_index=$codegenIndex", "codegen_record_index_sha256=$(Get-OptionalFileHash -Path $codegenIndex)" + ) + Set-PipelineTextFile -Path $manifestPath -Content (($lines -join "`n") + "`n") +} + +<# +.SYNOPSIS +Validates a native build manifest and every artifact index it binds. +.PARAMETER Artifact +Resolved cell artifact. +.PARAMETER Operation +Expected completed operation. +#> +function Assert-NativeManifest { + param([Parameter(Mandatory)]$Artifact, [Parameter(Mandatory)][string]$Operation) + $manifestName = if ($Operation -eq 'build-benchmarks') { 'benchmark-build.manifest' } else { 'validation-build.manifest' } + $path = Join-Path $Artifact.Provenance $manifestName + $manifest = Read-PipelineManifest -Path $path + $expected = @{ + schema = 'simdlib.build-manifest.v1'; operation = $Operation; status = 'complete' + fingerprint_sha256 = $Artifact.Fingerprint; fingerprint_document = $Artifact.FingerprintPath + compiler_id = $Artifact.Definition.Compiler; preset = $Artifact.Definition.Preset + build_profile = $Artifact.Definition.BuildProfile; sanitizer = $Artifact.Definition.Sanitizer + instrumentation = $Artifact.Definition.Instrumentation; codegen_mode = $Artifact.Definition.CodegenMode + aggregate = if ($Operation -eq 'build-benchmarks') { 'BenchmarkArtifacts' } else { $Artifact.Definition.Aggregate } + matrix_cell = Get-NativeValidationCellId -Artifact $Artifact + consumer_scope = Get-NativeConsumerScope -Artifact $Artifact + } + foreach ($key in $expected.Keys) { + if ($manifest[$key] -ne $expected[$key]) { throw "Manifest $path has mismatched $key" } + } + $sourceDigest = Get-PipelineSourceDigest -RepositoryRoot $repositoryRoot + if ($manifest.source_digest -ne $sourceDigest) { throw "Build manifest is stale for current source inputs: $path" } + $cache = Join-Path $Artifact.Build 'CMakeCache.txt' + if ($manifest.cmake_cache_sha256 -ne (Get-OptionalFileHash -Path $cache)) { throw "Build manifest is stale for CMake cache: $path" } + if ($manifest.target_inventory_sha256 -ne (Get-OptionalFileHash -Path $manifest.target_inventory)) { throw "Configured target inventory is missing or stale: $($manifest.target_inventory)" } + $matrixContract = Join-Path $PSScriptRoot 'validation-matrix.json' + Invoke-NativeValidationInventoryAudit -Artifact $Artifact + if ($manifest.matrix_contract_sha256 -ne (Get-OptionalFileHash -Path $matrixContract)) { throw "Validation matrix contract is stale for $($Artifact.Id)" } + if ($manifest.validation_inventory_audit_sha256 -ne (Get-OptionalFileHash -Path $manifest.validation_inventory_audit)) { throw "Validation inventory audit is missing or stale: $($manifest.validation_inventory_audit)" } + if ($Operation -eq 'build-validation') { + foreach ($pair in @( + @('main_test_inventory', 'main_test_inventory_sha256'), + @('consumer_test_inventory', 'consumer_test_inventory_sha256'), + @('codegen_record_index', 'codegen_record_index_sha256') + )) { + if ($manifest[$pair[1]] -ne (Get-OptionalFileHash -Path $manifest[$pair[0]])) { throw "Artifact index is missing or stale: $($manifest[$pair[0]])" } + } + $configuration = if ($Artifact.Definition.Compiler -eq 'msvc') { $Artifact.Definition.BuildProfile } else { '' } + Invoke-TestInventory -Mode VALIDATE -TestDirectory $Artifact.Build -InventoryPath $manifest.main_test_inventory -Configuration $configuration + if ($Artifact.Definition.Consumer) { + Invoke-TestInventory -Mode VALIDATE -TestDirectory $Artifact.Consumer -InventoryPath $manifest.consumer_test_inventory -Configuration $configuration + } + & $cmake "-DRECORD_INDEX=$($manifest.codegen_record_index)" -P (Join-Path $repositoryRoot 'cmake/ValidateCodegenRecords.cmake') + if ($LASTEXITCODE -ne 0) { throw "Generated-code records are stale for $($Artifact.Id)" } + } + return $manifest +} + +<# +.SYNOPSIS +Configures and builds one native validation cell. +.PARAMETER Artifact +Resolved cell artifact. +#> +function Build-NativeValidationCell { + param([Parameter(Mandatory)]$Artifact) + if ($InjectFailure -contains 'All' -or $InjectFailure -contains $Artifact.Id) { throw "Intentional native failure: $($Artifact.Id)" } + New-Item -ItemType Directory -Path $Artifact.Reports, $Artifact.Provenance -Force | Out-Null + $env:SIMDLIB_BUILD_DIRECTORY = $Artifact.Build + $configureArguments = @('--preset', $Artifact.Definition.Preset, '-S', $repositoryRoot) + if (Test-CiEnvironment) { $configureArguments = @('--fresh') + $configureArguments } + Invoke-PipelineCommand -FilePath $cmake -ArgumentList $configureArguments -LogPath (Join-Path $Artifact.Reports 'main-configure.log') + $buildArguments = @('--build', $Artifact.Build, '--parallel', '--target', $Artifact.Definition.Aggregate) + if ($Artifact.Definition.Compiler -eq 'msvc') { $buildArguments += @('--config', $Artifact.Definition.BuildProfile) } + Invoke-PipelineCommand -FilePath $cmake -ArgumentList $buildArguments -LogPath (Join-Path $Artifact.Reports 'main-build.log') + + if ($Artifact.Definition.Consumer) { + $consumerScope = Get-NativeConsumerScope -Artifact $Artifact + $registerConsumer = if ($consumerScope -eq 'core-register') { 'ON' } else { 'OFF' } + $consumerArguments = @('-S', (Join-Path $repositoryRoot 'tests/consumer'), '-B', $Artifact.Consumer, "-DSIMDLIB_SOURCE_DIR=$repositoryRoot", "-DSIMDLIB_BUILD_REGISTER_CONSUMER=$registerConsumer") + if ($Artifact.Definition.Compiler -eq 'msvc') { + $consumerArguments += @('-G', 'Visual Studio 17 2022', '-A', 'x64', "-DCMAKE_CONFIGURATION_TYPES=$($Artifact.Definition.BuildProfile)") + } else { + $consumerArguments += @('-G', 'Ninja', "-DCMAKE_BUILD_TYPE=$($Artifact.Definition.BuildProfile)", "-DCMAKE_CXX_COMPILER=$((Get-Command clang-cl.exe).Source)", "-DCMAKE_MAKE_PROGRAM=$ninja") + } + Invoke-PipelineCommand -FilePath $cmake -ArgumentList $consumerArguments -LogPath (Join-Path $Artifact.Reports 'consumer-configure.log') + $consumerBuildArguments = @('--build', $Artifact.Consumer, '--parallel') + if ($Artifact.Definition.Compiler -eq 'msvc') { $consumerBuildArguments += @('--config', $Artifact.Definition.BuildProfile) } + Invoke-PipelineCommand -FilePath $cmake -ArgumentList $consumerBuildArguments -LogPath (Join-Path $Artifact.Reports 'consumer-build.log') + } elseif (Test-Path -LiteralPath $Artifact.Consumer) { + throw "Consumer-free cell contains an external-consumer tree: $($Artifact.Consumer)" + } + + $mainInventory = Join-Path $Artifact.Provenance 'main-test-artifacts.inventory' + $consumerInventory = Join-Path $Artifact.Provenance 'consumer-test-artifacts.inventory' + $configuration = if ($Artifact.Definition.Compiler -eq 'msvc') { $Artifact.Definition.BuildProfile } else { '' } + Invoke-TestInventory -Mode RECORD -TestDirectory $Artifact.Build -InventoryPath $mainInventory -Configuration $configuration + if ($Artifact.Definition.Consumer) { + Invoke-TestInventory -Mode RECORD -TestDirectory $Artifact.Consumer -InventoryPath $consumerInventory -Configuration $configuration + } else { + Set-PipelineTextFile -Path $consumerInventory -Content '' + } + $allowEmptyCodegen = $Artifact.Definition.CodegenMode -eq 'OFF' + Write-CodegenRecordIndex -BuildDirectory $Artifact.Build -OutputPath (Join-Path $Artifact.Provenance 'codegen-records.index') -AllowEmpty:$allowEmptyCodegen + Invoke-NativeValidationInventoryAudit -Artifact $Artifact + Write-NativeManifest -Artifact $Artifact -Operation 'build-validation' +} + +<# +.SYNOPSIS +Runs the focused compiler-contract CTest inventory without building. +.PARAMETER Artifact +Resolved compiler-contract artifact. +#> +function Test-NativeCompilerContractCell { + param([Parameter(Mandatory)]$Artifact) + [void](Assert-NativeManifest -Artifact $Artifact -Operation 'build-validation') + $arguments = @('--test-dir', $Artifact.Build, '--output-on-failure') + if ($Artifact.Definition.Compiler -eq 'msvc') { $arguments += @('-C', $Artifact.Definition.BuildProfile) } + if ($TestRegex) { $arguments += @('--tests-regex', $TestRegex) } + if ($TestLabel) { $arguments += @('--label-regex', $TestLabel) } + Invoke-PipelineCommand -FilePath $ctest -ArgumentList $arguments -LogPath (Join-Path $Artifact.Reports 'compiler-contract-tests.log') +} + +<# +.SYNOPSIS +Writes dedicated provenance for one record-only native codegen diagnostic. +.PARAMETER Artifact +Resolved diagnostic fingerprint. +.PARAMETER InvocationCompilationSeconds +Elapsed fixture-object compilation time for the current invocation. +.PARAMETER InvocationComparisonSeconds +Elapsed disassembly and comparison time for the current invocation. +.PARAMETER MeasuredCompilationSeconds +Largest source-compatible compilation measurement retained across cached runs. +.PARAMETER MeasuredComparisonSeconds +Largest source-compatible comparison measurement retained across cached runs. +#> +function Write-NativeCodegenDiagnosticProvenance { + param( + [Parameter(Mandatory)]$Artifact, + [Parameter(Mandatory)][double]$InvocationCompilationSeconds, + [Parameter(Mandatory)][double]$InvocationComparisonSeconds, + [Parameter(Mandatory)][double]$MeasuredCompilationSeconds, + [Parameter(Mandatory)][double]$MeasuredComparisonSeconds + ) + $recordIndex = Join-Path $Artifact.Provenance 'codegen-records.index' + $compileCommands = Join-Path $Artifact.Build 'compile_commands.json' + if (-not (Test-Path -LiteralPath $compileCommands -PathType Leaf)) { + throw "Diagnostic compiler-flag inventory is missing: $compileCommands" + } + $recordPaths = @(Get-Content -LiteralPath $recordIndex | Where-Object { $_ }) + $recordTimings = @( + foreach ($recordPath in $recordPaths) { + $record = Get-Content -LiteralPath $recordPath -Raw | ConvertFrom-Json + $profileProperty = $record.policy.PSObject.Properties['codegen_profile'] + [pscustomobject]@{ + path = $recordPath + profile = if ($profileProperty) { $profileProperty.Value } else { 'default-abi' } + result = $record.result + seconds = [int]$record.timing.total_seconds + stackProtectorMode = $record.stack_protector_mode + disassemblyTool = [pscustomobject]@{ + path = $record.tool.path + version = $record.tool.version + sha256 = $record.tool.sha256 + } + } + } + ) + $slowestRecords = @($recordTimings | Sort-Object seconds -Descending | Select-Object -First 10) + $stackProtectorModes = @( + $recordTimings | Select-Object -ExpandProperty stackProtectorMode -Unique | + Sort-Object + ) + $disassemblyTools = @( + $recordTimings | Group-Object { + "$($_.disassemblyTool.path)|$($_.disassemblyTool.version)|$($_.disassemblyTool.sha256)" + } | ForEach-Object { $_.Group[0].disassemblyTool } + ) + $document = [ordered]@{ + schema = 'simdlib.codegen-diagnostic-provenance.v1' + operation = 'record-codegen' + status = 'complete' + sourceRevision = Get-PipelineRevision -RepositoryRoot $repositoryRoot + sourceDigest = Get-PipelineSourceDigest -RepositoryRoot $repositoryRoot + fingerprint = $Artifact.Fingerprint + compiler = $Artifact.CompilerIdentity + configuration = [ordered]@{ + preset = $Artifact.Definition.Preset + buildProfile = $Artifact.Definition.BuildProfile + sanitizer = $Artifact.Definition.Sanitizer + codegenMode = $Artifact.Definition.CodegenMode + } + compilerFlags = [ordered]@{ + path = $compileCommands + sha256 = Get-OptionalFileHash -Path $compileCommands + } + records = [ordered]@{ + index = $recordIndex + sha256 = Get-OptionalFileHash -Path $recordIndex + count = $recordPaths.Count + slowest = $slowestRecords + } + stackProtectorModes = $stackProtectorModes + disassemblyTools = $disassemblyTools + timing = [ordered]@{ + invocation = [ordered]@{ + compilationSeconds = [Math]::Round($InvocationCompilationSeconds, 3) + comparisonSeconds = [Math]::Round($InvocationComparisonSeconds, 3) + totalSeconds = [Math]::Round( + $InvocationCompilationSeconds + $InvocationComparisonSeconds, 3) + } + measured = [ordered]@{ + compilationSeconds = [Math]::Round($MeasuredCompilationSeconds, 3) + comparisonSeconds = [Math]::Round($MeasuredComparisonSeconds, 3) + totalSeconds = [Math]::Round( + $MeasuredCompilationSeconds + $MeasuredComparisonSeconds, 3) + } + } + } + $provenancePath = Join-Path $Artifact.Provenance 'codegen-diagnostic.json' + Set-PipelineTextFile -Path $provenancePath -Content ($document | ConvertTo-Json -Depth 10) + return $provenancePath +} + +<# +.SYNOPSIS +Compiles only native Register fixtures, then records and validates diagnostics. +.PARAMETER Artifact +Resolved diagnostic fingerprint. +#> +function Record-NativeCodegenDiagnostic { + param([Parameter(Mandatory)]$Artifact) + if ($InjectFailure -contains 'All' -or $InjectFailure -contains $Artifact.Id) { + throw "Intentional native failure: $($Artifact.Id)" + } + New-Item -ItemType Directory -Path $Artifact.Reports, $Artifact.Provenance -Force | Out-Null + $provenancePath = Join-Path $Artifact.Provenance 'codegen-diagnostic.json' + $priorProvenance = $null + if (Test-Path -LiteralPath $provenancePath -PathType Leaf) { + $priorProvenance = Get-Content -LiteralPath $provenancePath -Raw | ConvertFrom-Json + } + $priorCompilationSeconds = 0.0 + $priorComparisonSeconds = 0.0 + $env:SIMDLIB_BUILD_DIRECTORY = $Artifact.Build + $configureArguments = @('--preset', $Artifact.Definition.Preset, '-S', $repositoryRoot) + if (Test-CiEnvironment) { $configureArguments = @('--fresh') + $configureArguments } + Invoke-PipelineCommand -FilePath $cmake -ArgumentList $configureArguments -LogPath (Join-Path $Artifact.Reports 'codegen-configure.log') + + $compilationWatch = [System.Diagnostics.Stopwatch]::StartNew() + Invoke-PipelineCommand -FilePath $cmake -ArgumentList @( + '--build', $Artifact.Build, '--parallel', '--target', 'RegisterCodegenFixtureObjects' + ) -LogPath (Join-Path $Artifact.Reports 'codegen-compilation.log') + $compilationWatch.Stop() + + $comparisonWatch = [System.Diagnostics.Stopwatch]::StartNew() + Invoke-PipelineCommand -FilePath $cmake -ArgumentList @( + '--build', $Artifact.Build, '--parallel', '--target', 'SimdLibDebugDiagnosticArtifacts' + ) -LogPath (Join-Path $Artifact.Reports 'codegen-comparison.log') + $comparisonWatch.Stop() + + $recordIndex = Join-Path $Artifact.Provenance 'codegen-records.index' + Write-CodegenRecordIndex -BuildDirectory $Artifact.Build -OutputPath $recordIndex + $compileCommands = Join-Path $Artifact.Build 'compile_commands.json' + if ($priorProvenance) { + $priorCompilerFlags = $priorProvenance.PSObject.Properties['compilerFlags'] + $priorRecords = $priorProvenance.PSObject.Properties['records'] + $sameDiagnosticInputs = $priorCompilerFlags -and $priorRecords -and + $priorCompilerFlags.Value.sha256 -eq (Get-OptionalFileHash -Path $compileCommands) -and + $priorRecords.Value.sha256 -eq (Get-OptionalFileHash -Path $recordIndex) + if ($sameDiagnosticInputs) { + $measuredTiming = $priorProvenance.timing.PSObject.Properties['measured'] + if ($measuredTiming) { + $priorCompilationSeconds = [double]$measuredTiming.Value.compilationSeconds + $priorComparisonSeconds = [double]$measuredTiming.Value.comparisonSeconds + } else { + $priorCompilationSeconds = [double]$priorProvenance.timing.compilationSeconds + $priorComparisonSeconds = [double]$priorProvenance.timing.comparisonSeconds + } + } + } + & $cmake "-DRECORD_INDEX=$recordIndex" '-DEXPECTED_POLICY_MODE=RECORD' ` + '-DEXPECTED_CONFIGURATION=Debug' '-DREQUIRE_RECORDS=ON' ` + -P (Join-Path $repositoryRoot 'cmake/ValidateCodegenRecords.cmake') + if ($LASTEXITCODE -ne 0) { throw "Diagnostic records are invalid for $($Artifact.Id)" } + & $cmake "-DBINARY_DIRECTORY=$($Artifact.Build)" ` + "-DOWNERSHIP_FILE=$(Join-Path $Artifact.Build 'development-target-ownership.tsv')" ` + '-DPROFILE=CODEGEN_DIAGNOSTIC' '-DCODEGEN_MODE=RECORD' ` + -P (Join-Path $repositoryRoot 'cmake/VerifyCodegenProfileIsolation.cmake') + if ($LASTEXITCODE -ne 0) { throw "Diagnostic profile isolation failed for $($Artifact.Id)" } + $measuredCompilationSeconds = [Math]::Max( + $compilationWatch.Elapsed.TotalSeconds, $priorCompilationSeconds) + $measuredComparisonSeconds = [Math]::Max( + $comparisonWatch.Elapsed.TotalSeconds, $priorComparisonSeconds) + $provenance = Write-NativeCodegenDiagnosticProvenance -Artifact $Artifact ` + -InvocationCompilationSeconds $compilationWatch.Elapsed.TotalSeconds ` + -InvocationComparisonSeconds $comparisonWatch.Elapsed.TotalSeconds ` + -MeasuredCompilationSeconds $measuredCompilationSeconds ` + -MeasuredComparisonSeconds $measuredComparisonSeconds + Write-Host "Native codegen diagnostic provenance: $provenance" +} + +<# +.SYNOPSIS +Validates host ISA support required by native runtime tests. +#> +function Assert-NativeCpuFeatures { + $features = [ordered]@{ + 'sse4.2' = [System.Runtime.Intrinsics.X86.Sse42]::IsSupported + avx2 = [System.Runtime.Intrinsics.X86.Avx2]::IsSupported + fma = [System.Runtime.Intrinsics.X86.Fma]::IsSupported + bmi1 = [System.Runtime.Intrinsics.X86.Bmi1]::IsSupported + bmi2 = [System.Runtime.Intrinsics.X86.Bmi2]::IsSupported + } + foreach ($feature in $features.Keys) { if (-not $features[$feature]) { throw "Host CPU does not expose required feature: $feature" } } +} + +<# +.SYNOPSIS +Runs tests and optional coverage reporting for one validated native cell. +.PARAMETER Artifact +Resolved cell artifact. +#> +function Test-NativeCell { + param([Parameter(Mandatory)]$Artifact) + [void](Assert-NativeManifest -Artifact $Artifact -Operation 'build-validation') + Assert-NativeCpuFeatures + New-Item -ItemType Directory -Path $Artifact.Reports -Force | Out-Null + Invoke-RuntimeTestInventoryAudit -Artifact $Artifact + if ($Artifact.Definition.Coverage) { + & $cmake "-DBINARY_DIRECTORY=$($Artifact.Build)" -P (Join-Path $repositoryRoot 'cmake/ResetCoverage.cmake') + if ($LASTEXITCODE -ne 0) { throw "Coverage reset failed for $($Artifact.Id)" } + $env:LLVM_PROFILE_FILE = Join-Path $Artifact.Build 'ctest-%p-%m.profraw' + } + $testArguments = @('--test-dir', $Artifact.Build, '--output-on-failure', '--output-junit', (Join-Path $Artifact.Reports 'main-test.xml')) + if ($Artifact.Definition.Compiler -eq 'msvc') { $testArguments += @('-C', $Artifact.Definition.BuildProfile) } + if ($TestRegex) { $testArguments += @('--tests-regex', $TestRegex) } + if ($TestLabel) { $testArguments += @('--label-regex', $TestLabel) } + Invoke-PipelineCommand -FilePath $ctest -ArgumentList $testArguments -LogPath (Join-Path $Artifact.Reports 'main-test.log') + if ($Artifact.Definition.Consumer) { + $consumerArguments = @('--test-dir', $Artifact.Consumer, '--output-on-failure', '--output-junit', (Join-Path $Artifact.Reports 'consumer-test.xml')) + if ($Artifact.Definition.Compiler -eq 'msvc') { $consumerArguments += @('-C', $Artifact.Definition.BuildProfile) } + Invoke-PipelineCommand -FilePath $ctest -ArgumentList $consumerArguments -LogPath (Join-Path $Artifact.Reports 'consumer-test.log') + } + if ($Artifact.Definition.Coverage) { + $coverageManifest = Get-ChildItem -LiteralPath $Artifact.Build -Filter 'coverage-targets-*.txt' -File | Select-Object -First 1 + if (-not $coverageManifest) { throw "Coverage target manifest is missing below $($Artifact.Build)" } + $arguments = @( + "-DBINARY_DIRECTORY=$($Artifact.Build)", "-DSOURCE_DIRECTORY=$repositoryRoot", + "-DCOVERAGE_MANIFEST=$($coverageManifest.FullName)", "-DLLVM_PROFDATA=$((Get-Command llvm-profdata.exe).Source)", + "-DLLVM_COV=$((Get-Command llvm-cov.exe).Source)", "-DLLVM_READOBJ=$((Get-Command llvm-readobj.exe).Source)", + '-P', (Join-Path $repositoryRoot 'cmake/GenerateCoverageReport.cmake') + ) + Invoke-PipelineCommand -FilePath $cmake -ArgumentList $arguments -LogPath (Join-Path $Artifact.Reports 'coverage-report.log') + } +} + +<# +.SYNOPSIS +Builds benchmarks in an already validated native Release tree. +.PARAMETER Artifact +Resolved Release artifact. +#> +function Build-NativeBenchmarks { + param([Parameter(Mandatory)]$Artifact) + [void](Assert-NativeManifest -Artifact $Artifact -Operation 'build-validation') + $arguments = @('--build', $Artifact.Build, '--parallel', '--target', 'BenchmarkArtifacts') + if ($Artifact.Definition.Compiler -eq 'msvc') { $arguments += @('--config', 'Release') } + Invoke-PipelineCommand -FilePath $cmake -ArgumentList $arguments -LogPath (Join-Path $Artifact.Reports 'benchmark-build.log') + Write-NativeManifest -Artifact $Artifact -Operation 'build-benchmarks' +} + +<# +.SYNOPSIS +Runs the benchmark executable from a validated benchmark manifest. +.PARAMETER Artifact +Resolved Release artifact. +#> +function Run-NativeBenchmarks { + param([Parameter(Mandatory)]$Artifact) + [void](Assert-NativeManifest -Artifact $Artifact -Operation 'build-benchmarks') + Assert-NativeCpuFeatures + $benchmark = Get-ChildItem -LiteralPath $Artifact.Build -Filter 'Benchmarks.exe' -File -Recurse | Select-Object -First 1 + if (-not $benchmark) { throw "Required benchmark executable is missing below $($Artifact.Build)" } + Invoke-PipelineCommand -FilePath $benchmark.FullName -ArgumentList @('[simdlib][benchmark]', '--benchmark-samples', '25') -LogPath (Join-Path $Artifact.Reports 'benchmark-execution.txt') +} + +if (($TestRegex -or $TestLabel) -and $Action -notin @('Test', 'TestCompilerContracts')) { throw '-TestRegex and -TestLabel are valid only for Test and TestCompilerContracts.' } +if ($Cell -eq 'Coverage' -and $Compiler -notin @('All', 'ClangCoverage')) { throw 'Coverage is owned by the native Clang coverage compiler.' } +if ($Action -in @('BuildBenchmarks', 'RunBenchmarks') -and $Cell -notin @('All', 'Release')) { throw 'Benchmark operations use Release cells only.' } +if ($Action -in @('BuildCompilerContracts', 'TestCompilerContracts') -and $Cell -notin @('All', 'Release')) { + throw 'Focused compiler-contract operations use Release compiler identities.' +} +if ($Action -eq 'RecordCodegen') { + if ($Cell -notin @('All', 'Debug')) { throw 'Native codegen diagnostics use Debug cells only.' } + if ($Compiler -eq 'ClangCoverage') { throw 'Native coverage does not own a Register codegen diagnostic.' } +} + +$cells = @(Resolve-NativeCells -CompilerName $Compiler -CellScope $Cell -Operation $Action) +if ($cells.Count -eq 0) { throw 'The native compiler and cell selections identify no operation cells.' } +$artifacts = @($cells | ForEach-Object { Initialize-NativeArtifact -BuildCell $_ }) +$failures = [System.Collections.Generic.List[string]]::new() +foreach ($artifact in $artifacts) { + try { + Write-Host "Native operation: action=$Action cell=$($artifact.Id) root=$($artifact.Root)" + switch ($Action) { + 'Build' { Build-NativeValidationCell -Artifact $artifact } + 'BuildCompilerContracts' { Build-NativeValidationCell -Artifact $artifact } + 'TestCompilerContracts' { Test-NativeCompilerContractCell -Artifact $artifact } + 'Test' { Test-NativeCell -Artifact $artifact } + 'RecordCodegen' { Record-NativeCodegenDiagnostic -Artifact $artifact } + 'BuildBenchmarks' { Build-NativeBenchmarks -Artifact $artifact } + 'RunBenchmarks' { Run-NativeBenchmarks -Artifact $artifact } + } + } catch { + Write-Error -ErrorAction Continue "$($artifact.Id): $_" + $failures.Add($artifact.Id) + } +} +if ($failures.Count) { throw "Native operation $Action failed: $($failures -join ', ')" } +Write-Host "Native operation passed: action=$Action cells=$($artifacts.Count)" diff --git a/tools/Run-Tests.ps1 b/tools/Run-Tests.ps1 new file mode 100644 index 0000000..2d496dd --- /dev/null +++ b/tools/Run-Tests.ps1 @@ -0,0 +1,165 @@ +<# +.SYNOPSIS +Runs the requested SimdLib validation matrix from an exact build receipt. +.DESCRIPTION +The command validates the exact receipt produced by Build.ps1 and then runs only +test operations. A missing, stale, incomplete, or mismatched receipt is rejected +without configuring or rebuilding any target. +#> +[CmdletBinding()] +param( + [ValidateSet('All', 'Native', 'Containers')] + [string]$Scope = 'All', + [ValidateSet('All', 'Msvc', 'ClangCl', 'ClangCoverage', 'Gcc13', 'Gcc14', 'Clang22')] + [string[]]$Compiler = @('All'), + [string]$TestRegex = '', + [string]$TestLabel = '' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force +$repositoryRoot = Get-PipelineRepositoryRoot +$pipelineRoot = Join-Path $repositoryRoot 'out/pipeline' + +<# +.SYNOPSIS +Expands compiler filters and enforces their platform scope. +#> +function Resolve-TestSelection { + $nativeNames = @(Get-PipelineValidationCompilers -Platform native) + $containerNames = @(Get-PipelineValidationCompilers -Platform container) + if ('All' -in $Compiler -and $Compiler.Count -ne 1) { throw 'Compiler All cannot be combined with another compiler filter.' } + if ($Compiler -contains 'All') { + $selected = switch ($Scope) { + 'Native' { $nativeNames } + 'Containers' { $containerNames } + default { $nativeNames + $containerNames } + } + } else { $selected = @($Compiler | Select-Object -Unique) } + if ($Scope -eq 'Native' -and @($selected | Where-Object { $_ -in $containerNames }).Count) { throw 'Container compiler filters are invalid for Native scope.' } + if ($Scope -eq 'Containers' -and @($selected | Where-Object { $_ -in $nativeNames }).Count) { throw 'Native compiler filters are invalid for Containers scope.' } + return @($selected) +} + +<# +.SYNOPSIS +Validates the exact build receipt required by this test selection. +.PARAMETER SelectedCompilers +Canonical compiler selection. +#> +function Assert-BuildReceipt { + param([Parameter(Mandatory)][string[]]$SelectedCompilers) + $selectionText = "$Scope|$($SelectedCompilers -join ',')" + $selectionId = (Get-PipelineTextDigest -Text $selectionText).Substring(0, 16) + $receiptPath = Join-Path $pipelineRoot "provenance/build-$selectionId.json" + if (-not (Test-Path -LiteralPath $receiptPath -PathType Leaf)) { throw "Required unified build receipt is missing: $receiptPath" } + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json + if ($receipt.schema -ne 'simdlib.unified-build-receipt.v5' -or $receipt.status -ne 'complete' -or $receipt.scope -ne $Scope) { + throw "Unified build receipt is incomplete or incompatible: $receiptPath" + } + $receiptCompilers = @($receipt.compilers) + if (($receiptCompilers -join ',') -ne ($SelectedCompilers -join ',')) { throw "Unified build receipt compiler set does not match the requested tests: $receiptPath" } + $currentDigest = Get-PipelineSourceDigest -RepositoryRoot $repositoryRoot + $matrixPath = Join-Path $repositoryRoot 'tools/validation-matrix.json' + $matrixHash = (Get-FileHash -LiteralPath $matrixPath -Algorithm SHA256).Hash.ToLowerInvariant() + $matrix = Get-Content -LiteralPath $matrixPath -Raw | ConvertFrom-Json + if ($receipt.sourceDigest -ne $currentDigest) { throw "Unified build receipt is stale for current source inputs: $receiptPath" } + $toolingDigest = Get-PipelineToolingDigest -RepositoryRoot $repositoryRoot + [void](Assert-PipelineValidationEntry ` + -RepositoryRoot $repositoryRoot ` + -Entry $receipt.pipelineValidation ` + -ExpectedToolingDigest $toolingDigest) + $expectedPresets = @(Get-PipelineDefaultValidationPresets -SelectedCompilers $SelectedCompilers | Sort-Object) + $receiptPresets = @($receipt.manifests | ForEach-Object { $_.preset } | Sort-Object) + if (($receiptPresets -join ',') -ne ($expectedPresets -join ',')) { throw "Unified build receipt manifest set does not exactly match requested test cells: $receiptPath" } + foreach ($entry in $receipt.manifests) { + $manifestPath = Join-Path $repositoryRoot ([string]$entry.path) + if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) { throw "Receipt manifest is missing: $manifestPath" } + $hash = (Get-FileHash -LiteralPath $manifestPath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($hash -ne $entry.sha256) { throw "Receipt manifest changed after the unified build: $manifestPath" } + $manifest = Read-PipelineManifest -Path $manifestPath + if ($manifest.source_digest -ne $currentDigest -or $manifest.source_digest -ne $entry.sourceDigest) { + throw "Receipt manifest source digest does not match the unified receipt and current sources: $manifestPath" + } + $provenancePairs = @{ + matrix_cell = 'matrixCell' + aggregate = 'aggregate'; target_inventory_sha256 = 'targetInventorySha256' + main_test_inventory_sha256 = 'testInventorySha256'; build_profile = 'configuration' + instrumentation = 'instrumentation'; codegen_mode = 'generatedCodeMode'; consumer_scope = 'consumerScope' + matrix_contract_sha256 = 'matrixContractSha256' + validation_inventory_audit_sha256 = 'inventoryAuditSha256' + } + foreach ($manifestKey in $provenancePairs.Keys) { + $receiptValue = [string]$entry.($provenancePairs[$manifestKey]) + if ($manifest[$manifestKey] -ne $receiptValue) { + throw "Receipt manifest provenance $manifestKey does not match the unified receipt: $manifestPath" + } + } + if ($manifest.matrix_contract_sha256 -ne $matrixHash) { + throw "Receipt manifest uses a stale validation matrix contract: $manifestPath" + } + $inventoryAuditPath = Resolve-PipelineArtifactPath ` + -RepositoryRoot $repositoryRoot ` + -Path ([string]$manifest.validation_inventory_audit) + if (-not (Test-Path -LiteralPath $inventoryAuditPath -PathType Leaf)) { + throw "Receipt validation inventory audit is missing: $inventoryAuditPath" + } + $inventoryAuditHash = (Get-FileHash -LiteralPath $inventoryAuditPath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($inventoryAuditHash -ne $manifest.validation_inventory_audit_sha256) { + throw "Receipt validation inventory audit changed after the build: $inventoryAuditPath" + } + $inventoryAudit = Get-Content -LiteralPath $inventoryAuditPath -Raw | ConvertFrom-Json + $matrixCell = $matrix.cells.PSObject.Properties[[string]$manifest.matrix_cell] + if (-not $matrixCell -or + $inventoryAudit.schema -ne 'simdlib.validation-inventory-audit.v1' -or + $inventoryAudit.status -ne 'complete' -or + $inventoryAudit.cell -ne $manifest.matrix_cell -or + $inventoryAudit.profile -ne $matrixCell.Value.profile) { + throw "Receipt validation inventory audit is category-incompatible: $inventoryAuditPath" + } $canonicalCell = $matrixCell.Value + $ownsConsumer = $manifest.consumer_scope -ne 'none' + if ($manifest.preset -ne $canonicalCell.preset -or + $manifest.build_profile -ne $canonicalCell.configuration -or + $manifest.instrumentation -ne $canonicalCell.instrumentation -or + $manifest.codegen_mode -ne $canonicalCell.codegenMode -or + $manifest.aggregate -ne $canonicalCell.aggregate -or + $ownsConsumer -ne [bool]$canonicalCell.consumer) { + throw "Receipt manifest disagrees with canonical matrix cell $($manifest.matrix_cell): $manifestPath" + } + if ($manifest.aggregate -ne 'ExhaustiveArtifacts' -or + $manifest.target_inventory_sha256 -eq 'none' -or + $manifest.main_test_inventory_sha256 -eq 'none') { + throw "Receipt manifest does not cover the required default target and test inventories: $manifestPath" + } + } + return $receiptPath +} + +$selectedCompilers = @(Resolve-TestSelection) +$receiptPath = Assert-BuildReceipt -SelectedCompilers $selectedCompilers + +$operations = [System.Collections.Generic.List[object]]::new() +foreach ($name in @($selectedCompilers | Where-Object { $_ -in (Get-PipelineValidationCompilers -Platform native) })) { + $arguments = @('-Action', 'Test', '-Compiler', $name, '-Cell', 'All') + if ($TestRegex) { $arguments += @('-TestRegex', $TestRegex) } + if ($TestLabel) { $arguments += @('-TestLabel', $TestLabel) } + $operations.Add([pscustomobject]@{ Id = "native-$($name.ToLowerInvariant())"; Script = Join-Path $PSScriptRoot 'Run-NativeMatrix.ps1'; Arguments = $arguments }) +} +$containerCompilers = @($selectedCompilers | Where-Object { $_ -in (Get-PipelineValidationCompilers -Platform container) }) +if ($containerCompilers.Count -eq 3) { + $arguments = @('-Action', 'Test', '-Compiler', 'All', '-Cell', 'All') + if ($TestRegex) { $arguments += @('-TestRegex', $TestRegex) } + if ($TestLabel) { $arguments += @('-TestLabel', $TestLabel) } + $operations.Add([pscustomobject]@{ Id = 'containers'; Script = Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1'; Arguments = $arguments }) +} else { + foreach ($name in $containerCompilers) { + $arguments = @('-Action', 'Test', '-Compiler', $name, '-Cell', 'All') + if ($TestRegex) { $arguments += @('-TestRegex', $TestRegex) } + if ($TestLabel) { $arguments += @('-TestLabel', $TestLabel) } + $operations.Add([pscustomobject]@{ Id = "container-$($name.ToLowerInvariant())"; Script = Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1'; Arguments = $arguments }) + } +} +$logDirectory = Join-Path $pipelineRoot "logs/$(Get-Date -Format 'yyyyMMdd-HHmmssfff')-run-tests-$PID" +Invoke-PipelineChildOperations -Operations $operations.ToArray() -LogDirectory $logDirectory +Write-Host "Unified tests passed. Build receipt: $receiptPath" diff --git a/tools/Test-PublicConsumerBoundary.ps1 b/tools/Test-PublicConsumerBoundary.ps1 new file mode 100644 index 0000000..f03a8fd --- /dev/null +++ b/tools/Test-PublicConsumerBoundary.ps1 @@ -0,0 +1,21 @@ +<# +.SYNOPSIS +Checks that public-consumer fixtures use only the supported public surface. +.PARAMETER SourceDirectory +Source tree whose public-consumer fixtures are checked. +#> +[CmdletBinding()] +param([string]$SourceDirectory = '') + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +if (-not $SourceDirectory) { + $SourceDirectory = Split-Path -Parent $PSScriptRoot +} +$SourceDirectory = [System.IO.Path]::GetFullPath($SourceDirectory) +$cmake = (Get-Command cmake -ErrorAction Stop).Source +& $cmake "-DSOURCE_DIRECTORY=$SourceDirectory" -P ( + Join-Path (Split-Path -Parent $PSScriptRoot) 'cmake/CheckPublicConsumerBoundary.cmake') +if ($LASTEXITCODE -ne 0) { + throw 'Public-consumer boundary validation failed.' +} \ No newline at end of file diff --git a/tools/Test-ValidationPipeline.ps1 b/tools/Test-ValidationPipeline.ps1 new file mode 100644 index 0000000..0f8a9d7 --- /dev/null +++ b/tools/Test-ValidationPipeline.ps1 @@ -0,0 +1,475 @@ +<# +.SYNOPSIS +Runs focused validation-matrix, inventory, receipt, and no-rebuild regressions. +#> +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force + +$repositoryRoot = Get-PipelineRepositoryRoot +$cmake = (Get-Command cmake -ErrorAction Stop).Source +$ctest = (Get-Command ctest -ErrorAction Stop).Source +$matrixPath = Join-Path $PSScriptRoot 'validation-matrix.json' +$auditScript = Join-Path $repositoryRoot 'cmake/AuditValidationInventory.cmake' +$regressionRoot = Join-Path $repositoryRoot "out/pipeline/regression-$PID" + +<# +.SYNOPSIS +Imports one function definition without executing its owning script. +.PARAMETER Path +PowerShell script containing the function. +.PARAMETER Name +Function name to import into this script scope. +#> +function Import-ValidationFunction { + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][string]$Name + ) + + $tokens = $null + $errors = $null + $ast = [System.Management.Automation.Language.Parser]::ParseFile( + $Path, [ref]$tokens, [ref]$errors) + if ($errors.Count -ne 0) { + throw "Unable to parse $Path`: $($errors.Message -join '; ')" + } + $definitions = @($ast.FindAll({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -eq $Name + }, $true)) + if ($definitions.Count -ne 1) { + throw "Expected exactly one $Name definition in $Path" + } + Invoke-Expression "function script:$Name $($definitions[0].Body.Extent.Text)" +} + +<# +.SYNOPSIS +Writes one synthetic CTest JSON inventory. +.PARAMETER Path +Destination JSON path. +.PARAMETER Tests +Test objects containing name and owner. +#> +function Write-SyntheticTestInventory { + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Tests + ) + + $testEntries = @( + foreach ($test in $Tests) { + $labels = if ($test.Owner) { + @("SIMDLIB_OWNER_$($test.Owner)") + } else { + @('UNOWNED_TEST') + } + [ordered]@{ + name = [string]$test.Name + properties = @([ordered]@{ name = 'LABELS'; value = [object[]]@($labels) }) + } + } + ) + $document = [ordered]@{ + version = [ordered]@{ major = 1; minor = 0 } + tests = $testEntries + } + Set-PipelineTextFile -Path $Path -Content ( + $document | ConvertTo-Json -Depth 8) +} + +<# +.SYNOPSIS +Invokes the production inventory audit against a synthetic fixture. +.PARAMETER Name +Fixture name. +.PARAMETER TargetRows +Ownership rows excluding the TSV header. +.PARAMETER Tests +Synthetic test entries. +.PARAMETER ExpectFailure +Requires the audit to reject the fixture. +#> +function Invoke-InventoryFixture { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$TargetRows, + [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Tests, + [switch]$ExpectFailure + ) + + $fixtureRoot = Join-Path $regressionRoot "inventory-$Name" + New-Item -ItemType Directory -Path $fixtureRoot -Force | Out-Null + $ownership = "target`tcategory`towning_aggregate`tselected" + if ($TargetRows.Count) { + $ownership += "`n$($TargetRows -join "`n")" + } + Set-PipelineTextFile -Path ( + Join-Path $fixtureRoot 'development-target-ownership.tsv') ` + -Content "$ownership`n" + $testJson = Join-Path $fixtureRoot 'tests.json' + Write-SyntheticTestInventory -Path $testJson -Tests $Tests + $result = Join-Path $fixtureRoot 'result.json' + $arguments = @( + "-DMATRIX_FILE=$matrixPath", + '-DCELL_ID=msvc-release', + "-DBUILD_DIRECTORY=$fixtureRoot", + "-DCMAKE_CTEST_COMMAND=$ctest", + "-DTEST_JSON_FILE=$testJson", + "-DRESULT_FILE=$result", + '-P', $auditScript + ) + $logPath = Join-Path $fixtureRoot 'audit.log' + & $cmake @arguments *> $logPath + $failed = $LASTEXITCODE -ne 0 + if ($ExpectFailure -and -not $failed) { + throw "Inventory regression $Name was accepted unexpectedly" + } + if (-not $ExpectFailure -and $failed) { + throw "Inventory regression $Name failed unexpectedly: $((Get-Content -LiteralPath $logPath -Raw).Trim())" + } +} + +<# +.SYNOPSIS +Writes a receipt fixture and requires the production validator to reject it. +.PARAMETER Name +Fixture name. +.PARAMETER Receipt +Receipt document to validate. +.PARAMETER ExpectedPattern +Diagnostic pattern required from the rejection. +#> +function Assert-ReceiptRejected { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][object]$Receipt, + [Parameter(Mandatory)][string]$ExpectedPattern + ) + + Set-PipelineTextFile -Path $script:receiptPath -Content ( + $Receipt | ConvertTo-Json -Depth 8) + try { + [void](Assert-BuildReceipt -SelectedCompilers @('ClangCl')) + throw "Receipt regression $Name was accepted unexpectedly" + } catch { + if ($_.Exception.Message -notmatch $ExpectedPattern) { + throw "Receipt regression $Name emitted an unexpected diagnostic: $($_.Exception.Message)" + } + } +} + +try { + New-Item -ItemType Directory -Path $regressionRoot -Force | Out-Null + + $toolingFixtureRoot = Join-Path $regressionRoot 'tooling-digest' + $ownedToolingInputs = @(Get-PipelineToolingInputs -RepositoryRoot $repositoryRoot) + foreach ($input in $ownedToolingInputs) { + $destination = Join-Path $toolingFixtureRoot $input.RelativePath + New-Item -ItemType Directory -Path (Split-Path -Parent $destination) ` + -Force | Out-Null + Copy-Item -LiteralPath $input.FullName -Destination $destination + } + $baselineToolingDigest = Get-PipelineToolingDigest ` + -RepositoryRoot $toolingFixtureRoot + $productionFixture = Join-Path $toolingFixtureRoot 'include/SimdLib/Production.h' + New-Item -ItemType Directory -Path (Split-Path -Parent $productionFixture) ` + -Force | Out-Null + Set-PipelineTextFile -Path $productionFixture -Content '#pragma once' + $productionBaseline = Get-PipelineToolingDigest ` + -RepositoryRoot $toolingFixtureRoot + Set-PipelineTextFile -Path $productionFixture -Content '#pragma once // changed' + if ((Get-PipelineToolingDigest -RepositoryRoot $toolingFixtureRoot) -ne + $productionBaseline) { + throw 'An ordinary production-header change invalidated pipeline-tooling validation' + } + $inputClasses = @($ownedToolingInputs.Class | Select-Object -Unique) + foreach ($className in $inputClasses) { + $input = @($ownedToolingInputs | Where-Object Class -eq $className)[0] + $fixturePath = Join-Path $toolingFixtureRoot $input.RelativePath + $originalBytes = [System.IO.File]::ReadAllBytes($fixturePath) + try { + [System.IO.File]::AppendAllText($fixturePath, "`n", [Text.UTF8Encoding]::new($false)) + $changedDigest = Get-PipelineToolingDigest ` + -RepositoryRoot $toolingFixtureRoot + if ($changedDigest -eq $baselineToolingDigest) { + throw "Tooling-input class $className does not invalidate cached validation" + } + } finally { + [System.IO.File]::WriteAllBytes($fixturePath, $originalBytes) + } + } + + & (Join-Path $PSScriptRoot 'Test-PublicConsumerBoundary.ps1') + $boundaryFixture = Join-Path $regressionRoot 'public-consumer-boundary' + $forbiddenConsumer = Join-Path $boundaryFixture 'examples/Forbidden.cpp' + New-Item -ItemType Directory -Path (Split-Path -Parent $forbiddenConsumer) ` + -Force | Out-Null + Set-PipelineTextFile -Path $forbiddenConsumer -Content ( + '#include ') + $boundaryRejected = $false + try { + & (Join-Path $PSScriptRoot 'Test-PublicConsumerBoundary.ps1') ` + -SourceDirectory $boundaryFixture *> ( + Join-Path $boundaryFixture 'expected-rejection.log') + } catch { + $boundaryRejected = $true + } + if (-not $boundaryRejected) { + throw 'Public-consumer boundary accepted an implementation-detail include' + } + + Invoke-InventoryFixture -Name valid ` + -TargetRows @( + "RuntimeTarget`tRUNTIME_VALIDATION`tSimdLibRuntimeValidationArtifacts`tYES", + "BenchmarkTarget`tBENCHMARK`tBenchmarkArtifacts`tNO") ` + -Tests @( + [pscustomobject]@{ Name = 'Runtime.Case'; Owner = 'RUNTIME_VALIDATION' }, + [pscustomobject]@{ Name = 'Profile.Audit'; Owner = 'PROFILE_AUDIT' }) + Invoke-InventoryFixture -Name duplicate-target ` + -TargetRows @( + "RuntimeTarget`tRUNTIME_VALIDATION`tSimdLibRuntimeValidationArtifacts`tYES", + "RuntimeTarget`tRUNTIME_VALIDATION`tSimdLibRuntimeValidationArtifacts`tYES") ` + -Tests @() -ExpectFailure + Invoke-InventoryFixture -Name unexpected-target ` + -TargetRows @( + "DiagnosticTarget`tDEBUG_DIAGNOSTIC`tSimdLibDebugDiagnosticArtifacts`tYES") ` + -Tests @() -ExpectFailure + Invoke-InventoryFixture -Name unowned-test ` + -TargetRows @() ` + -Tests @([pscustomobject]@{ Name = 'Unowned.Case'; Owner = '' }) ` + -ExpectFailure + Invoke-InventoryFixture -Name duplicate-test ` + -TargetRows @() ` + -Tests @( + [pscustomobject]@{ Name = 'Duplicate.Case'; Owner = 'PROFILE_AUDIT' }, + [pscustomobject]@{ Name = 'Duplicate.Case'; Owner = 'PROFILE_AUDIT' }) ` + -ExpectFailure + Invoke-InventoryFixture -Name unexpected-test ` + -TargetRows @() ` + -Tests @([pscustomobject]@{ + Name = 'Diagnostic.Case' + Owner = 'DEBUG_DIAGNOSTIC' + }) ` + -ExpectFailure + + $runTestsPath = Join-Path $PSScriptRoot 'Run-Tests.ps1' + Import-ValidationFunction -Path $runTestsPath -Name Assert-BuildReceipt + $script:Scope = 'Native' + $script:pipelineRoot = Join-Path $regressionRoot 'receipt-pipeline' + New-Item -ItemType Directory -Path ( + Join-Path $script:pipelineRoot 'provenance') -Force | Out-Null + $sourceDigest = Get-PipelineSourceDigest -RepositoryRoot $repositoryRoot + $toolingDigest = Get-PipelineToolingDigest -RepositoryRoot $repositoryRoot + $pipelineValidationPath = Join-Path $regressionRoot 'pipeline-validation.json' + $pipelineValidationDocument = [ordered]@{ + schema = 'simdlib.pipeline-tooling-validation.v1' + status = 'complete' + toolingDigest = $toolingDigest + matrixSha256 = (Get-FileHash -LiteralPath $matrixPath -Algorithm SHA256).Hash.ToLowerInvariant() + } + Set-PipelineTextFile -Path $pipelineValidationPath -Content ( + $pipelineValidationDocument | ConvertTo-Json -Depth 4) + $inventoryAuditPath = Join-Path $regressionRoot 'inventory-audit.json' + Set-PipelineTextFile -Path $inventoryAuditPath -Content ( + '{"schema":"simdlib.validation-inventory-audit.v1","status":"complete","cell":"clangcl-release","profile":"RELEASE"}') + $manifestPath = Join-Path $regressionRoot 'validation-build.manifest' + $matrixHash = (Get-FileHash -LiteralPath $matrixPath -Algorithm SHA256).Hash.ToLowerInvariant() + $inventoryAuditHash = (Get-FileHash -LiteralPath $inventoryAuditPath -Algorithm SHA256).Hash.ToLowerInvariant() + $manifestLines = @( + 'schema=simdlib.build-manifest.v1', + 'operation=build-validation', + 'status=complete', + "source_digest=$sourceDigest", + 'preset=clangcl-release-exhaustive', + 'aggregate=ExhaustiveArtifacts', + 'matrix_cell=clangcl-release', + 'target_inventory_sha256=target-hash', + 'main_test_inventory_sha256=test-hash', + "matrix_contract_sha256=$matrixHash", + "validation_inventory_audit=$inventoryAuditPath", + "validation_inventory_audit_sha256=$inventoryAuditHash", + 'build_profile=Release', + 'sanitizer=none', + 'instrumentation=none', + 'codegen_mode=ENFORCE', + 'consumer_scope=core-register') + Set-PipelineTextFile -Path $manifestPath -Content ( + ($manifestLines -join "`n") + "`n") + $manifestHash = (Get-FileHash -LiteralPath $manifestPath -Algorithm SHA256).Hash.ToLowerInvariant() + $selectionId = (Get-PipelineTextDigest -Text 'Native|ClangCl').Substring(0, 16) + $script:receiptPath = Join-Path $script:pipelineRoot "provenance/build-$selectionId.json" + $receipt = [ordered]@{ + schema = 'simdlib.unified-build-receipt.v5' + status = 'complete' + scope = 'Native' + compilers = @('ClangCl') + sourceDigest = $sourceDigest + pipelineValidation = [ordered]@{ + path = [System.IO.Path]::GetRelativePath( + $repositoryRoot, $pipelineValidationPath).Replace('\', '/') + sha256 = (Get-FileHash -LiteralPath $pipelineValidationPath -Algorithm SHA256).Hash.ToLowerInvariant() + status = 'complete' + schema = 'simdlib.pipeline-tooling-validation.v1' + toolingDigest = $toolingDigest + } + manifests = @([ordered]@{ + preset = 'clangcl-release-exhaustive' + path = [System.IO.Path]::GetRelativePath( + $repositoryRoot, $manifestPath).Replace('\', '/') + sha256 = $manifestHash + sourceDigest = $sourceDigest + aggregate = 'ExhaustiveArtifacts' + matrixCell = 'clangcl-release' + targetInventorySha256 = 'target-hash' + testInventorySha256 = 'test-hash' + matrixContractSha256 = $matrixHash + inventoryAuditSha256 = $inventoryAuditHash + configuration = 'Release' + instrumentation = 'none' + generatedCodeMode = 'ENFORCE' + consumerScope = 'core-register' + }) + } + Set-PipelineTextFile -Path $script:receiptPath -Content ( + $receipt | ConvertTo-Json -Depth 8) + [void](Assert-BuildReceipt -SelectedCompilers @('ClangCl')) + + $containerAuditPath = '/workspace/out/' + ( + [System.IO.Path]::GetRelativePath( + (Join-Path $repositoryRoot 'out/pipeline'), + $inventoryAuditPath).Replace('\', '/')) + $containerManifestLines = @($manifestLines | ForEach-Object { + if ($_ -like 'validation_inventory_audit=*') { + "validation_inventory_audit=$containerAuditPath" + } else { + $_ + } + }) + Set-PipelineTextFile -Path $manifestPath -Content ( + ($containerManifestLines -join "`n") + "`n") + $receipt.manifests[0].sha256 = ( + Get-FileHash -LiteralPath $manifestPath -Algorithm SHA256 + ).Hash.ToLowerInvariant() + Set-PipelineTextFile -Path $script:receiptPath -Content ( + $receipt | ConvertTo-Json -Depth 8) + [void](Assert-BuildReceipt -SelectedCompilers @('ClangCl')) + + Set-PipelineTextFile -Path $manifestPath -Content ( + ($manifestLines -join "`n") + "`n") + $receipt.manifests[0].sha256 = ( + Get-FileHash -LiteralPath $manifestPath -Algorithm SHA256 + ).Hash.ToLowerInvariant() + Set-PipelineTextFile -Path $script:receiptPath -Content ( + $receipt | ConvertTo-Json -Depth 8) + + $case = $receipt | ConvertTo-Json -Depth 8 | ConvertFrom-Json + $case.sourceDigest = 'stale' + Assert-ReceiptRejected -Name stale -Receipt $case ` + -ExpectedPattern 'stale' + $case = $receipt | ConvertTo-Json -Depth 8 | ConvertFrom-Json + $case.status = 'building' + Assert-ReceiptRejected -Name incomplete -Receipt $case ` + -ExpectedPattern 'incomplete|incompatible' + $case = $receipt | ConvertTo-Json -Depth 8 | ConvertFrom-Json + $case.compilers = @('Msvc') + Assert-ReceiptRejected -Name mismatched-compiler -Receipt $case ` + -ExpectedPattern 'compiler set' + $case = $receipt | ConvertTo-Json -Depth 8 | ConvertFrom-Json + $case.manifests = @() + Assert-ReceiptRejected -Name mismatched-cells -Receipt $case ` + -ExpectedPattern 'manifest set' + $case = $receipt | ConvertTo-Json -Depth 8 | ConvertFrom-Json + $case.manifests[0].aggregate = 'BenchmarkArtifacts' + Assert-ReceiptRejected -Name category-incompatible -Receipt $case ` + -ExpectedPattern 'provenance aggregate' + $case = $receipt | ConvertTo-Json -Depth 8 | ConvertFrom-Json + $case.manifests[0].inventoryAuditSha256 = 'none' + Assert-ReceiptRejected -Name missing-inventory-audit -Receipt $case ` + -ExpectedPattern 'validation_inventory_audit_sha256' + $case = $receipt | ConvertTo-Json -Depth 8 | ConvertFrom-Json + $case.pipelineValidation = $null + Assert-ReceiptRejected -Name missing-pipeline-validation -Receipt $case ` + -ExpectedPattern 'pipeline-tooling validation' + $case = $receipt | ConvertTo-Json -Depth 8 | ConvertFrom-Json + $case.pipelineValidation.toolingDigest = 'stale' + Assert-ReceiptRejected -Name stale-pipeline-validation -Receipt $case ` + -ExpectedPattern 'pipeline-tooling validation' + + $originalValidationResult = Get-Content -LiteralPath $pipelineValidationPath -Raw + try { + Set-PipelineTextFile -Path $pipelineValidationPath -Content '{"modified":true}' + Assert-ReceiptRejected -Name modified-pipeline-validation -Receipt $receipt ` + -ExpectedPattern 'changed after the unified build' + } finally { + Set-PipelineTextFile -Path $pipelineValidationPath ` + -Content $originalValidationResult + } + $originalManifest = Get-Content -LiteralPath $manifestPath -Raw + try { + Set-PipelineTextFile -Path $manifestPath -Content ($originalManifest + '# modified') + Assert-ReceiptRejected -Name modified-manifest -Receipt $receipt ` + -ExpectedPattern 'manifest changed after the unified build' + } finally { + Set-PipelineTextFile -Path $manifestPath -Content $originalManifest + } + + $runTestsSource = Get-Content -LiteralPath $runTestsPath -Raw + if ($runTestsSource -match "(?i)&\s*\(Join-Path[^\r\n]*Build\.ps1|--build|'-Action',\s*'Build'") { + throw 'Run-Tests contains a configure or build dispatch' + } + if (@([regex]::Matches( + $runTestsSource, "'-Action',\s*'Test'")).Count -lt 2) { + throw 'Run-Tests does not dispatch both native and container test-only operations' + } + + $nativeSource = Get-Content -LiteralPath ( + Join-Path $PSScriptRoot 'Run-NativeMatrix.ps1') -Raw + if ($nativeSource -notmatch + '(?s)function Build-NativeBenchmarks.+Assert-NativeManifest.+build-validation.+--target.+BenchmarkArtifacts') { + throw 'Native benchmarks do not require and reuse the owning validation tree' + } + $containerSource = Get-Content -LiteralPath ( + Join-Path $repositoryRoot 'containers/container-entrypoint.sh') -Raw + if ($containerSource -notmatch + '(?s)build-benchmarks\).+can_reuse_validation_configuration.+Reusing validated Release configuration') { + throw 'Container benchmarks do not require and reuse the owning validation tree' + } + + $buildSource = Get-Content -LiteralPath ( + Join-Path $PSScriptRoot 'Build.ps1') -Raw + if (@([regex]::Matches( + $buildSource, 'Validate-PipelineTooling\.ps1')).Count -ne 1 -or + @([regex]::Matches( + $buildSource, 'Test-PublicConsumerBoundary\.ps1')).Count -ne 1 -or + $buildSource -notmatch 'pipelineValidation\s*=\s*\$pipelineValidationEntry') { + throw 'Unified build does not run and bind the focused pre-cell validations' + } + $validationSource = Get-Content -LiteralPath ( + Join-Path $PSScriptRoot 'Validate-PipelineTooling.ps1') -Raw + if (@([regex]::Matches( + $validationSource, + 'if \(-not \(Test-CurrentPipelineValidation\)\)')).Count -ne 2) { + throw 'Pipeline-tooling validation no longer has one cache guard plus one completion guard' + } + + Write-Host ( + 'Validation pipeline regressions passed: inventory ownership, tooling-digest ' + + 'invalidation, public-consumer rejection, receipt tamper detection, and ' + + 'no-rebuild ownership checks.') +} finally { + $resolvedRegressionRoot = [System.IO.Path]::GetFullPath($regressionRoot) + $resolvedPipelineRoot = [System.IO.Path]::GetFullPath( + (Join-Path $repositoryRoot 'out/pipeline')) + if ($resolvedRegressionRoot.StartsWith( + $resolvedPipelineRoot + [System.IO.Path]::DirectorySeparatorChar, + [System.StringComparison]::OrdinalIgnoreCase) -and + (Test-Path -LiteralPath $resolvedRegressionRoot)) { + Remove-Item -LiteralPath $resolvedRegressionRoot -Recurse -Force + } +} diff --git a/tools/Validate-PipelineTooling.ps1 b/tools/Validate-PipelineTooling.ps1 new file mode 100644 index 0000000..c834348 --- /dev/null +++ b/tools/Validate-PipelineTooling.ps1 @@ -0,0 +1,53 @@ +<# +.SYNOPSIS +Validates pipeline tooling once per reviewed tooling/configuration digest. +.PARAMETER ResultPath +Optional explicit machine-readable result path. +#> +[CmdletBinding()] +param([string]$ResultPath = '') + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force +$repositoryRoot = Get-PipelineRepositoryRoot +$toolingDigest = Get-PipelineToolingDigest -RepositoryRoot $repositoryRoot +if (-not $ResultPath) { + $ResultPath = Join-Path $repositoryRoot ( + "out/pipeline/provenance/pipeline-validation-$($toolingDigest.Substring(0, 16)).json") +} +$ResultPath = [System.IO.Path]::GetFullPath($ResultPath) + +<# +.SYNOPSIS +Returns whether an existing result owns the current tooling digest. +#> +function Test-CurrentPipelineValidation { + if (-not (Test-Path -LiteralPath $ResultPath -PathType Leaf)) { return $false } + try { + $result = Get-Content -LiteralPath $ResultPath -Raw | ConvertFrom-Json + return $result.schema -eq 'simdlib.pipeline-tooling-validation.v1' -and + $result.status -eq 'complete' -and + $result.toolingDigest -eq $toolingDigest + } catch { + return $false + } +} + +if (-not (Test-CurrentPipelineValidation)) { + & (Join-Path $PSScriptRoot 'Verify-ValidationMatrix.ps1') + & (Join-Path $PSScriptRoot 'Test-ValidationPipeline.ps1') + $matrixPath = Join-Path $PSScriptRoot 'validation-matrix.json' + $document = [ordered]@{ + schema = 'simdlib.pipeline-tooling-validation.v1' + status = 'complete' + toolingDigest = $toolingDigest + matrixSha256 = (Get-FileHash -LiteralPath $matrixPath -Algorithm SHA256).Hash.ToLowerInvariant() + } + Set-PipelineTextFile -Path $ResultPath -Content ( + $document | ConvertTo-Json -Depth 4) +} +if (-not (Test-CurrentPipelineValidation)) { + throw "Pipeline-tooling validation did not produce a current result: $ResultPath" +} +Write-Host "Pipeline-tooling validation result: $ResultPath" \ No newline at end of file diff --git a/tools/Verify-ValidationMatrix.ps1 b/tools/Verify-ValidationMatrix.ps1 new file mode 100644 index 0000000..2267b17 --- /dev/null +++ b/tools/Verify-ValidationMatrix.ps1 @@ -0,0 +1,371 @@ +<# +.SYNOPSIS +Verifies validation-matrix topology and its pipeline integrations. +#> +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'Pipeline.Common.psm1') -Force +$repositoryRoot = Get-PipelineRepositoryRoot +$matrix = Get-PipelineValidationMatrix -RepositoryRoot $repositoryRoot + +<# +.SYNOPSIS +Imports one function definition without executing its owning script. +.PARAMETER Path +PowerShell script containing the function. +.PARAMETER Name +Function name to import into this verifier's script scope. +#> +function Import-MatrixResolver { + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][string]$Name + ) + + $tokens = $null + $errors = $null + $ast = [System.Management.Automation.Language.Parser]::ParseFile( + $Path, [ref]$tokens, [ref]$errors) + if ($errors.Count -ne 0) { + throw "Unable to parse $Path`: $($errors.Message -join '; ')" + } + $definitions = @($ast.FindAll({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -eq $Name + }, $true)) + if ($definitions.Count -ne 1) { + throw "Expected exactly one $Name definition in $Path" + } + Invoke-Expression "function script:$Name $($definitions[0].Body.Extent.Text)" +} + +<# +.SYNOPSIS +Rejects duplicate values and returns an ordinal set. +.PARAMETER Name +Human-readable collection name. +.PARAMETER Values +Values required to be unique. +#> +function New-UniqueSet { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Values + ) + + $set = [System.Collections.Generic.HashSet[string]]::new( + [System.StringComparer]::Ordinal) + foreach ($value in $Values) { + if (-not $set.Add([string]$value)) { + throw "$Name duplicates $value" + } + } + return ,$set +} + +<# +.SYNOPSIS +Rejects two ownership collections that differ as ordinal sets. +.PARAMETER Name +Human-readable ownership name. +.PARAMETER Actual +Observed values. +.PARAMETER Expected +Required values. +#> +function Assert-SetEqual { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Actual, + [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Expected + ) + + $actualSet = New-UniqueSet -Name "$Name actual" -Values $Actual + $expectedSet = New-UniqueSet -Name "$Name expected" -Values $Expected + if (-not $actualSet.SetEquals($expectedSet)) { + throw "$Name mismatch. Expected '$(@($expectedSet) -join ', ')'; received '$(@($actualSet) -join ', ')'" + } +} + +<# +.SYNOPSIS +Rejects a sequence that differs from its matrix-owned execution order. +.PARAMETER Name +Human-readable sequence name. +.PARAMETER Actual +Observed sequence. +.PARAMETER Expected +Required matrix sequence. +#> +function Assert-SequenceEqual { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Actual, + [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Expected + ) + + if ((@($Actual) -join '|') -ne (@($Expected) -join '|')) { + throw "$Name execution order differs from validation-matrix.json" + } +} + +<# +.SYNOPSIS +Resolves one inherited configure-preset cache value. +.PARAMETER Name +Configure preset name. +.PARAMETER Variable +CMake cache variable to resolve. +.PARAMETER Presets +Configure-preset dictionary. +#> +function Get-ResolvedPresetValue { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string]$Variable, + [Parameter(Mandatory)][hashtable]$Presets + ) + + $visited = [System.Collections.Generic.HashSet[string]]::new() + <# + .SYNOPSIS + Resolves the requested value from one preset and its inherited parents. + .PARAMETER PresetName + Configure preset currently being inspected. + #> + function Resolve-OnePresetValue { + param([Parameter(Mandatory)][string]$PresetName) + if (-not $visited.Add($PresetName)) { return $null } + $preset = $Presets[$PresetName] + if (-not $preset) { throw "Configure preset inheritance references missing preset $PresetName" } + $cache = $preset.PSObject.Properties['cacheVariables'] + if ($cache -and $cache.Value.PSObject.Properties[$Variable]) { + return [string]$cache.Value.$Variable + } + $inherits = $preset.PSObject.Properties['inherits'] + if ($inherits) { + foreach ($parent in @($inherits.Value)) { + $value = Resolve-OnePresetValue -PresetName ([string]$parent) + if ($null -ne $value) { return $value } + } + } + return $null + } + return Resolve-OnePresetValue -PresetName $Name +} + +$categories = New-UniqueSet -Name 'targetCategories' -Values @($matrix.targetCategories) +$testOnlyOwners = New-UniqueSet -Name 'testOnlyOwners' -Values @($matrix.testOnlyOwners) +[void](New-UniqueSet -Name 'compilerOrder' -Values @($matrix.compilerOrder)) +foreach ($profileProperty in $matrix.profiles.PSObject.Properties) { + $profileName = $profileProperty.Name + $profile = $profileProperty.Value + $allowed = New-UniqueSet -Name "$profileName allowedTargetCategories" ` + -Values @($profile.allowedTargetCategories) + $selected = New-UniqueSet -Name "$profileName selectedTargetCategories" ` + -Values @($profile.selectedTargetCategories) + if (-not $selected.IsSubsetOf($allowed)) { + throw "$profileName selects a target category it does not allow" + } + foreach ($category in $allowed) { + if (-not $categories.Contains($category)) { + throw "$profileName references unknown target category $category" + } + } + [void](New-UniqueSet -Name "$profileName allowedTestOwners" ` + -Values @($profile.allowedTestOwners)) + foreach ($owner in @($profile.allowedTestOwners)) { + if (-not $categories.Contains([string]$owner) -and + -not $testOnlyOwners.Contains([string]$owner)) { + throw "$profileName references unknown test owner $owner" + } + } +} + +$operationCells = @{} +foreach ($operation in $matrix.operations.PSObject.Properties) { + $operationCells[$operation.Name] = @( + Get-PipelineValidationOperationCells -Operation $operation.Name) +} +$defaultBuild = @($operationCells.defaultBuild) +$defaultTests = @($operationCells.defaultTests) +Assert-SetEqual -Name 'Default build and test ownership' ` + -Actual @($defaultTests.MatrixCell) -Expected @($defaultBuild.MatrixCell) +foreach ($cell in @($operationCells.optionalDebug)) { + if ($cell.MatrixCell -in @($defaultBuild.MatrixCell)) { + throw "Ordinary opt-in Debug cell enters the default operation: $($cell.MatrixCell)" + } +} +$forbiddenInstrumentedCategories = @( + 'COMPILER_CONTRACT', 'CONSTEXPR_CONTRACT', 'OPTIMIZED_CODEGEN', + 'SMOKE_VALIDATION', 'DEBUG_DIAGNOSTIC') +foreach ($profileName in @('SANITIZER', 'COVERAGE')) { + $profile = $matrix.profiles.$profileName + $forbidden = @(@($profile.allowedTargetCategories) | + Where-Object { $_ -in $forbiddenInstrumentedCategories }) + if ($forbidden.Count -ne 0) { + throw "$profileName permits forbidden categories: $($forbidden -join ', ')" + } +} + +$releaseCompilerIdentities = @($matrix.cells.PSObject.Properties | + Where-Object { $_.Value.profile -eq 'RELEASE' -and $_.Value.instrumentation -eq 'none' } | + ForEach-Object { $_.Value.compilerIdentity } | Select-Object -Unique) +$contractIdentities = @($operationCells.compilerContracts.compilerIdentity) +Assert-SetEqual -Name 'Compiler-contract ownership' ` + -Actual $contractIdentities -Expected $releaseCompilerIdentities +foreach ($identity in $releaseCompilerIdentities) { + if (@($contractIdentities | Where-Object { $_ -eq $identity }).Count -ne 1) { + throw "Compiler identity $identity does not have exactly one compiler-contract owner" + } +} +foreach ($cellProperty in $matrix.cells.PSObject.Properties) { + $cellId = $cellProperty.Name + $cell = $cellProperty.Value + if (-not $matrix.profiles.PSObject.Properties[[string]$cell.profile]) { + throw "Validation cell references unknown profile $($cell.profile)" + } + if ($cell.profile -eq 'RELEASE' -and $cell.registerCapable -and + $cell.codegenMode -ne 'ENFORCE') { + throw "Register-capable Release cell does not enforce generated code: $($cell.preset)" + } + if ($cell.consumer -and ($cell.profile -ne 'RELEASE' -or + $cellId -notin @($defaultBuild.MatrixCell))) { + throw "Consumer ownership is not isolated to a default Release cell: $($cell.preset)" + } +} +foreach ($cell in @($operationCells.optionalDiagnostics)) { + if ($cell.codegenMode -ne 'RECORD' -or + $cell.MatrixCell -in @($defaultBuild.MatrixCell)) { + throw "Optional diagnostic is not isolated record-only evidence: $($cell.MatrixCell)" + } +} +foreach ($cell in @($operationCells.benchmarks)) { + if ($cell.profile -ne 'RELEASE' -or $cell.configuration -ne 'Release' -or + $cell.aggregate -ne 'ExhaustiveArtifacts') { + throw "Benchmark operation does not reuse an owning Release configuration: $($cell.MatrixCell)" + } +} + +$presetPath = Join-Path $repositoryRoot 'CMakePresets.json' +$presetDocument = Get-Content -LiteralPath $presetPath -Raw | ConvertFrom-Json +$presetByName = @{} +foreach ($preset in $presetDocument.configurePresets) { + if ($presetByName.ContainsKey($preset.name)) { throw "Duplicate configure preset: $($preset.name)" } + $presetByName[$preset.name] = $preset +} +$buildPresetByName = @{} +foreach ($preset in $presetDocument.buildPresets) { + if ($buildPresetByName.ContainsKey($preset.name)) { throw "Duplicate build preset: $($preset.name)" } + $buildPresetByName[$preset.name] = $preset +} +foreach ($cellProperty in $matrix.cells.PSObject.Properties) { + $cellId = $cellProperty.Name + $cell = $cellProperty.Value + if (-not $presetByName.ContainsKey([string]$cell.preset)) { + throw "Validation cell $cellId references missing configure preset $($cell.preset)" + } + $profile = Get-ResolvedPresetValue -Name $cell.preset ` + -Variable SIMDLIB_VALIDATION_PROFILE -Presets $presetByName + if ($profile -ne $cell.profile) { + throw "Preset $($cell.preset) resolves profile $profile instead of $($cell.profile)" + } + $configuration = Get-ResolvedPresetValue -Name $cell.preset ` + -Variable CMAKE_BUILD_TYPE -Presets $presetByName + if (-not $configuration) { + $configuration = Get-ResolvedPresetValue -Name $cell.preset ` + -Variable CMAKE_CONFIGURATION_TYPES -Presets $presetByName + } + if ($configuration -ne $cell.configuration) { + throw "Preset $($cell.preset) resolves configuration $configuration instead of $($cell.configuration)" + } + $codegenMode = Get-ResolvedPresetValue -Name $cell.preset ` + -Variable SIMDLIB_REGISTER_CODEGEN_MODE -Presets $presetByName + if ($codegenMode -ne $cell.codegenMode) { + throw "Preset $($cell.preset) resolves generated-code mode $codegenMode instead of $($cell.codegenMode)" + } + if ($cell.instrumentation -eq 'asan-ubsan') { + $sanitizerFlags = Get-ResolvedPresetValue -Name $cell.preset ` + -Variable CMAKE_CXX_FLAGS_DEBUG -Presets $presetByName + if ($sanitizerFlags -notmatch '-fsanitize=address,undefined') { + throw "Preset $($cell.preset) does not resolve ASan and UBSan instrumentation" + } + } elseif ($cell.instrumentation -eq 'coverage') { + if ((Get-ResolvedPresetValue -Name $cell.preset ` + -Variable SIMDLIB_ENABLE_COVERAGE -Presets $presetByName) -ne 'ON') { + throw "Preset $($cell.preset) does not resolve coverage instrumentation" + } + } + $buildPreset = $buildPresetByName[[string]$cell.preset] + if ($buildPreset -and ($buildPreset.configurePreset -ne $cell.preset -or + @($buildPreset.targets) -notcontains $cell.aggregate)) { + throw "Build preset $($cell.preset) disagrees with cell aggregate $($cell.aggregate)" + } +} +foreach ($cell in @($operationCells.benchmarks)) { + $benchmarkPreset = @($presetDocument.buildPresets | Where-Object { + $_.configurePreset -eq $cell.preset -and + @($_.targets) -contains 'BenchmarkArtifacts' + }) + if ($benchmarkPreset.Count -ne 1) { + throw "Release cell $($cell.MatrixCell) does not have exactly one benchmark aggregate preset" + } +} + +$artifactAggregates = Get-Content -LiteralPath ( + Join-Path $repositoryRoot 'cmake/development/ArtifactAggregates.cmake') -Raw +if ($artifactAggregates -notmatch 'file\(READ "\$\{simdlib_validation_matrix\}"' -or + $artifactAggregates -match 'simdlib_profile_allowed_RELEASE\s') { + throw 'CMake development profiles do not consume validation-matrix.json directly' +} + +Import-MatrixResolver -Path (Join-Path $PSScriptRoot 'Run-NativeMatrix.ps1') ` + -Name Resolve-NativeCells +Import-MatrixResolver -Path (Join-Path $PSScriptRoot 'Run-ContainerMatrix.ps1') ` + -Name Resolve-Cells +$services = @(Get-PipelineValidationCompilers -Platform container | + ForEach-Object { $_.ToLowerInvariant() }) +$resolverCases = @( + [pscustomobject]@{ Name='native default'; Actual=@(Resolve-NativeCells -CompilerName All -CellScope All -Operation Build); Expected=@($defaultBuild | Where-Object platform -eq native) }, + [pscustomobject]@{ Name='container default'; Actual=@(Resolve-Cells -Services $services -CellScope All -Operation Build); Expected=@($defaultBuild | Where-Object platform -eq container) }, + [pscustomobject]@{ Name='native benchmarks'; Actual=@(Resolve-NativeCells -CompilerName All -CellScope Release -Operation BuildBenchmarks); Expected=@($operationCells.benchmarks | Where-Object platform -eq native) }, + [pscustomobject]@{ Name='container benchmarks'; Actual=@(Resolve-Cells -Services $services -CellScope Release -Operation BuildBenchmarks); Expected=@($operationCells.benchmarks | Where-Object platform -eq container) }, + [pscustomobject]@{ Name='native compiler contracts'; Actual=@(Resolve-NativeCells -CompilerName All -CellScope Release -Operation BuildCompilerContracts); Expected=@($operationCells.compilerContracts | Where-Object platform -eq native) }, + [pscustomobject]@{ Name='container compiler contracts'; Actual=@(Resolve-Cells -Services $services -CellScope Release -Operation BuildCompilerContracts); Expected=@($operationCells.compilerContracts | Where-Object platform -eq container) }, + [pscustomobject]@{ Name='native diagnostics'; Actual=@(Resolve-NativeCells -CompilerName All -CellScope Debug -Operation RecordCodegen); Expected=@($operationCells.optionalDiagnostics | Where-Object platform -eq native) }, + [pscustomobject]@{ Name='container diagnostics'; Actual=@(Resolve-Cells -Services $services -CellScope All -Operation RecordCodegen); Expected=@($operationCells.optionalDiagnostics | Where-Object platform -eq container) } +) +foreach ($case in $resolverCases) { + Assert-SequenceEqual -Name $case.Name -Actual @($case.Actual.MatrixCell) ` + -Expected @($case.Expected.MatrixCell) + foreach ($resolved in $case.Actual) { + $canonical = $matrix.cells.PSObject.Properties[[string]$resolved.MatrixCell].Value + if ($resolved.Preset -ne $canonical.preset -or + $resolved.BuildProfile -ne $canonical.configuration -or + $resolved.Instrumentation -ne $canonical.instrumentation -or + $resolved.Aggregate -ne $canonical.aggregate -or + $resolved.CodegenMode -ne $canonical.codegenMode -or + $resolved.Consumer -ne $canonical.consumer) { + throw "$($case.Name) resolver disagrees with cell $($resolved.MatrixCell)" + } + } +} + +$compose = Get-Content -LiteralPath (Join-Path $repositoryRoot 'compose.yml') -Raw +$contractPresets = @($operationCells.compilerContracts | + Where-Object platform -eq container | ForEach-Object preset | Select-Object -Unique) +if ($contractPresets.Count -ne 1 -or + $compose -notmatch [regex]::Escape("SIMDLIB_CONTAINER_PRESET:-$($contractPresets[0])")) { + throw 'Docker Compose does not select the matrix-owned container compiler-contract operation' +} +$runTestsSource = Get-Content -LiteralPath (Join-Path $PSScriptRoot 'Run-Tests.ps1') -Raw +if ($runTestsSource -match '(?i)&\s*\(Join-Path[^\r\n]*Build\.ps1|--build|''-Action'',\s*''Build''|cmake\s+--preset') { + throw 'Run-Tests contains a configure or build path' +} + +[void](Get-PipelineToolingInputs -RepositoryRoot $repositoryRoot) +Write-Host "Validation matrix invariants passed for $(@($matrix.cells.PSObject.Properties).Count) cells and $(@($matrix.operations.PSObject.Properties).Count) operations." \ No newline at end of file diff --git a/tools/validation-matrix.json b/tools/validation-matrix.json new file mode 100644 index 0000000..8096e2d --- /dev/null +++ b/tools/validation-matrix.json @@ -0,0 +1,538 @@ +{ + "schema": "simdlib.validation-matrix.v1", + "targetCategories": [ + "COMPILER_CONTRACT", + "CONSTEXPR_CONTRACT", + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION", + "SMOKE_VALIDATION", + "OPTIMIZED_CODEGEN", + "DEBUG_DIAGNOSTIC", + "COVERAGE_SUPPORT", + "BENCHMARK" + ], + "testOnlyOwners": [ + "PROFILE_AUDIT" + ], + "profiles": { + "RELEASE": { + "allowedTargetCategories": [ + "COMPILER_CONTRACT", + "CONSTEXPR_CONTRACT", + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION", + "SMOKE_VALIDATION", + "OPTIMIZED_CODEGEN", + "BENCHMARK" + ], + "selectedTargetCategories": [ + "COMPILER_CONTRACT", + "CONSTEXPR_CONTRACT", + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION", + "SMOKE_VALIDATION", + "OPTIMIZED_CODEGEN" + ], + "allowedTestOwners": [ + "COMPILER_CONTRACT", + "CONSTEXPR_CONTRACT", + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION", + "SMOKE_VALIDATION", + "OPTIMIZED_CODEGEN", + "PROFILE_AUDIT" + ] + }, + "DEBUG": { + "allowedTargetCategories": [ + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION" + ], + "selectedTargetCategories": [ + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION" + ], + "allowedTestOwners": [ + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION", + "PROFILE_AUDIT" + ] + }, + "SANITIZER": { + "allowedTargetCategories": [ + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION" + ], + "selectedTargetCategories": [ + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION" + ], + "allowedTestOwners": [ + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION", + "PROFILE_AUDIT" + ] + }, + "COVERAGE": { + "allowedTargetCategories": [ + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION", + "COVERAGE_SUPPORT" + ], + "selectedTargetCategories": [ + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION" + ], + "allowedTestOwners": [ + "RUNTIME_VALIDATION", + "CHECKS_VALIDATION", + "PROFILE_AUDIT" + ] + }, + "COMPILER_CONTRACTS": { + "allowedTargetCategories": [ + "COMPILER_CONTRACT" + ], + "selectedTargetCategories": [ + "COMPILER_CONTRACT" + ], + "allowedTestOwners": [ + "COMPILER_CONTRACT", + "PROFILE_AUDIT" + ] + }, + "CODEGEN_DIAGNOSTIC": { + "allowedTargetCategories": [ + "DEBUG_DIAGNOSTIC" + ], + "selectedTargetCategories": [ + "DEBUG_DIAGNOSTIC" + ], + "allowedTestOwners": [ + "DEBUG_DIAGNOSTIC", + "PROFILE_AUDIT" + ] + } + }, + "cells": { + "msvc-release": { + "platform": "native", + "compiler": "Msvc", + "compilerIdentity": "msvc", + "preset": "msvc-release-exhaustive", + "profile": "RELEASE", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "ENFORCE", + "aggregate": "ExhaustiveArtifacts", + "consumer": true, + "registerCapable": true, + "artifactKey": "release", + "generator": "Visual Studio 17 2022" + }, + "msvc-debug": { + "platform": "native", + "compiler": "Msvc", + "compilerIdentity": "msvc", + "preset": "msvc-debug-diagnostics", + "profile": "DEBUG", + "configuration": "Debug", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "ExhaustiveArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug", + "generator": "Visual Studio 17 2022" + }, + "clangcl-release": { + "platform": "native", + "compiler": "ClangCl", + "compilerIdentity": "clangcl", + "preset": "clangcl-release-exhaustive", + "profile": "RELEASE", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "ENFORCE", + "aggregate": "ExhaustiveArtifacts", + "consumer": true, + "registerCapable": true, + "artifactKey": "release", + "generator": "Ninja" + }, + "clangcl-debug": { + "platform": "native", + "compiler": "ClangCl", + "compilerIdentity": "clangcl", + "preset": "clangcl-debug-diagnostics", + "profile": "DEBUG", + "configuration": "Debug", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "ExhaustiveArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug", + "generator": "Ninja" + }, + "clang-coverage": { + "platform": "native", + "compiler": "ClangCoverage", + "compilerIdentity": "clang-coverage", + "preset": "clang-debug-coverage", + "profile": "COVERAGE", + "configuration": "Debug", + "instrumentation": "coverage", + "codegenMode": "OFF", + "aggregate": "ExhaustiveArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug-coverage", + "generator": "Ninja" + }, + "gcc13-release": { + "platform": "container", + "compiler": "Gcc13", + "compilerIdentity": "gcc13", + "preset": "gcc13-core-release-exhaustive", + "profile": "RELEASE", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "ExhaustiveArtifacts", + "consumer": true, + "registerCapable": false, + "artifactKey": "release" + }, + "gcc13-debug": { + "platform": "container", + "compiler": "Gcc13", + "compilerIdentity": "gcc13", + "preset": "gcc13-core-debug-diagnostics", + "profile": "DEBUG", + "configuration": "Debug", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "ExhaustiveArtifacts", + "consumer": false, + "registerCapable": false, + "artifactKey": "debug" + }, + "gcc14-release": { + "platform": "container", + "compiler": "Gcc14", + "compilerIdentity": "gcc14", + "preset": "gcc14-release-exhaustive", + "profile": "RELEASE", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "ENFORCE", + "aggregate": "ExhaustiveArtifacts", + "consumer": true, + "registerCapable": true, + "artifactKey": "release" + }, + "gcc14-debug": { + "platform": "container", + "compiler": "Gcc14", + "compilerIdentity": "gcc14", + "preset": "gcc14-debug-diagnostics", + "profile": "DEBUG", + "configuration": "Debug", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "ExhaustiveArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug" + }, + "clang22-release": { + "platform": "container", + "compiler": "Clang22", + "compilerIdentity": "clang22", + "preset": "clang22-release-exhaustive", + "profile": "RELEASE", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "ENFORCE", + "aggregate": "ExhaustiveArtifacts", + "consumer": true, + "registerCapable": true, + "artifactKey": "release" + }, + "clang22-debug": { + "platform": "container", + "compiler": "Clang22", + "compilerIdentity": "clang22", + "preset": "clang22-debug-diagnostics", + "profile": "DEBUG", + "configuration": "Debug", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "ExhaustiveArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug" + }, + "clang22-sanitizer": { + "platform": "container", + "compiler": "Clang22", + "compilerIdentity": "clang22", + "preset": "clang22-debug-asan-ubsan", + "profile": "SANITIZER", + "configuration": "Debug", + "instrumentation": "asan-ubsan", + "codegenMode": "OFF", + "aggregate": "ExhaustiveArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug-asan-ubsan" + }, + "msvc-contracts": { + "platform": "native", + "compiler": "Msvc", + "compilerIdentity": "msvc", + "preset": "msvc-compiler-contracts", + "profile": "COMPILER_CONTRACTS", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "SimdLibCompilerContractArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "compiler-contracts", + "generator": "Visual Studio 17 2022" + }, + "clangcl-contracts": { + "platform": "native", + "compiler": "ClangCl", + "compilerIdentity": "clangcl", + "preset": "clangcl-compiler-contracts", + "profile": "COMPILER_CONTRACTS", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "SimdLibCompilerContractArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "compiler-contracts", + "generator": "Ninja" + }, + "gcc13-contracts": { + "platform": "container", + "compiler": "Gcc13", + "compilerIdentity": "gcc13", + "preset": "container-release-contracts", + "profile": "COMPILER_CONTRACTS", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "SimdLibCompilerContractArtifacts", + "consumer": false, + "registerCapable": false, + "artifactKey": "compiler-contracts" + }, + "gcc14-contracts": { + "platform": "container", + "compiler": "Gcc14", + "compilerIdentity": "gcc14", + "preset": "container-release-contracts", + "profile": "COMPILER_CONTRACTS", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "SimdLibCompilerContractArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "compiler-contracts" + }, + "clang22-contracts": { + "platform": "container", + "compiler": "Clang22", + "compilerIdentity": "clang22", + "preset": "container-release-contracts", + "profile": "COMPILER_CONTRACTS", + "configuration": "Release", + "instrumentation": "none", + "codegenMode": "OFF", + "aggregate": "SimdLibCompilerContractArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "compiler-contracts" + }, + "msvc-diagnostic": { + "platform": "native", + "compiler": "Msvc", + "compilerIdentity": "msvc", + "preset": "msvc-debug-codegen-diagnostic", + "profile": "CODEGEN_DIAGNOSTIC", + "configuration": "Debug", + "instrumentation": "none", + "codegenMode": "RECORD", + "aggregate": "SimdLibDebugDiagnosticArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug-codegen", + "generator": "Ninja" + }, + "clangcl-diagnostic": { + "platform": "native", + "compiler": "ClangCl", + "compilerIdentity": "clangcl", + "preset": "clangcl-debug-codegen-diagnostic", + "profile": "CODEGEN_DIAGNOSTIC", + "configuration": "Debug", + "instrumentation": "none", + "codegenMode": "RECORD", + "aggregate": "SimdLibDebugDiagnosticArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug-codegen", + "generator": "Ninja" + }, + "gcc14-diagnostic": { + "platform": "container", + "compiler": "Gcc14", + "compilerIdentity": "gcc14", + "preset": "gcc14-debug-codegen-diagnostic", + "profile": "CODEGEN_DIAGNOSTIC", + "configuration": "Debug", + "instrumentation": "none", + "codegenMode": "RECORD", + "aggregate": "SimdLibDebugDiagnosticArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug-codegen" + }, + "clang22-diagnostic": { + "platform": "container", + "compiler": "Clang22", + "compilerIdentity": "clang22", + "preset": "clang22-debug-codegen-diagnostic", + "profile": "CODEGEN_DIAGNOSTIC", + "configuration": "Debug", + "instrumentation": "none", + "codegenMode": "RECORD", + "aggregate": "SimdLibDebugDiagnosticArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "debug-codegen" + }, + "clang22-sanitizer-diagnostic": { + "platform": "container", + "compiler": "Clang22", + "compilerIdentity": "clang22", + "preset": "clang22-asan-ubsan-codegen-diagnostic", + "profile": "CODEGEN_DIAGNOSTIC", + "configuration": "Debug", + "instrumentation": "asan-ubsan", + "codegenMode": "RECORD", + "aggregate": "SimdLibDebugDiagnosticArtifacts", + "consumer": false, + "registerCapable": true, + "artifactKey": "asan-ubsan-codegen" + } + }, + "operations": { + "defaultBuild": [ + "msvc-release", + "msvc-debug", + "clangcl-release", + "clang-coverage", + "gcc13-release", + "gcc14-release", + "clang22-release", + "clang22-sanitizer" + ], + "defaultTests": [ + "msvc-release", + "msvc-debug", + "clangcl-release", + "clang-coverage", + "gcc13-release", + "gcc14-release", + "clang22-release", + "clang22-sanitizer" + ], + "coverage": [ + "clang-coverage" + ], + "sanitizer": [ + "clang22-sanitizer" + ], + "benchmarks": [ + "msvc-release", + "clangcl-release", + "gcc13-release", + "gcc14-release", + "clang22-release" + ], + "compilerContracts": [ + "msvc-contracts", + "clangcl-contracts", + "gcc13-contracts", + "gcc14-contracts", + "clang22-contracts" + ], + "optionalDiagnostics": [ + "msvc-diagnostic", + "clangcl-diagnostic", + "gcc14-diagnostic", + "clang22-diagnostic", + "clang22-sanitizer-diagnostic" + ], + "optionalDebug": [ + "clangcl-debug", + "gcc13-debug", + "gcc14-debug", + "clang22-debug" + ] + }, + "compilerOrder": [ + "Msvc", + "ClangCl", + "ClangCoverage", + "Gcc13", + "Gcc14", + "Clang22" + ], + "toolingValidation": { + "inputClasses": { + "matrix": [ + "tools/validation-matrix.json" + ], + "presets": [ + "CMakePresets.json" + ], + "resolvers": [ + "tools/Pipeline.Common.psm1", + "tools/Run-NativeMatrix.ps1", + "tools/Run-ContainerMatrix.ps1" + ], + "orchestration": [ + "tools/Build.ps1", + "tools/Run-Tests.ps1", + "tools/Build-Benchmarks.ps1", + "tools/Run-Benchmarks.ps1", + "tools/Record-Codegen.ps1" + ], + "toolingTests": [ + "tools/Verify-ValidationMatrix.ps1", + "tools/Test-ValidationPipeline.ps1", + "tools/Validate-PipelineTooling.ps1" + ], + "cmakeDevelopment": [ + "cmake/development", + "cmake/AuditValidationInventory.cmake" + ], + "containerRouting": [ + "compose.yml", + "containers/container-entrypoint.sh" + ], + "publicConsumerBoundary": [ + "tools/Test-PublicConsumerBoundary.ps1", + "cmake/CheckPublicConsumerBoundary.cmake" + ] + } + } +} diff --git a/wiki/Api.md b/wiki/Api.md index b0ae29f..960b5aa 100644 --- a/wiki/Api.md +++ b/wiki/Api.md @@ -12,16 +12,16 @@ - [`add_saturated`](#add-saturated) - [`add_subtract`](#add-subtract) - [`avg`](#avg) -- [`bit_shift_left`](#bit-shift-left) -- [`bit_shift_right`](#bit-shift-right) +- [`shift_bits_left` and `shift_bits_left_slow`](#shift-bits-left) +- [`shift_bits_right` and `shift_bits_right_slow`](#shift-bits-right) - [`bitwise_and`](#bitwise-and) - [`bitwise_andnot`](#bitwise-andnot) - [`bitwise_not`](#bitwise-not) - [`bitwise_or`](#bitwise-or) - [`bitwise_xor`](#bitwise-xor) - [`blend`](#blend) -- [`byte_shift_left`](#byte-shift-left) -- [`byte_shift_right`](#byte-shift-right) +- [`shift_bytes_left` and `shift_bytes_left_slow`](#shift-bytes-left) +- [`shift_bytes_right` and `shift_bytes_right_slow`](#shift-bytes-right) - [`cmp_eq`](#cmp-eq) - [`cmp_eq_mask`](#cmp-eq-mask) - [`cmp_ge`](#cmp-ge) @@ -36,10 +36,10 @@ - [`divide`](#divide) - [`dot_product`](#dot-product) - [`expand`](#expand) -- [`extract`](#extract) +- [`extract` and `extract_slow`](#extract) - [`hadd_saturated`](#hadd-saturated) - [`hsubtract_saturated`](#hsubtract-saturated) -- [`insert`](#insert) +- [`insert` and `insert_slow`](#insert) - [`load`](#load) - [`load_aligned`](#load-aligned) - [`load_partial`](#load-partial) @@ -70,9 +70,10 @@ - [`shift_left`](#shift-left) - [`shift_right`](#shift-right) - [`shift_right_arithmetic`](#shift-right-arithmetic) -- [`shuffle`](#shuffle) -- [`shuffle_hi`](#shuffle-hi) -- [`shuffle_lo`](#shuffle-lo) +- [`shuffle` and `shuffle_slow`](#shuffle) +- [`shuffle_32` and `shuffle_32_slow`](#shuffle-32) +- [`shuffle_hi` and `shuffle_hi_slow`](#shuffle-hi) +- [`shuffle_lo` and `shuffle_lo_slow`](#shuffle-lo) - [`sqrt`](#sqrt) - [`store`](#store) - [`store_aligned`](#store-aligned) @@ -215,42 +216,46 @@ using U8 = SimdLib::Api<128, std::uint8_t>; U8::avg(U8::set1(2U), U8::set1(6U)); // => every lane is 4U ``` - -## `bit_shift_left` + +## `shift_bits_left` and `shift_bits_left_slow` -Shifts the complete 128-bit register left, carrying bits across lane boundaries. Unlike `shift_left`, this treats the register as one unsigned 128-bit bit string. A zero or negative runtime count returns the input; counts of 128 or more return zero. +Shifts the complete 128-bit register left as one unsigned bit string, carrying across element boundaries. The unsuffixed template form encodes a compile-time count. The `_slow` form accepts a runtime count; nonpositive counts return the input and counts of 128 or more return zero. Signatures: ```cpp -static int_vector_t bit_shift_left(int_vector_t lhs, int shift) -template static int_vector_t bit_shift_left(int_vector_t lhs) +template static int_vector_t shift_bits_left(int_vector_t lhs) +static int_vector_t shift_bits_left_slow(int_vector_t lhs, int shift) ``` -Example: +Examples: ```cpp using U32x4 = SimdLib::Api<128, std::uint32_t>; -U32x4::bit_shift_left(U32x4::construct({3U, 3U, 3U, 3U}), 1); // => {6U, 6U, 6U, 6U} +const auto value = U32x4::construct({3U, 3U, 3U, 3U}); +U32x4::shift_bits_left<1>(value); // => {6U, 6U, 6U, 6U} +U32x4::shift_bits_left_slow(value, 1); // same semantics with a runtime count ``` - -## `bit_shift_right` + +## `shift_bits_right` and `shift_bits_right_slow` -Shifts the complete 128-bit register right, carrying bits across lane boundaries. Unlike `shift_right`, this treats the register as one unsigned 128-bit bit string. A zero or negative runtime count returns the input; counts of 128 or more return zero. +Shifts the complete 128-bit register right as one unsigned bit string, carrying across element boundaries. The unsuffixed template form encodes a compile-time count. The `_slow` form accepts a runtime count; nonpositive counts return the input and counts of 128 or more return zero. Signatures: ```cpp -static int_vector_t bit_shift_right(int_vector_t lhs, int shift) -template static int_vector_t bit_shift_right(int_vector_t lhs) +template static int_vector_t shift_bits_right(int_vector_t lhs) +static int_vector_t shift_bits_right_slow(int_vector_t lhs, int shift) ``` -Example: +Examples: ```cpp using U32x4 = SimdLib::Api<128, std::uint32_t>; -U32x4::bit_shift_right(U32x4::construct({8U, 8U, 8U, 8U}), 1); // => {4U, 4U, 4U, 4U} +const auto value = U32x4::construct({8U, 8U, 8U, 8U}); +U32x4::shift_bits_right<1>(value); // => {4U, 4U, 4U, 4U} +U32x4::shift_bits_right_slow(value, 1); // same semantics with a runtime count ``` @@ -352,62 +357,68 @@ U32::bitwise_xor( ``` -## `blend` +## `blend` and `blend_slow` -Blends two registers according to the implementation-specific control form. +Selects corresponding lanes from two registers. `blend` uses a compile-time immediate. An unsuffixed register-mask overload remains available where the instruction set provides a native runtime mask. `blend_slow` emulates immediate-mask semantics for a runtime scalar control. Signatures: ```cpp +template static vector_t blend(vector_t lhs, vector_t rhs) template static auto blend(Args &&...args) +template static auto blend_slow(Args &&...args) ``` -Example: +Examples: ```cpp using I32x4 = SimdLib::Api<128, std::int32_t>; -I32x4::blend( - I32x4::construct({10, 20, 30, 40}), - I32x4::construct({1, 2, 3, 4}), - 0b0101); // => {1, 20, 3, 40} +const auto lhs = I32x4::construct({10, 20, 30, 40}); +const auto rhs = I32x4::construct({1, 2, 3, 4}); +I32x4::blend<0b0101>(lhs, rhs); // => {1, 20, 3, 40} +I32x4::blend_slow(lhs, rhs, 0b0101); // same semantics with a runtime control ``` - -## `byte_shift_left` + +## `shift_bytes_left` and `shift_bytes_left_slow` -Shifts every byte in a 128-bit register toward higher byte indices. +Shifts a complete integral register toward higher byte indices. The immediate template treats the value as one contiguous byte sequence, crossing element, 64-bit, and—at 256 bits—128-bit-half boundaries. `shift_bytes_left` accepts a nonnegative compile-time count at 128 or 256 bits. Zero is identity; counts at least 16 for 128 bits or 32 for 256 bits produce zero. The `_slow` form accepts a runtime count but is intentionally available only for 128-bit registers. Signatures: ```cpp -static int_vector_t byte_shift_left(int_vector_t lhs, int shift) +template static int_vector_t shift_bytes_left(int_vector_t lhs) +static int_vector_t shift_bytes_left_slow(int_vector_t lhs, int count) // 128-bit only ``` -Example: +Examples: ```cpp +using U8x32 = SimdLib::Api<256, std::uint8_t>; using U8x16 = SimdLib::Api<128, std::uint8_t>; -U8x16::byte_shift_left(U8x16::set1(7U), 1); // => {0U, 7U, 7U, ..., 7U} +U8x32::shift_bytes_left<17>(U8x32::set1(7U)); // crosses the 128-bit boundary +U8x16::shift_bytes_left_slow(U8x16::set1(7U), 1); // => {0U, 7U, 7U, ..., 7U} ``` + +## `shift_bytes_right` and `shift_bytes_right_slow` - -## `byte_shift_right` - -Shifts every byte in a 128-bit register toward lower byte indices. +Shifts a complete integral register toward lower byte indices. The immediate template uses the same contiguous-register semantics as `shift_bytes_left`, including crossing the 128-bit boundary at 256 bits. `shift_bytes_right` accepts a nonnegative compile-time count at 128 or 256 bits. Zero is identity; counts at least 16 for 128 bits or 32 for 256 bits produce zero. The `_slow` form accepts a runtime count but is intentionally available only for 128-bit registers. Signatures: ```cpp -static int_vector_t byte_shift_right(int_vector_t lhs, int shift) +template static int_vector_t shift_bytes_right(int_vector_t lhs) +static int_vector_t shift_bytes_right_slow(int_vector_t lhs, int count) // 128-bit only ``` -Example: +Examples: ```cpp +using U8x32 = SimdLib::Api<256, std::uint8_t>; using U8x16 = SimdLib::Api<128, std::uint8_t>; -U8x16::byte_shift_right(U8x16::set1(7U), 1); // => {7U, 7U, ..., 7U, 0U} +U8x32::shift_bytes_right<17>(U8x32::set1(7U)); // crosses the 128-bit boundary +U8x16::shift_bytes_right_slow(U8x16::set1(7U), 1); // => {7U, 7U, ..., 7U, 0U} ``` - ## `cmp_eq` @@ -677,22 +688,24 @@ using I8x16 = SimdLib::Api<128, std::int8_t>; ``` -## `extract` +## `extract` and `extract_slow` -Extracts a lane or subvalue from a register. +Extracts one logical lane. The unsuffixed template form uses a compile-time lane index. `extract_slow` accepts a runtime-selected lane index. Signatures: ```cpp template static auto extract(vector_t lhs) -template static auto extract(vector_t lhs, selector_t rhs) +template static auto extract_slow(vector_t lhs, selector_t rhs) ``` -Example: +Examples: ```cpp using I32x4 = SimdLib::Api<128, std::int32_t>; -I32x4::extract<0>(I32x4::construct({7, 8, 9, 10})); // => 7 +const auto value = I32x4::construct({7, 8, 9, 10}); +I32x4::extract<0>(value); // => 7 +I32x4::extract_slow(value, 2); // => 9 with a runtime lane index ``` @@ -737,21 +750,24 @@ I16::hsubtract_saturated( ``` -## `insert` +## `insert` and `insert_slow` -Inserts a lane or subvalue into a register. +Replaces one logical lane. The unsuffixed template form uses a compile-time lane index. `insert_slow` accepts a runtime-selected lane index. Signatures: ```cpp -template static auto insert(Args &&...args) +template static vector_t insert(vector_t lhs, element_t rhs) +static vector_t insert_slow(vector_t lhs, element_t rhs, int index) ``` -Example: +Examples: ```cpp using I32x4 = SimdLib::Api<128, std::int32_t>; -I32x4::insert(I32x4::construct({0, 0, 0, 0}), 9, 0); // => {9, 0, 0, 0} +const auto zero = I32x4::setzero(); +I32x4::insert<0>(zero, 9); // => {9, 0, 0, 0} +I32x4::insert_slow(zero, 9, 2); // => {0, 0, 9, 0} with a runtime lane index ``` @@ -918,7 +934,7 @@ Example: ```cpp using U16x8 = SimdLib::Api<128, std::uint16_t>; -const auto values = U16x8::insert(U16x8::set1(4), 9, 3); +const auto values = U16x8::insert<3>(U16x8::set1(4), 9); U16x8::max_position(values); // => 3 ``` @@ -956,7 +972,7 @@ Example: ```cpp using U16x8 = SimdLib::Api<128, std::uint16_t>; -const auto values = U16x8::insert(U16x8::set1(4), 1, 3); +const auto values = U16x8::insert<3>(U16x8::set1(4), 1); U16x8::min_position(values); // => 3 ``` @@ -1312,62 +1328,105 @@ I32::shift_right_arithmetic(I32::construct({-8, -8, -8, -8}), 1); // => every la ``` -## `shuffle` +## `shuffle` and `shuffle_slow` + +The compile-time logical overload constructs each output lane from the source lane named by the selector at the same output position. It requires exactly one selector per lane, permits repeated selectors, and rejects selectors outside the complete source register. At 256 bits, any selector may cross the 128-bit boundary. Floating-point lanes are moved by object representation, preserving NaN payloads and signed zero. -Shuffles register contents according to the implementation-specific control form. +An unsuffixed register-selector overload remains available for byte shuffles backed by a native runtime selector register. `shuffle_slow` provides immediate-mask floating shuffle semantics for a runtime scalar control. Signatures: ```cpp -template static auto shuffle(int_vector_t lhs) +template static vector_t shuffle(vector_t lhs) template static auto shuffle(Args &&...args) +template static auto shuffle_slow(Args &&...args) ``` -Example: +Examples: ```cpp +using U16x8 = SimdLib::Api<128, std::uint16_t>; +const auto words = U16x8::construct({0, 1, 2, 3, 4, 5, 6, 7}); +U16x8::shuffle<7, 6, 5, 4, 3, 2, 1, 0>(words); // => {7, 6, 5, 4, 3, 2, 1, 0} + +using I32x8 = SimdLib::Api<256, std::int32_t>; +const auto integers = I32x8::construct({0, 1, 2, 3, 4, 5, 6, 7}); +I32x8::shuffle<4, 5, 6, 7, 0, 1, 2, 3>(integers); // => exchanges the 128-bit halves + +using F64x4 = SimdLib::Api<256, double>; +const auto doubles = F64x4::construct({1.0, 2.0, 3.0, 4.0}); +F64x4::shuffle<3, 3, 0, 0>(doubles); // => {4.0, 4.0, 1.0, 1.0} + using U8x16 = SimdLib::Api<128, std::uint8_t>; U8x16::shuffle( U8x16::set1(7U), - U8x16::set1(0x80U)); // => every lane is cleared to 0U by the mask''s high bit + U8x16::set1(0x80U)); // native selector-register shuffle: high bits clear output bytes + +using F32x4 = SimdLib::Api<128, float>; +F32x4::shuffle_slow(F32x4::set1(1.0F), F32x4::set1(2.0F), 0b1110'0100); +``` + + +## `shuffle_32` and `shuffle_32_slow` + +Shuffles 32-bit lanes within each 128-bit group. The unsuffixed template uses an immediate control byte; `_slow` accepts a runtime scalar control. + +Signatures: + +```cpp +template static int_vector_t shuffle_32(int_vector_t lhs) +static int_vector_t shuffle_32_slow(int_vector_t lhs, std::uint32_t imm8) +``` + +Example: + +```cpp +using U32x4 = SimdLib::Api<128, std::uint32_t>; +const auto values = U32x4::construct({0U, 1U, 2U, 3U}); +U32x4::shuffle_32<0b00'01'10'11>(values); // => {3U, 2U, 1U, 0U} +U32x4::shuffle_32_slow(values, 0b00'01'10'11); // same semantics with a runtime control ``` -## `shuffle_hi` +## `shuffle_hi` and `shuffle_hi_slow` -Shuffles the high half of a register where the specialization supports it. +Shuffles the high four 16-bit lanes in each 128-bit group. The unsuffixed template uses an immediate control byte; `_slow` accepts a runtime scalar control. Signatures: ```cpp -template static auto shuffle_hi(Args &&...args) +template static auto shuffle_hi(vector_t lhs) +template static auto shuffle_hi_slow(Args &&...args) ``` Example: ```cpp using I16x8 = SimdLib::Api<128, std::int16_t>; -const auto high = I16x8::byte_shift_left(I16x8::setr_partial(1, 2, 3, 4), 8); -I16x8::shuffle_hi(high, 0b0001'1011); // => {0, 0, 0, 0, 4, 3, 2, 1} +const auto high = I16x8::shift_bytes_left_slow(I16x8::setr_partial(1, 2, 3, 4), 8); +I16x8::shuffle_hi<0b0001'1011>(high); // => {0, 0, 0, 0, 4, 3, 2, 1} +I16x8::shuffle_hi_slow(high, 0b0001'1011); // same semantics with a runtime control ``` -## `shuffle_lo` +## `shuffle_lo` and `shuffle_lo_slow` -Shuffles the low half of a register where the specialization supports it. +Shuffles the low four 16-bit lanes in each 128-bit group. The unsuffixed template uses an immediate control byte; `_slow` accepts a runtime scalar control. Signatures: ```cpp -template static auto shuffle_lo(Args &&...args) +template static auto shuffle_lo(vector_t lhs) +template static auto shuffle_lo_slow(Args &&...args) ``` Example: ```cpp using I16x8 = SimdLib::Api<128, std::int16_t>; -I16x8::shuffle_lo( - I16x8::setr_partial(1, 2, 3, 4), 0b0001'1011); // => {4, 3, 2, 1, 0, 0, 0, 0} +const auto value = I16x8::setr_partial(1, 2, 3, 4); +I16x8::shuffle_lo<0b0001'1011>(value); // => {4, 3, 2, 1, 0, 0, 0, 0} +I16x8::shuffle_lo_slow(value, 0b0001'1011); // same semantics with a runtime control ``` diff --git a/wiki/Config.md b/wiki/Config.md index 78d36b7..841489d 100644 --- a/wiki/Config.md +++ b/wiki/Config.md @@ -7,6 +7,7 @@ - [Version constants](#version-constants) - [Compiler and target constants](#compiler-and-target-constants) - [Instruction constants](#instruction-constants) +- [Register interface availability](#register-interface-availability) - [Customization macros](#customization-macros) ## Version constants @@ -21,6 +22,11 @@ SimdLib::Config::version_major; // => 0 for version 0.2.0 `compiler_clang`, `compiler_msvc`, `compiler_gcc`, `target_x86`, `target_x64`, and `vectorcall_enabled` describe the active compiler and ABI target. +`vectorcall_enabled` is true for supported MSVC and Clang Windows x64 +targets. GNU-like Clang on Linux leaves the `SIMD_FLAGS(...)` +vector-calling-convention adapter empty because `__vectorcall` is a Windows ABI +boundary, not a portable x86 convention. + ```cpp SimdLib::Config::target_x64; // => true when compiling for x64 ``` @@ -33,9 +39,27 @@ SimdLib::Config::target_x64; // => true when compiling for x64 SimdLib::Config::has_avx2; // => true when AVX2 code generation is enabled ``` +## Register interface availability + +`SIMDLIB_REGISTER_INTERFACE_AVAILABLE` is `1` when the current translation unit supports the C++23 explicit-object syntax required by ``. SimdLib computes this macro from `__cpp_explicit_this_parameter >= 202110L`, or from the documented Microsoft C++ 19.44 fallback when `_MSVC_LANG` selects a post-C++20 mode. The Microsoft fallback intentionally excludes clang-cl. + +Unlike the customization macros below, this availability result is not caller-overridable. `SIMDLIB_REQUIRE_REGISTER_INTERFACE=1` can require the capability and produce a focused diagnostic when it is unavailable, but it cannot enable the interface. Linking the opt-in `SimdLib::Register` CMake target publishes this requirement and requests C++23; `SimdLib::SimdLib` remains C++20. + ## Customization macros -All `SIMDLIB_*` configuration macros are caller-overridable before including SimdLib. `SIMDLIB_PRECONDITION`, `SIMDLIB_ENABLE_CHECKS`, `SIMDLIB_FORCE_INLINE`, and `VECTORCALL` control contracts, diagnostics, inlining, and the public calling convention. +Except for the computed `SIMDLIB_REGISTER_INTERFACE_AVAILABLE` result, +documented `SIMDLIB_*` configuration macros are caller-overridable before +including SimdLib. `SIMDLIB_PRECONDITION` and `SIMDLIB_ENABLE_CHECKS` control +diagnostics. Public function declarations express ABI and optimization +contracts through `SIMD_FLAGS(...)`. + +Custom toolchains may define the paired +`SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL`/`SIMDLIB_METHOD_FLAGS_VECTORCALL`, +`SIMDLIB_METHOD_FLAGS_HAS_SAFE_BUFFERS`/`SIMDLIB_METHOD_FLAGS_SAFE_BUFFERS`, +`SIMDLIB_METHOD_FLAGS_HAS_FORCE_INLINE`/`SIMDLIB_METHOD_FLAGS_FORCE_INLINE`, +and `SIMDLIB_METHOD_FLAGS_HAS_FLATTEN`/`SIMDLIB_METHOD_FLAGS_FLATTEN` +capability and token adapters before the first SimdLib include. Downstream +function declarations still use only `SIMD_FLAGS(...)`. ```cpp #define SIMDLIB_ENABLE_CHECKS 1 diff --git a/wiki/NativeApi.md b/wiki/NativeApi.md index c61bf01..b5c0328 100644 --- a/wiki/NativeApi.md +++ b/wiki/NativeApi.md @@ -1,6 +1,10 @@ # NativeApi -`NativeApi` selects the widest `Api` specialization enabled by the compile target, so normal users do not need to choose between 128-bit and 256-bit registers. +`NativeApi` selects the widest `Api` specialization enabled by the +compile target. It remains the supported facade for C++20, collection helpers, +and direct backend operations. C++23 complete-register expressions should use +`NativeRegister` instead; explicit `Register` is +required when storage or ABI must remain stable across target configurations. ## Contents diff --git a/wiki/SimdVector.md b/wiki/SimdVector.md index 20c2d94..8da4cf6 100644 --- a/wiki/SimdVector.md +++ b/wiki/SimdVector.md @@ -611,7 +611,7 @@ template auto multi_sum_absolute_byte_differences(vector_t rhs) const Example: ```cpp -using U8x16 = SimdLib::uint8x16; +using U8x16 = SimdLib::SimdVector; U8x16{9}.multi_sum_absolute_byte_differences<0>(U8x16{ 4}); // => every selected 16-bit result lane is 20 ``` @@ -667,7 +667,7 @@ auto multiply_add_unsigned_signed_bytes(vector_t rhs) const Example: ```cpp -using U8x16 = SimdLib::uint8x16; +using U8x16 = SimdLib::SimdVector; U8x16{2}.multiply_add_unsigned_signed_bytes( U8x16{3}); // => every signed 16-bit result lane is 12 ``` @@ -1467,7 +1467,7 @@ auto sum_absolute_byte_differences(vector_t rhs) const Example: ```cpp -using U8x16 = SimdLib::uint8x16; +using U8x16 = SimdLib::SimdVector; U8x16{9}.sum_absolute_byte_differences(U8x16{4}); // => both 64-bit result lanes are 40 ``` @@ -1563,4 +1563,4 @@ Vector3{1.0F, 2.0F, 3.0F}.z(); // => 3.0F ## Related types and constants -The header provides `VectorInt8`, `VectorUInt8`, `VectorInt16`, `VectorUInt16`, `VectorInt32`, `VectorUInt32`, `VectorInt64`, and `VectorUInt64`, plus register-sized aliases such as `uint8x16`, `uint32x8`, `int16x8`, and `int64x4`. Use `SimdVector` directly for position-like dimensions such as two, three, or four. +`` provides C++23 complete-register aliases such as `uint8x16`, `uint32x8`, `int16x8`, and `int64x4` when their register width is available. Use `SimdVector` directly for logical vector dimensions such as two, three, or four. diff --git a/wiki/Technical-Reference.md b/wiki/Technical-Reference.md index 495a4c3..ca21df8 100644 --- a/wiki/Technical-Reference.md +++ b/wiki/Technical-Reference.md @@ -19,14 +19,19 @@ example, start with the [project README](../README.md). ## Library model -SimdLib is a C++20 header-only library. Its CMake target is an -`INTERFACE_LIBRARY`; it does not produce a DLL or static library. The public -API lives in the `SimdLib` namespace, while `SimdLib::Detail` contains -implementation details that consumer code must not name. +SimdLib is a header-only library with a C++20 core and an opt-in C++23 +complete-register interface. Its CMake targets are `INTERFACE_LIBRARY` +targets; they do not produce a DLL or static library. The public API lives in +the `SimdLib` namespace, while `SimdLib::Detail` contains implementation +details that consumer code must not name. The main API families are: -- `NativeApi`, the recommended facade that selects the widest +- `NativeRegister`, the recommended C++23 complete-register value + that selects the widest available register; +- `Register` and `RegisterMask`, the explicit-width + complete-register value and predicate types; +- `NativeApi`, the C++20 backend facade that selects the widest available register; - `Api`, a typed intrinsic facade; - `SimdVector`, a fixed-size value type backed by one @@ -50,6 +55,13 @@ add_subdirectory(external/SimdLib) target_link_libraries(MyTarget PRIVATE SimdLib::SimdLib) ``` +Targets that use `Register`, `RegisterMask`, or `NativeRegister` link the +C++23 interface target instead: + +```cmake +target_link_libraries(MyRegisterTarget PRIVATE SimdLib::Register) +``` + ### `FetchContent` Replace the repository URL and revision with the location used by your @@ -71,22 +83,23 @@ header where practical, or use `` for the complete non-formatting surface. `` is intentionally separate so translation units pay for formatting support only when they use it. -The repository's CMake project requires CMake 4.4 or newer. Consumers that -integrate the headers without the provided CMake project need only a supported -C++20 compiler and the appropriate target flags. +The repository's CMake project requires CMake 3.31 or newer. Consumers that +integrate the headers without the provided CMake project need a supported C++20 +compiler for the core, a supported C++23 compiler for the Register interface, +and the appropriate target flags. ## Supported environments The current validation matrix covers: -| Compiler family | Validated frontend | Targets | -| --- | --- | --- | -| MSVC | Visual Studio 2022 / MSVC 19.44 | Windows x86 and x64 | -| clang-cl | LLVM Clang 22 with the MSVC ABI | Windows x86 and x64 | -| Clang | LLVM Clang 22 | Linux x86 and x64 | -| GCC | GCC 13.2 or newer | Linux and MinGW x86 and x64 | +| Compiler family | Validated frontend | Targets | +| --------------- | ------------------------------- | ------------------- | +| MSVC | Visual Studio 2022 / MSVC 19.44 | Windows x64 | +| clang-cl | LLVM Clang 20.1.8 with the MSVC ABI | Windows x64 | +| Clang | LLVM Clang 22 | Linux x64 | +| GCC | GCC 13.2 or newer | Linux x64 | -The SIMD backends require x86/x64 intrinsic headers. The portable +The SIMD backends require x86-family intrinsic headers on an x64 target. The portable configuration layer, BMI fallback algorithms, and two-word `uint128_t` representation do not perform runtime CPU dispatch. @@ -97,10 +110,16 @@ binary was compiled. ## SIMD availability and instruction families -For ordinary SIMD work, use `SimdLib::NativeApi`. It resolves to -`Api<256, element_t>` when the compile target enables AVX2 and SSE4.2, and -otherwise resolves to `Api<128, element_t>` when SSE4.2 is enabled. This is a -compile-time choice based on compiler flags; it is not runtime CPU detection. +For C++23 complete-register work, use `SimdLib::NativeRegister`. It +resolves to the widest available `Register` specialization. Use explicit +`Register` when storage or ABI must not vary with +the target configuration. This is a compile-time choice based on compiler +flags; it is not runtime CPU detection. + +Use `SimdLib::NativeApi` for C++20, collection helpers, or direct +backend access. It resolves to `Api<256, element_t>` when the compile target +enables AVX2 and SSE4.2, and otherwise resolves to `Api<128, element_t>` when +SSE4.2 is enabled. Use the explicit-width `Api` form when a data layout, ABI, or algorithm specifically requires 128-bit or 256-bit registers. @@ -118,18 +137,21 @@ FMA-disabled paths, and all four BMI1/BMI2 combinations. ## Public headers -| Header | Public entry point | -| --- | --- | -| `` | Version, compiler, target, instruction, assertion, and ABI configuration | -| `` | Auto-sized `NativeApi`, explicit-width `Api`, and availability query | -| `` | Deprecated compatibility forwarding header; use `Api.h` | -| `` | `SimdVector` value type | -| `` | Fixed-extent and dynamic-span `SimdAlgo` operations | -| `` | Byte-mask reduction and expansion functions | -| `` | Portable and intrinsic `SimdLib::Bmi` bit helpers | -| `` | `uint128_t`, literals, bit utilities, hash, and numeric limits | -| `` | Opt-in `std::formatter` specializations | -| `` | Complete non-formatting public surface | +| Header | Public entry point | +| -------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `` | Version, compiler, target, instruction, assertion, and ABI configuration | +| `` | Auto-sized `NativeApi`, explicit-width `Api`, and availability query | +| `` | C++23 `Register` and `NativeRegister` complete-register values | +| `` | C++23 `RegisterMask` predicate values | +| `` | Deprecated compatibility forwarding header; use `Api.h` | +| `` | C++23 named `Register` aliases exposed when their SSE4.2 or AVX2 width is available | +| `` | `SimdVector` value type | +| `` | Fixed-extent and dynamic-span `SimdAlgo` operations | +| `` | Byte-mask reduction and expansion functions | +| `` | Portable and intrinsic `SimdLib::Bmi` bit helpers | +| `` | `uint128_t`, literals, bit utilities, hash, and numeric limits | +| `` | Opt-in `std::formatter` specializations | +| `` | Complete non-formatting public surface | Headers and declarations below `SimdLib::Detail` are implementation-only. @@ -151,20 +173,26 @@ first SimdLib include. `SIMDLIB_HAS_FMA`, `SIMDLIB_HAS_BMI1`, and `SIMDLIB_HAS_BMI2` describe compiler-enabled instruction families. They do not provide runtime CPU detection. -- `SIMDLIB_FORCE_INLINE` selects the supported compiler attribute together - with `inline` and may be replaced with ordinary `inline`. +- `SIMD_FLAGS(..., ForceInline)` selects the supported compiler attribute + together with `inline`. +- `SIMD_FLAGS(..., Flatten)` selects the supported recursive-inlining + attribute independently from `ForceInline`. - `SIMDLIB_PRECONDITION(condition, message)` is the assertion replacement point and defaults to standard `assert`. - `SIMDLIB_ENABLE_CHECKS` defaults to enabled without `NDEBUG` and disabled with `NDEBUG`. -- `VECTORCALL` affects the ABI. It is `__vectorcall` on supported MSVC and - Clang x86/x64 targets and empty elsewhere. - -A caller that overrides `VECTORCALL` with an empty definition must also set -`SIMDLIB_VECTORCALL_ENABLED=0` consistently in every translation unit. An -empty `VECTORCALL` changes only the calling convention; it does not disable -SSE, AVX, FMA, BMI, or any other target-specific instruction. Those remain -controlled by compiler flags and the corresponding `SIMDLIB_HAS_*` values. +- The `In`, `Out`, and `InOut` boundary modes affect the ABI. They emit the + configured vector-calling-convention adapter on supported MSVC and Clang + Windows x64 targets and emit no calling-convention token on unsupported + targets. + +A custom toolchain may override the paired +`SIMDLIB_METHOD_FLAGS_HAS_VECTORCALL` and +`SIMDLIB_METHOD_FLAGS_VECTORCALL` definitions consistently in every +translation unit. An empty adapter changes only the calling convention; it +does not disable SSE, AVX, FMA, BMI, or any other target-specific instruction. +Those remain controlled by compiler flags and the corresponding +`SIMDLIB_HAS_*` values. All linked translation units must use the same ABI-affecting configuration. See [CompilerConfiguration.md](../cmake/CompilerConfiguration.md) for compiler @@ -229,31 +257,83 @@ other presentation types throw `std::format_error`. ## Development workflow -The checked-in presets provide the standard MSVC test build and a Clang/LLVM -coverage build: +The repository-owned commands require PowerShell 7+ and CMake 3.31. A complete +Windows-hosted run additionally requires Visual Studio 2022 with the x64 C++ +tools, LLVM 20 or newer on `PATH`, and Docker Desktop using Linux containers. Container- +only runs require Docker and do not require the native Windows compilers. + +Build the complete native and Linux validation matrix, excluding benchmark +artifacts, with an explicit scope: ```powershell -cmake --preset msvc -cmake --build --preset msvc-release -ctest --preset msvc-release +tools/Build.ps1 -Scope All +``` + +Build once and run every assigned correctness, ABI, generated-code, sanitizer, +consumer, and coverage test cell with: -cmake --preset clang-coverage -cmake --build --preset coverage -ctest --preset coverage +```powershell +tools/Run-Tests.ps1 -Scope All +``` + +Benchmark compilation and execution are supplemental and remain outside the +default build and correctness testing: + +```powershell +tools/Build-Benchmarks.ps1 -Scope All +tools/Run-Benchmarks.ps1 -Scope All ``` -The main CMake options are: +The accepted scopes and compiler filters are: + +| Scope | Compiler filters | Default owned cells | +| ------------ | ---------------------------------- | -------------------------------------------------------- | +| `All` | `All` or any compatible subset | Every retained native and container cell | +| `Native` | `Msvc`, `ClangCl`, `ClangCoverage` | MSVC and clang-cl Release, MSVC Debug, and Clang coverage | +| `Containers` | `Gcc13`, `Gcc14`, `Clang22` | Linux Release plus Clang ASan+UBSan | + +For example, a Linux-only CI worker uses `tools/Build.ps1 -Scope Containers` +followed by `tools/Run-Tests.ps1 -Scope Containers`. A focused local +diagnostic can use `tools/Run-Tests.ps1 -Scope Native -Compiler Msvc` or +`tools/Run-Tests.ps1 -Scope Containers -Compiler Gcc14`. Ordinary clang-cl, +GCC 13, GCC 14, and Clang Debug cells are opt-in troubleshooting configurations, +not default-matrix members. Record-only Debug or sanitizer generated-code work +uses an explicit `tools/Record-Codegen.ps1` compiler and cell selection and +cannot satisfy the mandatory optimized Release gate. + +Each compiler/configuration owns a fingerprinted tree below `out/pipeline`. +The fingerprint includes compiler and image identity, generator, configuration, +instrumentation, required flags, dependencies, and CPU requirements. Source +inputs have a separate digest in the completed manifest. Consequently, +test-only and benchmark-execution operations reject missing, stale, or +incompatible artifacts and never configure or compile. The explicit benchmark +build requires completed validation manifests and targets only +`BenchmarkArtifacts` in the owning Release trees. Objects are reusable only +when their complete compilation fingerprint matches. See [Unified build and +validation](../docs/BuildPipeline.md) for the complete identity and receipt-consumption contract. + +Instrumentation boundaries are explicit. Release and Debug use separate trees; +Clang ASan+UBSan has its own instrumented Debug fingerprint; source coverage has +its own native Clang tree; and benchmark compilation reuses only an already +validated Release tree. Coverage is enabled only for top-level SimdLib +development builds and is never introduced into an `add_subdirectory` +consumer. + +The following development CMake options exist only when SimdLib is the top-level +project. They are not declared for an `add_subdirectory` consumer: - `SIMDLIB_BUILD_SMOKE_TESTS=ON` builds the two-translation-unit ODR smoke executable. It is enabled by default. -- `SIMDLIB_BUILD_HEADER_TESTS=ON` compiles every public header as the first and +- `SIMDLIB_BUILD_HEADER_PROBES=ON` compiles every public header as the first and only SimdLib header in its translation unit. It is enabled by default. -- `SIMDLIB_BUILD_TESTS=ON` builds the Catch2 test suite. Catch2 v3 is fetched +- `SIMDLIB_BUILD_RUNTIME_TESTS=ON` builds the Catch2 test suite. Catch2 v3 is fetched when it is not installed and `SIMDLIB_FETCH_TEST_DEPENDENCIES=ON`. -- `SIMDLIB_BUILD_TESTS_128`, `SIMDLIB_BUILD_TESTS_256`, and - `SIMDLIB_BUILD_TESTS_FMA` independently control the SSE4.2, AVX2, and FMA +- `SIMDLIB_FETCH_TEST_DEPENDENCIES=ON` permits a top-level development build to + fetch Catch2 when no suitable package is already available. +- `SIMDLIB_BUILD_API_SSE42_TESTS`, `SIMDLIB_BUILD_API_AVX2_TESTS`, and + `SIMDLIB_BUILD_FMA_TESTS` independently control the SSE4.2, AVX2, and FMA executables. Disable instruction families the test host cannot execute. -- `SIMDLIB_BUILD_TESTS_OPTIONAL=ON` enables BMI1/BMI2 intrinsic-path testing +- `SIMDLIB_BUILD_BMI_TESTS=ON` enables BMI1/BMI2 intrinsic-path testing and deterministic comparison with the always-built portable path. It is off by default so unsupported hosts do not execute BMI instructions. - `SIMDLIB_BUILD_VECTOR_ALGORITHM_TESTS=ON` builds the `SimdVector`, @@ -261,30 +341,50 @@ The main CMake options are: - `SIMDLIB_BUILD_BENCHMARKS=ON` builds the Catch2 benchmarks and requires a discoverable Catch2 v3 package. - `SIMDLIB_BUILD_EXAMPLES=ON` builds and registers the complete API example. -- `SIMDLIB_BUILD_CONFIGURATION_TESTS=ON` builds compile-only configuration +- `SIMDLIB_BUILD_CONFIGURATION_PROBES=ON` builds compile-only configuration probes. It is enabled by default. +- `SIMDLIB_BUILD_CONSTEXPR_PROBES=ON` builds compile-only constant-evaluation + contracts. Exhaustive Release profiles own the compiler and feature matrix; + Debug and sanitizer profiles disable duplicate evaluation, while native + Clang coverage retains its distinct driver and platform contract. +- `SIMDLIB_BUILD_METHOD_FLAGS_CODEGEN_GATES=ON` builds the method-attribute generated-code comparison owned by Release profiles. +- `SIMDLIB_BUILD_REGISTER_CODEGEN_GATES=ON` builds the Register wrapper/raw + generated-code and ABI comparison corpus when the compiler supports the + C++23 Register interface. +- `SIMDLIB_REGISTER_CODEGEN_MODE=ENFORCE|RECORD` selects whether generated-code + differences fail the supported optimized gate or are retained as diagnostic + records. +- `SIMDLIB_VALIDATE_EXHAUSTIVE_TARGETS=ON` makes configuration fail when an + exhaustive profile does not define its required target inventory. - `SIMDLIB_STRICT_WARNINGS=ON` enables the compiler-specific strict warning policy and treats warnings as errors for SimdLib-owned targets. - `SIMDLIB_ENABLE_COVERAGE=ON` instruments supported Clang targets and configures LLVM source coverage. CTest labels identify instruction families and test groups so automation can -include or exclude them explicitly. The `SimdLibCoverageReset` and -`SimdLibCoverageReport` targets produce -`build-coverage/coverage.info` for command-line use and VS Code CMake Tools. +include or exclude them explicitly. The `CoverageReset` and `CoverageReport` +targets produce `coverage.info` below the active fingerprint's build directory, +for example +`out/pipeline/windows-clang-coverage/debug-coverage-/build/coverage.info`. ## Continuous validation -`.github/workflows/ci.yml` defines Debug and Release jobs for MSVC, clang-cl, -Clang, and GCC on supported x86/x64 targets. It also contains Clang ASan/UBSan -coverage, an independent instruction-family matrix, and explicit constexpr, -first-include header-hygiene, multi-translation-unit ODR, example, and consumer -gates. +`.github/workflows/ci.yml` delegates to the same scoped `Build.ps1` and +`Run-Tests.ps1` commands used locally. Native MSVC, native clang-cl +plus coverage, and Linux container compilers each build their assigned +fingerprints once and then run test-only operations. Each benchmark-owning CI +job invokes `Build-Benchmarks.ps1` explicitly after correctness testing; the +default build remains benchmark-free. Clang ASan+UBSan remains an independent +instrumented fingerprint. Mandatory instruction-family labels, +constexpr probes, first-include header hygiene, ODR, examples, consumers, +generated-code comparisons, and ABI gates are members of those owned cells, +not separate rebuild scenarios. The consumer smoke project under `tests/consumer` imports SimdLib with `add_subdirectory`, verifies that `SimdLib` is an `INTERFACE_LIBRARY`, and links only the consumer executable. No SimdLib runtime binary is produced. -The completed compiler, sanitizer, consumer, benchmark, and test evidence is -recorded in [Validation.md](../docs/Validation.md). Broader coverage details -and known gaps are recorded in [TestCoverage.md](../docs/TestCoverage.md). +The supported compiler, sanitizer, consumer, benchmark, and test commands are +documented in [BuildPipeline.md](../docs/BuildPipeline.md). Coverage ownership +and known gaps are recorded in [TestCoverage.md](../docs/TestCoverage.md); +individual outcomes remain in generated reports and CI artifacts.