Add Asymmetric Scalar Hashing (ASH) quantization

.github/workflows/run-bench.yml

@@ -92,9 +92,9 @@ jobs:
       - name: Set up GCC
         run: |
           sudo apt install -y gcc
-      - name: Install Meson and Ninja
+      - name: Install native build dependencies
         run: |
-          sudo apt update && sudo apt install -y meson ninja-build
+          sudo apt update && sudo apt install -y cmake meson ninja-build
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.jdk }}
         uses: actions/setup-java@v3
@@ -131,6 +131,8 @@ jobs:
           ref: ${{ matrix.branch }}
           fetch-depth: 0
 
+      # Branches from before the Highway Meson wrap still use a submodule;
+      # this step is a no-op on newer ones.
       - name: Initialize Git Submodules
         run: git submodule update --init
 
@@ -154,7 +156,7 @@ jobs:
 
       # Build the branch
       - name: Build branch
-        run: mvn -B -Punix-amd64-profile package --file pom.xml
+        run: mvn -B -Punix-amd64-profile -Dnative.onednn=true package --file pom.xml
 
       # Run the benchmark if jvector-examples exists
       - name: Run benchmark
@@ -255,9 +257,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Initialize Git Submodules
-        run: git submodule update --init
-
       - name: Download all benchmark results
         uses: actions/download-artifact@v4
         with:

.github/workflows/run-compaction.yml

@@ -57,9 +57,9 @@ jobs:
     steps:
       - name: Set up GCC
         run: sudo apt install -y gcc
-      - name: Install Meson and Ninja
+      - name: Install native build dependencies
         run: |
-          sudo apt update && sudo apt install -y meson ninja-build
+          sudo apt update && sudo apt install -y cmake meson ninja-build
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.jdk }}
         uses: actions/setup-java@v3
@@ -74,11 +74,13 @@ jobs:
           ref: ${{ matrix.branch }}
           fetch-depth: 0
 
+      # Branches from before the Highway Meson wrap still use a submodule;
+      # this step is a no-op on newer ones.
       - name: Initialize Git Submodules
         run: git submodule update --init
 
       - name: Build branch
-        run: mvn -B -Punix-amd64-profile package --file pom.xml
+        run: mvn -B -Punix-amd64-profile -Dnative.onednn=true package --file pom.xml
 
       - name: Run CompactorBenchmark
         id: run-benchmark

.github/workflows/unit-tests-arm64.yaml

@@ -53,15 +53,12 @@ jobs:
         run: |
           sudo apt install -y gcc g++
 
-      - name: Install Meson, Ninja, and GTest
+      - name: Install native build dependencies and GTest
         run: |
-          sudo apt update && sudo apt install -y meson ninja-build pkg-config libgtest-dev
+          sudo apt update && sudo apt install -y cmake meson ninja-build pkg-config libgtest-dev
 
       - uses: actions/checkout@v4
 
-      - name: Initialize Git Submodules
-        run: git submodule update --init
-
       - name: Build test_simd_kernels (native C++)
         # Meson detects aarch64 and compiles all three ISA variants (neon/sve/sve2)
         # regardless of what the host CPU supports at runtime.
@@ -110,7 +107,7 @@ jobs:
         env:
           JVECTOR_MAX_ISA: ${{ matrix.max_isa }}
         run: >-
-          mvn -B -Punix-amd64-profile -pl jvector-tests -am test
+          mvn -B -Punix-amd64-profile -Dnative.onednn=true -pl jvector-tests -am test
           -DTest_RequireSpecificVectorizationProvider=NativeVectorizationProvider
           -Dsurefire.failIfNoSpecifiedTests=false
           -Dtest=TestVectorizationProvider
@@ -119,7 +116,7 @@ jobs:
         env:
           JVECTOR_MAX_ISA: ${{ matrix.max_isa }}
         run: >-
-          mvn -B -Punix-amd64-profile test
+          mvn -B -Punix-amd64-profile -Dnative.onednn=true test
           -DTest_RequireSpecificVectorizationProvider=NativeVectorizationProvider
 
       - name: Test Summary for (ARM64/max:${{ matrix.max_isa }},JDK${{ matrix.jdk }})

.github/workflows/unit-tests.yaml

@@ -50,13 +50,10 @@ jobs:
       - name: Set up GCC
         run: |
           sudo apt install -y gcc
-      - name: Install Meson, Ninja, and GTest
+      - name: Install native build dependencies and GTest
         run: |
-          sudo apt update && sudo apt install -y meson ninja-build pkg-config libgtest-dev
+          sudo apt update && sudo apt install -y cmake meson ninja-build pkg-config libgtest-dev
       - uses: actions/checkout@v4
-      - name: Initialize Git Submodules
-        run: git submodule update --init
-
       - name: Build test_simd_kernels (native C++)
         working-directory: jvector-native/src/main/native
         run: |
@@ -108,7 +105,7 @@ jobs:
         env:
           JVECTOR_MAX_ISA: ${{ matrix.max_isa }}
         run: >-
-          mvn -B -Punix-amd64-profile -pl jvector-tests -am test
+          mvn -B -Punix-amd64-profile -Dnative.onednn=true -pl jvector-tests -am test
           -DTest_RequireSpecificVectorizationProvider=NativeVectorizationProvider
           -Dsurefire.failIfNoSpecifiedTests=false
           -Dtest=TestVectorizationProvider
@@ -118,7 +115,7 @@ jobs:
         env:
           JVECTOR_MAX_ISA: ${{ matrix.max_isa }}
         run: >-
-          mvn -B -Punix-amd64-profile test
+          mvn -B -Punix-amd64-profile -Dnative.onednn=true test
           -DTest_RequireSpecificVectorizationProvider=NativeVectorizationProvider
 
       - name: Test Summary for (ISA:isa-avx512f/max:${{ matrix.max_isa }},JDK${{ matrix.jdk }})
@@ -147,8 +144,6 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
-      - name: Initialize Git Submodules
-        run: git submodule update --init
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -159,12 +154,12 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt install -y gcc
-      - name: Install Meson and Ninja
+      - name: Install native build dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
-          sudo apt update && sudo apt install -y meson ninja-build
+          sudo apt update && sudo apt install -y cmake meson ninja-build
       - name: Compile, run tests, and package (JDK 22)
-        run: mvn -B verify
+        run: mvn -B "-Dnative.onednn=true" verify
         if: matrix.jdk == '22'
       - name: Compile, run tests, and package (JDK 20)
         run: mvn -B -Pjdk20 -am -pl jvector-tests test

.gitignore

@@ -11,6 +11,7 @@ dataset_
 **results**.jsonl
 
 ### Bench caches
+compressor_cache/
 pq_cache/
 index_cache/
 dataset_cache/
@@ -64,3 +65,6 @@ dependency-reduced-pom.xml
 results.csv
 **/datasets/custom/**
 **/dataset_cache/**
+
+### Benchmark server profiles (machine-specific) ###
+jvector-examples/bench-profiles/local/

.run/ASH vs PQ single JVM - MPNet 10M 2-bit ASH.run.xml

@@ -0,0 +1,19 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="ASH vs PQ single JVM - MPNet 10M 2-bit ASH" type="ShConfigurationType" folderName="ASH vs PQ single JVM">
+    <option name="SCRIPT_TEXT" value="" />
+    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
+    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/jvector-examples/benchyaml-play.sh" />
+    <option name="SCRIPT_OPTIONS" value="jvector-examples/yaml-configs/index-parameters/studies/ash-vs-pq-single-jvm/mpnet10m-2bit-ash.yml" />
+    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
+    <option name="INTERPRETER_PATH" value="/bin/bash" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="EXECUTE_IN_TERMINAL" value="false" />
+    <option name="EXECUTE_SCRIPT_FILE" value="true" />
+    <envs>
+      <env name="BENCH_RUN_CONFIG" value="jvector-examples/yaml-configs/ash-pq/run-single-jvm.yml" />
+    </envs>
+    <method v="2" />
+  </configuration>
+</component>