From 1c745ef587c2231696c85b9959463be88ce1d55e Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Thu, 12 Nov 2026 12:26:04 -0500 Subject: [PATCH] update on disk version to v7. Update all test yamls to v7. Add unit tests. Added fix for Feature hard coding on disk version that has already been applied in ash-dev as well --- UPGRADING.md | 13 ++ docs/release notes/4.1.0/709.feature.md | 3 +- .../graph/disk/AbstractGraphIndexFormat.java | 4 +- .../graph/disk/GraphIndexFormatFactory.java | 3 +- .../graph/disk/GraphIndexFormatV6.java | 10 +- .../graph/disk/GraphIndexFormatV7.java | 39 ++++++ .../jvector/graph/disk/OnDiskGraphIndex.java | 2 +- .../jvector/graph/disk/feature/Feature.java | 19 +++ .../jvector/graph/disk/feature/FusedPQ.java | 8 +- .../graph/disk/feature/InlineByteVectors.java | 7 ++ .../graph/disk/feature/InlineVectors.java | 7 ++ .../jvector/graph/disk/feature/NVQ.java | 8 +- .../graph/disk/feature/SeparatedNVQ.java | 8 +- .../graph/disk/feature/SeparatedVectors.java | 7 ++ .../yaml-configs/index-parameters/cap-10M.yml | 2 +- .../yaml-configs/index-parameters/cap-1M.yml | 2 +- .../cohere-english-v3-10M.yml | 2 +- .../index-parameters/cohere-english-v3-1M.yml | 2 +- .../index-parameters/colbert-1M.yml | 2 +- .../yaml-configs/index-parameters/default.yml | 2 +- .../index-parameters/dpr-gemma-10M.yml | 2 +- .../index-parameters/dpr-gemma-1M.yml | 2 +- .../index-parameters/glove-100-angular.yml | 2 +- .../index-parameters/glove-200-angular.yml | 2 +- .../index-parameters/glove-25-angular.yml | 2 +- .../index-parameters/glove-50-angular.yml | 2 +- .../index-parameters/lastfm-64-dot.yml | 2 +- .../index-parameters/nytimes-256-angular.yml | 2 +- .../index-parameters/sift-128-euclidean.yml | 2 +- .../wiki-multilingual-v3-int8-1m.yml | 2 +- .../wikipedia-2023-11-en-mxbai-int8-1m.yml | 2 +- jvector-examples/yaml-configs/run-config.yml | 2 +- .../example/TestDatasetPrefetching.java | 10 +- .../graph/disk/TestFeatureHeaderVersion.java | 111 ++++++++++++++++++ .../disk/TestGraphIndexFormatFactory.java | 12 +- .../graph/disk/TestOnDiskGraphIndex.java | 11 ++ 36 files changed, 283 insertions(+), 35 deletions(-) create mode 100644 jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatV7.java create mode 100644 jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestFeatureHeaderVersion.java diff --git a/UPGRADING.md b/UPGRADING.md index c99de606f..999e0c448 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,3 +1,16 @@ +# Upgrading from 4.0.x to 4.1.x + +## On-disk format version 7 +- Version 7 of the on-disk graph format adds one feature, inline int8 vectors (`FeatureId.INLINE_BYTE_VECTORS`): + each node's byte vector stored inline, one byte per dimension. Apart from that feature, a version 7 file has + the version 6 layout. +- JVector 4.0.x reads formats up to 6 and rejects version 7 files. `OnDiskGraphIndex.CURRENT_VERSION`, + the default for `OnDiskGraphIndexWriter.Builder` and for `CompressedVectors.write(out)`, is now 7. +- To keep a rollback to 4.0.x possible, pass the version explicitly until every reader runs 4.1: + `withVersion(6)` on graph writer builders and `write(out, 6)` for compressors and compressed vectors. + A version 6 writer rejects `INLINE_BYTE_VECTORS`. +- `OnDiskGraphIndexCompactor` always writes `CURRENT_VERSION`; it has no option to write an older version. + # Upgrading from 3.0.x to 4.0.x ## New features diff --git a/docs/release notes/4.1.0/709.feature.md b/docs/release notes/4.1.0/709.feature.md index cbb188527..dcd6a71c0 100644 --- a/docs/release notes/4.1.0/709.feature.md +++ b/docs/release notes/4.1.0/709.feature.md @@ -49,4 +49,5 @@ try (var builder = GraphIndexBuilder.builder(vectors, ByteVectorSimilarityFuncti * Zero vectors are not supported with `COSINE` (the result is undefined); do not index them. * Existing float32 callers keep working: `RandomAccessVectorValues.getVector` keeps its `VectorFloat` signature and `GraphIndexBuilder.build(RandomAccessVectorValues)` is retained as a forwarding overload. -* Int8 indexes require on-disk format version 6 or later. +* Int8 indexes require on-disk format version 7 or later. JVector 4.0.x reads formats up to 6 and rejects + version 7 files; a version 6 writer rejects `INLINE_BYTE_VECTORS`. diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/AbstractGraphIndexFormat.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/AbstractGraphIndexFormat.java index 2f77cf768..12bb1d28a 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/AbstractGraphIndexFormat.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/AbstractGraphIndexFormat.java @@ -381,7 +381,7 @@ public int commonHeaderSize() { public void writeHeaderFeatures(IndexWriter out, Map features) throws IOException { // we restrict pre-version-3 writers to INLINE_VECTORS features, so we don't need additional version-handling here for (Feature writer : features.values()) { - writer.writeHeader(out); + writer.writeHeader(out, getVersion()); } } @@ -415,7 +415,7 @@ public OnDiskGraphIndex loadOnDiskIndex(RandomAccessReader reader, Header header */ protected static Set allFeatures() { return EnumSet.of(FeatureId.INLINE_VECTORS, FeatureId.FUSED_PQ, FeatureId.NVQ_VECTORS, - FeatureId.SEPARATED_VECTORS, FeatureId.SEPARATED_NVQ, FeatureId.INLINE_BYTE_VECTORS); + FeatureId.SEPARATED_VECTORS, FeatureId.SEPARATED_NVQ); } /** diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatFactory.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatFactory.java index 12010bc1e..b7c0014a1 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatFactory.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatFactory.java @@ -35,7 +35,8 @@ public class GraphIndexFormatFactory { 3, new GraphIndexFormatV3(), 4, new GraphIndexFormatV4(), 5, new GraphIndexFormatV5(), - 6, new GraphIndexFormatV6() + 6, new GraphIndexFormatV6(), + 7, new GraphIndexFormatV7() ); /** diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatV6.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatV6.java index c9d965aa4..a9a46ed5c 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatV6.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatV6.java @@ -26,6 +26,7 @@ import java.util.EnumMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.function.IntFunction; import java.util.stream.Collectors; @@ -44,7 +45,12 @@ class GraphIndexFormatV6 extends GraphIndexFormatV5 { /** Creates the singleton format for version 6. */ GraphIndexFormatV6() { - super(6, allFeatures()); + this(6, allFeatures()); + } + + /** Allows later feature extensions to reuse the V6 record and hierarchy layout. */ + protected GraphIndexFormatV6(int version, Set supportedFeatures) { + super(version, supportedFeatures); } /** Places fused features last so non-fused inline features occupy a contiguous prefix. */ @@ -112,7 +118,7 @@ public void writeHeaderFeatures(IndexWriter out, Map supportedFeatures() { + var features = EnumSet.copyOf(allFeatures()); + features.add(FeatureId.INLINE_BYTE_VECTORS); + return features; + } +} diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/OnDiskGraphIndex.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/OnDiskGraphIndex.java index 92f43292d..c67461162 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/OnDiskGraphIndex.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/OnDiskGraphIndex.java @@ -71,7 +71,7 @@ public class OnDiskGraphIndex implements ImmutableGraphIndex, AutoCloseable, Accountable { private static final Logger logger = LoggerFactory.getLogger(OnDiskGraphIndex.class); - public static final int CURRENT_VERSION = 6; + public static final int CURRENT_VERSION = 7; static final int MAGIC = 0xFFFF0D61; // FFFF to distinguish from old graphs, which should never start with a negative size "ODGI" static final VectorTypeSupport vectorTypeSupport = VectorizationProvider.getInstance().getVectorTypeSupport(); final ReaderSupplier readerSupplier; diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/Feature.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/Feature.java index 03dd2ec98..1602aa2fa 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/Feature.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/Feature.java @@ -36,8 +36,27 @@ default boolean isFused() { int featureSize(); + /** + * Writes this feature's header at {@link io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex#CURRENT_VERSION}. + * + * @deprecated a header embedded in a graph must be written at the graph's version; use + * {@link #writeHeader(IndexWriter, int)} + */ + @Deprecated void writeHeader(IndexWriter out) throws IOException; + /** + * Writes this feature's header for a graph written at {@code version}. Features that serialize a + * compressor write it at this version, so a graph written at an older version is laid out as that + * version wrote it. The default ignores the version, for headers that don't depend on it. + * + * @param out the output to write to + * @param version the on-disk format version of the graph being written + */ + default void writeHeader(IndexWriter out, int version) throws IOException { + writeHeader(out); + } + default void writeInline(IndexWriter out, State state) throws IOException { // default no-op } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/FusedPQ.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/FusedPQ.java index 3dc404481..a8d39aa1b 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/FusedPQ.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/FusedPQ.java @@ -135,9 +135,15 @@ public QuantizationCompactionStrategy createCompactionStrategy(CompactionContext return new FusedCompactionStrategy(ctx, this, retrainer); } + @Deprecated @Override public void writeHeader(IndexWriter out) throws IOException { - pq.write(out, OnDiskGraphIndex.CURRENT_VERSION); + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) throws IOException { + pq.write(out, version); } // this is an awkward fit for the Feature.State design since we need to diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineByteVectors.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineByteVectors.java index 86ebfff3c..981999b95 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineByteVectors.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineByteVectors.java @@ -20,6 +20,7 @@ import io.github.jbellis.jvector.disk.IndexWriter; import io.github.jbellis.jvector.disk.RandomAccessReader; import io.github.jbellis.jvector.graph.disk.CommonHeader; +import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex; import io.github.jbellis.jvector.vector.VectorizationProvider; import io.github.jbellis.jvector.vector.types.ByteSequence; import io.github.jbellis.jvector.vector.types.VectorTypeSupport; @@ -72,8 +73,14 @@ static InlineByteVectors load(CommonHeader header, RandomAccessReader reader) { return new InlineByteVectors(header.dimension); } + @Deprecated @Override public void writeHeader(IndexWriter out) { + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) { // common header carries dimension; nothing extra needed } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineVectors.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineVectors.java index 005103ddb..cac33f6f2 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineVectors.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineVectors.java @@ -19,6 +19,7 @@ import io.github.jbellis.jvector.disk.IndexWriter; import io.github.jbellis.jvector.disk.RandomAccessReader; import io.github.jbellis.jvector.graph.disk.CommonHeader; +import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex; import io.github.jbellis.jvector.vector.VectorizationProvider; import io.github.jbellis.jvector.vector.types.VectorFloat; import io.github.jbellis.jvector.vector.types.VectorTypeSupport; @@ -58,8 +59,14 @@ static InlineVectors load(CommonHeader header, RandomAccessReader reader) { return new InlineVectors(header.dimension); } + @Deprecated @Override public void writeHeader(IndexWriter out) { + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) { // common header contains dimension, which is sufficient } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/NVQ.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/NVQ.java index 492bb0dfe..863780bf1 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/NVQ.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/NVQ.java @@ -69,9 +69,15 @@ static NVQ load(CommonHeader header, RandomAccessReader reader) { } } + @Deprecated @Override public void writeHeader(IndexWriter out) throws IOException { - nvq.write(out, OnDiskGraphIndex.CURRENT_VERSION); + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) throws IOException { + nvq.write(out, version); } @Override diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedNVQ.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedNVQ.java index b65370632..a0e3ddd21 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedNVQ.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedNVQ.java @@ -67,9 +67,15 @@ public int featureSize() { return nvq.compressedVectorSize(); } + @Deprecated @Override public void writeHeader(IndexWriter out) throws IOException { - nvq.write(out, OnDiskGraphIndex.CURRENT_VERSION); + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) throws IOException { + nvq.write(out, version); out.writeLong(offset); } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedVectors.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedVectors.java index 017aef53f..b399574af 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedVectors.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedVectors.java @@ -19,6 +19,7 @@ import io.github.jbellis.jvector.disk.IndexWriter; import io.github.jbellis.jvector.disk.RandomAccessReader; import io.github.jbellis.jvector.graph.disk.CommonHeader; +import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex; import io.github.jbellis.jvector.vector.VectorizationProvider; import io.github.jbellis.jvector.vector.types.VectorTypeSupport; @@ -60,8 +61,14 @@ public int featureSize() { return dimension * Float.BYTES; } + @Deprecated @Override public void writeHeader(IndexWriter out) throws IOException { + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) throws IOException { out.writeLong(offset); } diff --git a/jvector-examples/yaml-configs/index-parameters/cap-10M.yml b/jvector-examples/yaml-configs/index-parameters/cap-10M.yml index c9c6c09d8..93b5b4eb9 100644 --- a/jvector-examples/yaml-configs/index-parameters/cap-10M.yml +++ b/jvector-examples/yaml-configs/index-parameters/cap-10M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: cap-10M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/cap-1M.yml b/jvector-examples/yaml-configs/index-parameters/cap-1M.yml index 6aa8454b5..08a8e38a4 100644 --- a/jvector-examples/yaml-configs/index-parameters/cap-1M.yml +++ b/jvector-examples/yaml-configs/index-parameters/cap-1M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: cap-1M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-10M.yml b/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-10M.yml index 17b3dd54a..97fb04c36 100644 --- a/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-10M.yml +++ b/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-10M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: cohere-english-v3-10M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-1M.yml b/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-1M.yml index 56fb2849f..dc9dcf7f2 100644 --- a/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-1M.yml +++ b/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-1M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: cohere-english-v3-1M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/colbert-1M.yml b/jvector-examples/yaml-configs/index-parameters/colbert-1M.yml index 66b47a056..3bcf9668a 100644 --- a/jvector-examples/yaml-configs/index-parameters/colbert-1M.yml +++ b/jvector-examples/yaml-configs/index-parameters/colbert-1M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: colbert-1M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/default.yml b/jvector-examples/yaml-configs/index-parameters/default.yml index e0a4ed3a5..fdb3da125 100644 --- a/jvector-examples/yaml-configs/index-parameters/default.yml +++ b/jvector-examples/yaml-configs/index-parameters/default.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: default # Prefetch bounded batches instead of preloading the entire dataset into memory. diff --git a/jvector-examples/yaml-configs/index-parameters/dpr-gemma-10M.yml b/jvector-examples/yaml-configs/index-parameters/dpr-gemma-10M.yml index 90eac34e8..16acbe6cf 100644 --- a/jvector-examples/yaml-configs/index-parameters/dpr-gemma-10M.yml +++ b/jvector-examples/yaml-configs/index-parameters/dpr-gemma-10M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: dpr-gemma-10M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/dpr-gemma-1M.yml b/jvector-examples/yaml-configs/index-parameters/dpr-gemma-1M.yml index 354c06436..a194ace14 100644 --- a/jvector-examples/yaml-configs/index-parameters/dpr-gemma-1M.yml +++ b/jvector-examples/yaml-configs/index-parameters/dpr-gemma-1M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: dpr-gemma-1M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/glove-100-angular.yml b/jvector-examples/yaml-configs/index-parameters/glove-100-angular.yml index 1b03b1edd..920fee6a8 100644 --- a/jvector-examples/yaml-configs/index-parameters/glove-100-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/glove-100-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: glove-100-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/glove-200-angular.yml b/jvector-examples/yaml-configs/index-parameters/glove-200-angular.yml index 1630baaba..f39457919 100644 --- a/jvector-examples/yaml-configs/index-parameters/glove-200-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/glove-200-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: glove-200-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/glove-25-angular.yml b/jvector-examples/yaml-configs/index-parameters/glove-25-angular.yml index bc2293a9d..e25426f0f 100644 --- a/jvector-examples/yaml-configs/index-parameters/glove-25-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/glove-25-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: glove-25-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/glove-50-angular.yml b/jvector-examples/yaml-configs/index-parameters/glove-50-angular.yml index 309ef9769..cde4630f9 100644 --- a/jvector-examples/yaml-configs/index-parameters/glove-50-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/glove-50-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: glove-50-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/lastfm-64-dot.yml b/jvector-examples/yaml-configs/index-parameters/lastfm-64-dot.yml index c2e2fb98a..5ac19c195 100644 --- a/jvector-examples/yaml-configs/index-parameters/lastfm-64-dot.yml +++ b/jvector-examples/yaml-configs/index-parameters/lastfm-64-dot.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: lastfm-64-dot useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/nytimes-256-angular.yml b/jvector-examples/yaml-configs/index-parameters/nytimes-256-angular.yml index 4e2db21a7..50e1ba072 100644 --- a/jvector-examples/yaml-configs/index-parameters/nytimes-256-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/nytimes-256-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: nytimes-256-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/sift-128-euclidean.yml b/jvector-examples/yaml-configs/index-parameters/sift-128-euclidean.yml index 94362ec70..93569a009 100644 --- a/jvector-examples/yaml-configs/index-parameters/sift-128-euclidean.yml +++ b/jvector-examples/yaml-configs/index-parameters/sift-128-euclidean.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: sift-128-euclidean useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/wiki-multilingual-v3-int8-1m.yml b/jvector-examples/yaml-configs/index-parameters/wiki-multilingual-v3-int8-1m.yml index d7692b697..e7d98b7dc 100644 --- a/jvector-examples/yaml-configs/index-parameters/wiki-multilingual-v3-int8-1m.yml +++ b/jvector-examples/yaml-configs/index-parameters/wiki-multilingual-v3-int8-1m.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: wiki-multilingual-v3-int8-1m diff --git a/jvector-examples/yaml-configs/index-parameters/wikipedia-2023-11-en-mxbai-int8-1m.yml b/jvector-examples/yaml-configs/index-parameters/wikipedia-2023-11-en-mxbai-int8-1m.yml index 02978921f..23b470734 100644 --- a/jvector-examples/yaml-configs/index-parameters/wikipedia-2023-11-en-mxbai-int8-1m.yml +++ b/jvector-examples/yaml-configs/index-parameters/wikipedia-2023-11-en-mxbai-int8-1m.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: wikipedia-2023-11-en-mxbai-int8-1m diff --git a/jvector-examples/yaml-configs/run-config.yml b/jvector-examples/yaml-configs/run-config.yml index ba5ee1077..04dd1e160 100644 --- a/jvector-examples/yaml-configs/run-config.yml +++ b/jvector-examples/yaml-configs/run-config.yml @@ -1,6 +1,6 @@ # Run-level controls for BenchYAML (applies to the whole invocation) yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 # Benchmark stats to compute benchmarks: diff --git a/jvector-tests/src/test/java/io/github/jbellis/jvector/example/TestDatasetPrefetching.java b/jvector-tests/src/test/java/io/github/jbellis/jvector/example/TestDatasetPrefetching.java index 5379cb0ef..af6f85f4f 100644 --- a/jvector-tests/src/test/java/io/github/jbellis/jvector/example/TestDatasetPrefetching.java +++ b/jvector-tests/src/test/java/io/github/jbellis/jvector/example/TestDatasetPrefetching.java @@ -49,18 +49,18 @@ private MultiConfig config(String build, String search, boolean fused, String re @Test public void yamlDefaultsToResidentAndOptInIsParsed() throws Exception { var file = Files.createTempFile("prefetch-flag", ".yml"); try { - Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\n"); + Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\n"); assertFalse(MultiConfig.getConfig(file.toString()).useDatasetPrefetching); - Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\nuseDatasetPrefetching: false\n"); + Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\nuseDatasetPrefetching: false\n"); assertFalse(MultiConfig.getConfig(file.toString()).useDatasetPrefetching); - Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\nuseDatasetPrefetching: true\n"); + Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\nuseDatasetPrefetching: true\n"); assertTrue(MultiConfig.getConfig(file.toString()).useDatasetPrefetching); } finally { Files.delete(file); } } @Test public void datasetPrefetchingAndCompactionOptionsAreIndependent() throws Exception { var file = Files.createTempFile("prefetch-compaction-flags", ".yml"); try { - String common = "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\nuseDatasetPrefetching: true\n"; + String common = "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\nuseDatasetPrefetching: true\n"; Files.writeString(file, common); var config = MultiConfig.getConfig(file.toString()); assertTrue(config.useDatasetPrefetching); @@ -78,7 +78,7 @@ private MultiConfig config(String build, String search, boolean fused, String re @Test public void versionedYamlRejectsUnknownSettings() throws Exception { var file = Files.createTempFile("prefetch-unknown-setting", ".yml"); try { - Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\nuseDatasetPrefetching: false\nunknownSetting: true\n"); + Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\nuseDatasetPrefetching: false\nunknownSetting: true\n"); assertThrows(org.yaml.snakeyaml.error.YAMLException.class, () -> MultiConfig.getConfig(file.toString())); } finally { Files.delete(file); } diff --git a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestFeatureHeaderVersion.java b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestFeatureHeaderVersion.java new file mode 100644 index 000000000..8f9be0f12 --- /dev/null +++ b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestFeatureHeaderVersion.java @@ -0,0 +1,111 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.jbellis.jvector.graph.disk; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import io.github.jbellis.jvector.TestUtil; +import io.github.jbellis.jvector.disk.ByteBufferIndexWriter; +import io.github.jbellis.jvector.disk.IndexWriter; +import io.github.jbellis.jvector.graph.ListRandomAccessVectorValues; +import io.github.jbellis.jvector.graph.disk.feature.Feature; +import io.github.jbellis.jvector.graph.disk.feature.FeatureId; +import io.github.jbellis.jvector.graph.disk.feature.FusedPQ; +import io.github.jbellis.jvector.graph.disk.feature.NVQ; +import io.github.jbellis.jvector.graph.disk.feature.SeparatedNVQ; +import io.github.jbellis.jvector.quantization.NVQuantization; +import io.github.jbellis.jvector.quantization.ProductQuantization; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; + +/** + * A feature header embedded in a graph serializes its compressor at the graph's version, not at + * {@link OnDiskGraphIndex#CURRENT_VERSION}, so a graph written at an older version is laid out as + * that version wrote it. + */ +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class TestFeatureHeaderVersion extends RandomizedTest { + private static final int DIMENSION = 32, MAX_DEGREE = 16; + + private interface Body { + void write(IndexWriter out, int version) throws IOException; + } + + @Test + public void featureHeadersAreWrittenAtTheGraphVersion() throws IOException { + var ravv = new ListRandomAccessVectorValues(TestUtil.createRandomVectors(512, DIMENSION), DIMENSION); + var pq = ProductQuantization.compute(ravv, 8, 256, false); + var nvq = NVQuantization.compute(ravv, 2); + + // Each feature's header is its compressor at the graph's version, plus any fields of its own. + var cases = List.of( + new Case(FeatureId.FUSED_PQ, new FusedPQ(MAX_DEGREE, pq), pq::write), + new Case(FeatureId.NVQ_VECTORS, new NVQ(nvq), nvq::write), + new Case(FeatureId.SEPARATED_NVQ, new SeparatedNVQ(nvq, 1234L), (out, v) -> { + nvq.write(out, v); + out.writeLong(1234L); + })); + + for (int version = 2; version <= OnDiskGraphIndex.CURRENT_VERSION; version++) { + var format = GraphIndexFormatFactory.forVersion(version); + for (var c : cases) { + if (!format.getSupportedFeatures().contains(c.id)) continue; + var features = new EnumMap(FeatureId.class); + features.put(c.id, c.feature); + byte[] written = bytes(out -> format.writeHeaderFeatures(out, features)); + int v = version; + byte[] expected = bytes(out -> c.body.write(out, v)); + // The format's own framing (feature ids, counts) precedes the header. + byte[] header = Arrays.copyOfRange(written, written.length - expected.length, written.length); + assertArrayEquals(c.id + " at version " + version, expected, header); + } + } + } + + private static final class Case { + final FeatureId id; + final Feature feature; + final Body body; + + Case(FeatureId id, Feature feature, Body body) { + this.id = id; + this.feature = feature; + this.body = body; + } + } + + private interface Writes { + void to(IndexWriter out) throws IOException; + } + + private static byte[] bytes(Writes writes) throws IOException { + try (var out = ByteBufferIndexWriter.create(1024 * 1024, false)) { + writes.to(out); + ByteBuffer data = out.getWrittenData(); + byte[] result = new byte[data.remaining()]; + data.get(result); + return result; + } + } +} diff --git a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestGraphIndexFormatFactory.java b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestGraphIndexFormatFactory.java index a3296937a..67d15e0f1 100644 --- a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestGraphIndexFormatFactory.java +++ b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestGraphIndexFormatFactory.java @@ -89,7 +89,7 @@ public void testFormatCharacteristicsPerVersion() { assertFalse(v2.usesFooter()); assertEquals(EnumSet.of(FeatureId.INLINE_VECTORS), v2.getSupportedFeatures()); - // versions 3-5 support every feature except FUSED_PQ and INLINE_BYTE_VECTORS (both v6+); + // versions 3-5 support every feature except FUSED_PQ (v6+) and INLINE_BYTE_VECTORS (v7+); // multi-layer arrives at v4, footer-based metadata arrives at v5. var nonFused = EnumSet.complementOf(EnumSet.of(FeatureId.FUSED_PQ, FeatureId.INLINE_BYTE_VECTORS)); for (int version : new int[]{3, 4, 5}) { @@ -104,7 +104,15 @@ public void testFormatCharacteristicsPerVersion() { assertTrue(v6.supportsMultiLayer()); assertTrue(v6.usesFooter()); assertTrue(v6.supportsFeature(FeatureId.FUSED_PQ)); - assertEquals(EnumSet.allOf(FeatureId.class), v6.getSupportedFeatures()); + assertFalse(v6.supportsFeature(FeatureId.INLINE_BYTE_VECTORS)); + assertEquals(EnumSet.complementOf(EnumSet.of(FeatureId.INLINE_BYTE_VECTORS)), v6.getSupportedFeatures()); + + var v7 = GraphIndexFormatFactory.forVersion(7); + assertTrue(v7.supportsMultiLayer()); + assertTrue(v7.usesFooter()); + assertTrue(v7.supportsFeature(FeatureId.FUSED_PQ)); + assertTrue(v7.supportsFeature(FeatureId.INLINE_BYTE_VECTORS)); + assertEquals(EnumSet.allOf(FeatureId.class), v7.getSupportedFeatures()); } @Test diff --git a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestOnDiskGraphIndex.java b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestOnDiskGraphIndex.java index f9c505b4e..d82c4e0b4 100644 --- a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestOnDiskGraphIndex.java +++ b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestOnDiskGraphIndex.java @@ -662,6 +662,17 @@ public void testInlineByteVectorsFeatureSize() { } } + @Test + public void testInlineByteVectorsRequiresVersion7() { + var graph = new TestUtil.RandomlyConnectedGraphIndex(32, 4, getRandom()); + var e = assertThrows(IllegalArgumentException.class, () -> new OnDiskGraphIndexWriter.Builder(graph, + testDirectory.resolve("version_6_inline_byte_vectors")) + .withVersion(6) + .with(new InlineByteVectors(8)) + .build()); + assertTrue(e.getMessage(), e.getMessage().startsWith("Feature INLINE_BYTE_VECTORS is not supported by version 6")); + } + /** * getByteVector throws UnsupportedOperationException when the graph has no INLINE_BYTE_VECTORS feature. */