diff --git a/UPGRADING.md b/UPGRADING.md index c99de606f..999e0c448 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,3 +1,16 @@ +# Upgrading from 4.0.x to 4.1.x + +## On-disk format version 7 +- Version 7 of the on-disk graph format adds one feature, inline int8 vectors (`FeatureId.INLINE_BYTE_VECTORS`): + each node's byte vector stored inline, one byte per dimension. Apart from that feature, a version 7 file has + the version 6 layout. +- JVector 4.0.x reads formats up to 6 and rejects version 7 files. `OnDiskGraphIndex.CURRENT_VERSION`, + the default for `OnDiskGraphIndexWriter.Builder` and for `CompressedVectors.write(out)`, is now 7. +- To keep a rollback to 4.0.x possible, pass the version explicitly until every reader runs 4.1: + `withVersion(6)` on graph writer builders and `write(out, 6)` for compressors and compressed vectors. + A version 6 writer rejects `INLINE_BYTE_VECTORS`. +- `OnDiskGraphIndexCompactor` always writes `CURRENT_VERSION`; it has no option to write an older version. + # Upgrading from 3.0.x to 4.0.x ## New features diff --git a/docs/release notes/4.1.0/709.feature.md b/docs/release notes/4.1.0/709.feature.md index cbb188527..dcd6a71c0 100644 --- a/docs/release notes/4.1.0/709.feature.md +++ b/docs/release notes/4.1.0/709.feature.md @@ -49,4 +49,5 @@ try (var builder = GraphIndexBuilder.builder(vectors, ByteVectorSimilarityFuncti * Zero vectors are not supported with `COSINE` (the result is undefined); do not index them. * Existing float32 callers keep working: `RandomAccessVectorValues.getVector` keeps its `VectorFloat` signature and `GraphIndexBuilder.build(RandomAccessVectorValues)` is retained as a forwarding overload. -* Int8 indexes require on-disk format version 6 or later. +* Int8 indexes require on-disk format version 7 or later. JVector 4.0.x reads formats up to 6 and rejects + version 7 files; a version 6 writer rejects `INLINE_BYTE_VECTORS`. diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/AbstractGraphIndexFormat.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/AbstractGraphIndexFormat.java index 2f77cf768..12bb1d28a 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/AbstractGraphIndexFormat.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/AbstractGraphIndexFormat.java @@ -381,7 +381,7 @@ public int commonHeaderSize() { public void writeHeaderFeatures(IndexWriter out, Map features) throws IOException { // we restrict pre-version-3 writers to INLINE_VECTORS features, so we don't need additional version-handling here for (Feature writer : features.values()) { - writer.writeHeader(out); + writer.writeHeader(out, getVersion()); } } @@ -415,7 +415,7 @@ public OnDiskGraphIndex loadOnDiskIndex(RandomAccessReader reader, Header header */ protected static Set allFeatures() { return EnumSet.of(FeatureId.INLINE_VECTORS, FeatureId.FUSED_PQ, FeatureId.NVQ_VECTORS, - FeatureId.SEPARATED_VECTORS, FeatureId.SEPARATED_NVQ, FeatureId.INLINE_BYTE_VECTORS); + FeatureId.SEPARATED_VECTORS, FeatureId.SEPARATED_NVQ); } /** diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatFactory.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatFactory.java index 12010bc1e..b7c0014a1 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatFactory.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatFactory.java @@ -35,7 +35,8 @@ public class GraphIndexFormatFactory { 3, new GraphIndexFormatV3(), 4, new GraphIndexFormatV4(), 5, new GraphIndexFormatV5(), - 6, new GraphIndexFormatV6() + 6, new GraphIndexFormatV6(), + 7, new GraphIndexFormatV7() ); /** diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatV6.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatV6.java index c9d965aa4..a9a46ed5c 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatV6.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/GraphIndexFormatV6.java @@ -26,6 +26,7 @@ import java.util.EnumMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.function.IntFunction; import java.util.stream.Collectors; @@ -44,7 +45,12 @@ class GraphIndexFormatV6 extends GraphIndexFormatV5 { /** Creates the singleton format for version 6. */ GraphIndexFormatV6() { - super(6, allFeatures()); + this(6, allFeatures()); + } + + /** Allows later feature extensions to reuse the V6 record and hierarchy layout. */ + protected GraphIndexFormatV6(int version, Set supportedFeatures) { + super(version, supportedFeatures); } /** Places fused features last so non-fused inline features occupy a contiguous prefix. */ @@ -112,7 +118,7 @@ public void writeHeaderFeatures(IndexWriter out, Map supportedFeatures() { + var features = EnumSet.copyOf(allFeatures()); + features.add(FeatureId.INLINE_BYTE_VECTORS); + return features; + } +} diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/OnDiskGraphIndex.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/OnDiskGraphIndex.java index 92f43292d..c67461162 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/OnDiskGraphIndex.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/OnDiskGraphIndex.java @@ -71,7 +71,7 @@ public class OnDiskGraphIndex implements ImmutableGraphIndex, AutoCloseable, Accountable { private static final Logger logger = LoggerFactory.getLogger(OnDiskGraphIndex.class); - public static final int CURRENT_VERSION = 6; + public static final int CURRENT_VERSION = 7; static final int MAGIC = 0xFFFF0D61; // FFFF to distinguish from old graphs, which should never start with a negative size "ODGI" static final VectorTypeSupport vectorTypeSupport = VectorizationProvider.getInstance().getVectorTypeSupport(); final ReaderSupplier readerSupplier; diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/Feature.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/Feature.java index 03dd2ec98..1602aa2fa 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/Feature.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/Feature.java @@ -36,8 +36,27 @@ default boolean isFused() { int featureSize(); + /** + * Writes this feature's header at {@link io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex#CURRENT_VERSION}. + * + * @deprecated a header embedded in a graph must be written at the graph's version; use + * {@link #writeHeader(IndexWriter, int)} + */ + @Deprecated void writeHeader(IndexWriter out) throws IOException; + /** + * Writes this feature's header for a graph written at {@code version}. Features that serialize a + * compressor write it at this version, so a graph written at an older version is laid out as that + * version wrote it. The default ignores the version, for headers that don't depend on it. + * + * @param out the output to write to + * @param version the on-disk format version of the graph being written + */ + default void writeHeader(IndexWriter out, int version) throws IOException { + writeHeader(out); + } + default void writeInline(IndexWriter out, State state) throws IOException { // default no-op } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/FusedPQ.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/FusedPQ.java index 3dc404481..a8d39aa1b 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/FusedPQ.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/FusedPQ.java @@ -135,9 +135,15 @@ public QuantizationCompactionStrategy createCompactionStrategy(CompactionContext return new FusedCompactionStrategy(ctx, this, retrainer); } + @Deprecated @Override public void writeHeader(IndexWriter out) throws IOException { - pq.write(out, OnDiskGraphIndex.CURRENT_VERSION); + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) throws IOException { + pq.write(out, version); } // this is an awkward fit for the Feature.State design since we need to diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineByteVectors.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineByteVectors.java index 86ebfff3c..981999b95 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineByteVectors.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineByteVectors.java @@ -20,6 +20,7 @@ import io.github.jbellis.jvector.disk.IndexWriter; import io.github.jbellis.jvector.disk.RandomAccessReader; import io.github.jbellis.jvector.graph.disk.CommonHeader; +import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex; import io.github.jbellis.jvector.vector.VectorizationProvider; import io.github.jbellis.jvector.vector.types.ByteSequence; import io.github.jbellis.jvector.vector.types.VectorTypeSupport; @@ -72,8 +73,14 @@ static InlineByteVectors load(CommonHeader header, RandomAccessReader reader) { return new InlineByteVectors(header.dimension); } + @Deprecated @Override public void writeHeader(IndexWriter out) { + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) { // common header carries dimension; nothing extra needed } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineVectors.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineVectors.java index 005103ddb..cac33f6f2 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineVectors.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/InlineVectors.java @@ -19,6 +19,7 @@ import io.github.jbellis.jvector.disk.IndexWriter; import io.github.jbellis.jvector.disk.RandomAccessReader; import io.github.jbellis.jvector.graph.disk.CommonHeader; +import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex; import io.github.jbellis.jvector.vector.VectorizationProvider; import io.github.jbellis.jvector.vector.types.VectorFloat; import io.github.jbellis.jvector.vector.types.VectorTypeSupport; @@ -58,8 +59,14 @@ static InlineVectors load(CommonHeader header, RandomAccessReader reader) { return new InlineVectors(header.dimension); } + @Deprecated @Override public void writeHeader(IndexWriter out) { + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) { // common header contains dimension, which is sufficient } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/NVQ.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/NVQ.java index 492bb0dfe..863780bf1 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/NVQ.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/NVQ.java @@ -69,9 +69,15 @@ static NVQ load(CommonHeader header, RandomAccessReader reader) { } } + @Deprecated @Override public void writeHeader(IndexWriter out) throws IOException { - nvq.write(out, OnDiskGraphIndex.CURRENT_VERSION); + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) throws IOException { + nvq.write(out, version); } @Override diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedNVQ.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedNVQ.java index b65370632..a0e3ddd21 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedNVQ.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedNVQ.java @@ -67,9 +67,15 @@ public int featureSize() { return nvq.compressedVectorSize(); } + @Deprecated @Override public void writeHeader(IndexWriter out) throws IOException { - nvq.write(out, OnDiskGraphIndex.CURRENT_VERSION); + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) throws IOException { + nvq.write(out, version); out.writeLong(offset); } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedVectors.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedVectors.java index 017aef53f..b399574af 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedVectors.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/SeparatedVectors.java @@ -19,6 +19,7 @@ import io.github.jbellis.jvector.disk.IndexWriter; import io.github.jbellis.jvector.disk.RandomAccessReader; import io.github.jbellis.jvector.graph.disk.CommonHeader; +import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex; import io.github.jbellis.jvector.vector.VectorizationProvider; import io.github.jbellis.jvector.vector.types.VectorTypeSupport; @@ -60,8 +61,14 @@ public int featureSize() { return dimension * Float.BYTES; } + @Deprecated @Override public void writeHeader(IndexWriter out) throws IOException { + writeHeader(out, OnDiskGraphIndex.CURRENT_VERSION); + } + + @Override + public void writeHeader(IndexWriter out, int version) throws IOException { out.writeLong(offset); } diff --git a/jvector-examples/yaml-configs/index-parameters/cap-10M.yml b/jvector-examples/yaml-configs/index-parameters/cap-10M.yml index c9c6c09d8..93b5b4eb9 100644 --- a/jvector-examples/yaml-configs/index-parameters/cap-10M.yml +++ b/jvector-examples/yaml-configs/index-parameters/cap-10M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: cap-10M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/cap-1M.yml b/jvector-examples/yaml-configs/index-parameters/cap-1M.yml index 6aa8454b5..08a8e38a4 100644 --- a/jvector-examples/yaml-configs/index-parameters/cap-1M.yml +++ b/jvector-examples/yaml-configs/index-parameters/cap-1M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: cap-1M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-10M.yml b/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-10M.yml index 17b3dd54a..97fb04c36 100644 --- a/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-10M.yml +++ b/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-10M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: cohere-english-v3-10M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-1M.yml b/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-1M.yml index 56fb2849f..dc9dcf7f2 100644 --- a/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-1M.yml +++ b/jvector-examples/yaml-configs/index-parameters/cohere-english-v3-1M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: cohere-english-v3-1M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/colbert-1M.yml b/jvector-examples/yaml-configs/index-parameters/colbert-1M.yml index 66b47a056..3bcf9668a 100644 --- a/jvector-examples/yaml-configs/index-parameters/colbert-1M.yml +++ b/jvector-examples/yaml-configs/index-parameters/colbert-1M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: colbert-1M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/default.yml b/jvector-examples/yaml-configs/index-parameters/default.yml index e0a4ed3a5..fdb3da125 100644 --- a/jvector-examples/yaml-configs/index-parameters/default.yml +++ b/jvector-examples/yaml-configs/index-parameters/default.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: default # Prefetch bounded batches instead of preloading the entire dataset into memory. diff --git a/jvector-examples/yaml-configs/index-parameters/dpr-gemma-10M.yml b/jvector-examples/yaml-configs/index-parameters/dpr-gemma-10M.yml index 90eac34e8..16acbe6cf 100644 --- a/jvector-examples/yaml-configs/index-parameters/dpr-gemma-10M.yml +++ b/jvector-examples/yaml-configs/index-parameters/dpr-gemma-10M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: dpr-gemma-10M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/dpr-gemma-1M.yml b/jvector-examples/yaml-configs/index-parameters/dpr-gemma-1M.yml index 354c06436..a194ace14 100644 --- a/jvector-examples/yaml-configs/index-parameters/dpr-gemma-1M.yml +++ b/jvector-examples/yaml-configs/index-parameters/dpr-gemma-1M.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: dpr-gemma-1M useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/glove-100-angular.yml b/jvector-examples/yaml-configs/index-parameters/glove-100-angular.yml index 1b03b1edd..920fee6a8 100644 --- a/jvector-examples/yaml-configs/index-parameters/glove-100-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/glove-100-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: glove-100-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/glove-200-angular.yml b/jvector-examples/yaml-configs/index-parameters/glove-200-angular.yml index 1630baaba..f39457919 100644 --- a/jvector-examples/yaml-configs/index-parameters/glove-200-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/glove-200-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: glove-200-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/glove-25-angular.yml b/jvector-examples/yaml-configs/index-parameters/glove-25-angular.yml index bc2293a9d..e25426f0f 100644 --- a/jvector-examples/yaml-configs/index-parameters/glove-25-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/glove-25-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: glove-25-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/glove-50-angular.yml b/jvector-examples/yaml-configs/index-parameters/glove-50-angular.yml index 309ef9769..cde4630f9 100644 --- a/jvector-examples/yaml-configs/index-parameters/glove-50-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/glove-50-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: glove-50-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/lastfm-64-dot.yml b/jvector-examples/yaml-configs/index-parameters/lastfm-64-dot.yml index c2e2fb98a..5ac19c195 100644 --- a/jvector-examples/yaml-configs/index-parameters/lastfm-64-dot.yml +++ b/jvector-examples/yaml-configs/index-parameters/lastfm-64-dot.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: lastfm-64-dot useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/nytimes-256-angular.yml b/jvector-examples/yaml-configs/index-parameters/nytimes-256-angular.yml index 4e2db21a7..50e1ba072 100644 --- a/jvector-examples/yaml-configs/index-parameters/nytimes-256-angular.yml +++ b/jvector-examples/yaml-configs/index-parameters/nytimes-256-angular.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: nytimes-256-angular useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/sift-128-euclidean.yml b/jvector-examples/yaml-configs/index-parameters/sift-128-euclidean.yml index 94362ec70..93569a009 100644 --- a/jvector-examples/yaml-configs/index-parameters/sift-128-euclidean.yml +++ b/jvector-examples/yaml-configs/index-parameters/sift-128-euclidean.yml @@ -1,6 +1,6 @@ # This is here for loader testing purposes only, use other datasets for realistic test data yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: sift-128-euclidean useDatasetPrefetching: false diff --git a/jvector-examples/yaml-configs/index-parameters/wiki-multilingual-v3-int8-1m.yml b/jvector-examples/yaml-configs/index-parameters/wiki-multilingual-v3-int8-1m.yml index d7692b697..e7d98b7dc 100644 --- a/jvector-examples/yaml-configs/index-parameters/wiki-multilingual-v3-int8-1m.yml +++ b/jvector-examples/yaml-configs/index-parameters/wiki-multilingual-v3-int8-1m.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: wiki-multilingual-v3-int8-1m diff --git a/jvector-examples/yaml-configs/index-parameters/wikipedia-2023-11-en-mxbai-int8-1m.yml b/jvector-examples/yaml-configs/index-parameters/wikipedia-2023-11-en-mxbai-int8-1m.yml index 02978921f..23b470734 100644 --- a/jvector-examples/yaml-configs/index-parameters/wikipedia-2023-11-en-mxbai-int8-1m.yml +++ b/jvector-examples/yaml-configs/index-parameters/wikipedia-2023-11-en-mxbai-int8-1m.yml @@ -1,5 +1,5 @@ yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 dataset: wikipedia-2023-11-en-mxbai-int8-1m diff --git a/jvector-examples/yaml-configs/run-config.yml b/jvector-examples/yaml-configs/run-config.yml index ba5ee1077..04dd1e160 100644 --- a/jvector-examples/yaml-configs/run-config.yml +++ b/jvector-examples/yaml-configs/run-config.yml @@ -1,6 +1,6 @@ # Run-level controls for BenchYAML (applies to the whole invocation) yamlSchemaVersion: 1 -onDiskIndexVersion: 6 +onDiskIndexVersion: 7 # Benchmark stats to compute benchmarks: diff --git a/jvector-tests/src/test/java/io/github/jbellis/jvector/example/TestDatasetPrefetching.java b/jvector-tests/src/test/java/io/github/jbellis/jvector/example/TestDatasetPrefetching.java index 5379cb0ef..af6f85f4f 100644 --- a/jvector-tests/src/test/java/io/github/jbellis/jvector/example/TestDatasetPrefetching.java +++ b/jvector-tests/src/test/java/io/github/jbellis/jvector/example/TestDatasetPrefetching.java @@ -49,18 +49,18 @@ private MultiConfig config(String build, String search, boolean fused, String re @Test public void yamlDefaultsToResidentAndOptInIsParsed() throws Exception { var file = Files.createTempFile("prefetch-flag", ".yml"); try { - Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\n"); + Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\n"); assertFalse(MultiConfig.getConfig(file.toString()).useDatasetPrefetching); - Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\nuseDatasetPrefetching: false\n"); + Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\nuseDatasetPrefetching: false\n"); assertFalse(MultiConfig.getConfig(file.toString()).useDatasetPrefetching); - Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\nuseDatasetPrefetching: true\n"); + Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\nuseDatasetPrefetching: true\n"); assertTrue(MultiConfig.getConfig(file.toString()).useDatasetPrefetching); } finally { Files.delete(file); } } @Test public void datasetPrefetchingAndCompactionOptionsAreIndependent() throws Exception { var file = Files.createTempFile("prefetch-compaction-flags", ".yml"); try { - String common = "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\nuseDatasetPrefetching: true\n"; + String common = "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\nuseDatasetPrefetching: true\n"; Files.writeString(file, common); var config = MultiConfig.getConfig(file.toString()); assertTrue(config.useDatasetPrefetching); @@ -78,7 +78,7 @@ private MultiConfig config(String build, String search, boolean fused, String re @Test public void versionedYamlRejectsUnknownSettings() throws Exception { var file = Files.createTempFile("prefetch-unknown-setting", ".yml"); try { - Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: 6\nuseDatasetPrefetching: false\nunknownSetting: true\n"); + Files.writeString(file, "yamlSchemaVersion: 1\nonDiskIndexVersion: " + OnDiskGraphIndex.CURRENT_VERSION + "\nuseDatasetPrefetching: false\nunknownSetting: true\n"); assertThrows(org.yaml.snakeyaml.error.YAMLException.class, () -> MultiConfig.getConfig(file.toString())); } finally { Files.delete(file); } diff --git a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestFeatureHeaderVersion.java b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestFeatureHeaderVersion.java new file mode 100644 index 000000000..8f9be0f12 --- /dev/null +++ b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestFeatureHeaderVersion.java @@ -0,0 +1,111 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.jbellis.jvector.graph.disk; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import io.github.jbellis.jvector.TestUtil; +import io.github.jbellis.jvector.disk.ByteBufferIndexWriter; +import io.github.jbellis.jvector.disk.IndexWriter; +import io.github.jbellis.jvector.graph.ListRandomAccessVectorValues; +import io.github.jbellis.jvector.graph.disk.feature.Feature; +import io.github.jbellis.jvector.graph.disk.feature.FeatureId; +import io.github.jbellis.jvector.graph.disk.feature.FusedPQ; +import io.github.jbellis.jvector.graph.disk.feature.NVQ; +import io.github.jbellis.jvector.graph.disk.feature.SeparatedNVQ; +import io.github.jbellis.jvector.quantization.NVQuantization; +import io.github.jbellis.jvector.quantization.ProductQuantization; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; + +/** + * A feature header embedded in a graph serializes its compressor at the graph's version, not at + * {@link OnDiskGraphIndex#CURRENT_VERSION}, so a graph written at an older version is laid out as + * that version wrote it. + */ +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class TestFeatureHeaderVersion extends RandomizedTest { + private static final int DIMENSION = 32, MAX_DEGREE = 16; + + private interface Body { + void write(IndexWriter out, int version) throws IOException; + } + + @Test + public void featureHeadersAreWrittenAtTheGraphVersion() throws IOException { + var ravv = new ListRandomAccessVectorValues(TestUtil.createRandomVectors(512, DIMENSION), DIMENSION); + var pq = ProductQuantization.compute(ravv, 8, 256, false); + var nvq = NVQuantization.compute(ravv, 2); + + // Each feature's header is its compressor at the graph's version, plus any fields of its own. + var cases = List.of( + new Case(FeatureId.FUSED_PQ, new FusedPQ(MAX_DEGREE, pq), pq::write), + new Case(FeatureId.NVQ_VECTORS, new NVQ(nvq), nvq::write), + new Case(FeatureId.SEPARATED_NVQ, new SeparatedNVQ(nvq, 1234L), (out, v) -> { + nvq.write(out, v); + out.writeLong(1234L); + })); + + for (int version = 2; version <= OnDiskGraphIndex.CURRENT_VERSION; version++) { + var format = GraphIndexFormatFactory.forVersion(version); + for (var c : cases) { + if (!format.getSupportedFeatures().contains(c.id)) continue; + var features = new EnumMap(FeatureId.class); + features.put(c.id, c.feature); + byte[] written = bytes(out -> format.writeHeaderFeatures(out, features)); + int v = version; + byte[] expected = bytes(out -> c.body.write(out, v)); + // The format's own framing (feature ids, counts) precedes the header. + byte[] header = Arrays.copyOfRange(written, written.length - expected.length, written.length); + assertArrayEquals(c.id + " at version " + version, expected, header); + } + } + } + + private static final class Case { + final FeatureId id; + final Feature feature; + final Body body; + + Case(FeatureId id, Feature feature, Body body) { + this.id = id; + this.feature = feature; + this.body = body; + } + } + + private interface Writes { + void to(IndexWriter out) throws IOException; + } + + private static byte[] bytes(Writes writes) throws IOException { + try (var out = ByteBufferIndexWriter.create(1024 * 1024, false)) { + writes.to(out); + ByteBuffer data = out.getWrittenData(); + byte[] result = new byte[data.remaining()]; + data.get(result); + return result; + } + } +} diff --git a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestGraphIndexFormatFactory.java b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestGraphIndexFormatFactory.java index a3296937a..67d15e0f1 100644 --- a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestGraphIndexFormatFactory.java +++ b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestGraphIndexFormatFactory.java @@ -89,7 +89,7 @@ public void testFormatCharacteristicsPerVersion() { assertFalse(v2.usesFooter()); assertEquals(EnumSet.of(FeatureId.INLINE_VECTORS), v2.getSupportedFeatures()); - // versions 3-5 support every feature except FUSED_PQ and INLINE_BYTE_VECTORS (both v6+); + // versions 3-5 support every feature except FUSED_PQ (v6+) and INLINE_BYTE_VECTORS (v7+); // multi-layer arrives at v4, footer-based metadata arrives at v5. var nonFused = EnumSet.complementOf(EnumSet.of(FeatureId.FUSED_PQ, FeatureId.INLINE_BYTE_VECTORS)); for (int version : new int[]{3, 4, 5}) { @@ -104,7 +104,15 @@ public void testFormatCharacteristicsPerVersion() { assertTrue(v6.supportsMultiLayer()); assertTrue(v6.usesFooter()); assertTrue(v6.supportsFeature(FeatureId.FUSED_PQ)); - assertEquals(EnumSet.allOf(FeatureId.class), v6.getSupportedFeatures()); + assertFalse(v6.supportsFeature(FeatureId.INLINE_BYTE_VECTORS)); + assertEquals(EnumSet.complementOf(EnumSet.of(FeatureId.INLINE_BYTE_VECTORS)), v6.getSupportedFeatures()); + + var v7 = GraphIndexFormatFactory.forVersion(7); + assertTrue(v7.supportsMultiLayer()); + assertTrue(v7.usesFooter()); + assertTrue(v7.supportsFeature(FeatureId.FUSED_PQ)); + assertTrue(v7.supportsFeature(FeatureId.INLINE_BYTE_VECTORS)); + assertEquals(EnumSet.allOf(FeatureId.class), v7.getSupportedFeatures()); } @Test diff --git a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestOnDiskGraphIndex.java b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestOnDiskGraphIndex.java index f9c505b4e..d82c4e0b4 100644 --- a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestOnDiskGraphIndex.java +++ b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/disk/TestOnDiskGraphIndex.java @@ -662,6 +662,17 @@ public void testInlineByteVectorsFeatureSize() { } } + @Test + public void testInlineByteVectorsRequiresVersion7() { + var graph = new TestUtil.RandomlyConnectedGraphIndex(32, 4, getRandom()); + var e = assertThrows(IllegalArgumentException.class, () -> new OnDiskGraphIndexWriter.Builder(graph, + testDirectory.resolve("version_6_inline_byte_vectors")) + .withVersion(6) + .with(new InlineByteVectors(8)) + .build()); + assertTrue(e.getMessage(), e.getMessage().startsWith("Feature INLINE_BYTE_VECTORS is not supported by version 6")); + } + /** * getByteVector throws UnsupportedOperationException when the graph has no INLINE_BYTE_VECTORS feature. */