From f434af83d35311cdd7da386f6797406089f78294 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 10:22:03 +0900 Subject: [PATCH 1/4] Tests: segment file damage, byte by byte (#17) Truncation at every length, one flipped bit in every byte that a CRC or a length check covers (header, footer, locator, every section), a wrong magic, appended bytes, and files that are no segment. Every case must be refused with a CorruptException (UnsupportedException for the magic and the major version) and never return a graph or fail in any other way. The zero fill that no CRC covers is pinned as "damage there changes nothing that is served". SegmentForge takes a good file apart and rebuilds it with chosen fields changed and the CRCs above the change recomputed; the next commit uses it. Co-Authored-By: Claude Sonnet 5.5 --- .../humemai/vec/SegmentCorruptionTest.java | 326 ++++++++++++++++++ .../java/com/humemai/vec/SegmentForge.java | 189 ++++++++++ 2 files changed, 515 insertions(+) create mode 100644 src/test/java/com/humemai/vec/SegmentCorruptionTest.java create mode 100644 src/test/java/com/humemai/vec/SegmentForge.java diff --git a/src/test/java/com/humemai/vec/SegmentCorruptionTest.java b/src/test/java/com/humemai/vec/SegmentCorruptionTest.java new file mode 100644 index 0000000..733c414 --- /dev/null +++ b/src/test/java/com/humemai/vec/SegmentCorruptionTest.java @@ -0,0 +1,326 @@ +package com.humemai.vec; + +import static com.humemai.vec.SegmentForge.refused; +import static com.humemai.vec.SegmentTestSupport.graph; +import static com.humemai.vec.SegmentTestSupport.params; +import static com.humemai.vec.SegmentTestSupport.vectors; +import static com.humemai.vec.SegmentTestSupport.versOf; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.humemai.vec.SegmentFormat.CorruptException; +import com.humemai.vec.SegmentFormat.UnsupportedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Damage to a segment file, byte by byte. A file that is cut short, or has any bit of its header, footer, locator or sections flipped, is + * refused with a {@link CorruptException} (or an {@link UnsupportedException} where the damaged bytes are the magic or the major version), + * never loaded, and never fails in any other way. The sweeps are exhaustive over a small file: every cut length, and every byte that a + * CRC or a length check covers. The only bytes whose damage is not detected are the zero fill that no CRC covers (the rest of the header + * block and the gaps between sections), and a test below shows that damage there changes nothing a reader serves. + * + *

The code reads files with plain {@code FileChannel} reads into heap arrays (no memory mapping and no native memory in this version), + * so a damaged file cannot take the JVM down; what the tests rule out is every other uncontrolled failure: an exception that is not one of + * the two named ones, an out-of-memory error from a length read out of the file, or a graph that is returned. + */ +class SegmentCorruptionTest { + private static final int N = 300, DIM = 4; + + @TempDir + Path tmp; + + private SealedGraph orig; + private byte[] good; + private SegmentInfo info; + private Path scratch; + + @BeforeEach + void write() throws IOException { + orig = graph(vectors(N, DIM, 17), Metric.L2, params(true)); // SQ8 on: all twelve sections are in the file + Path f = tmp.resolve("good.tvs"); + info = SegmentWriter.write(orig, versOf(N), f); + good = Files.readAllBytes(f); + scratch = tmp.resolve("scratch.tvs"); + Files.write(scratch, good); + assertEquals(12, info.sections().size()); + for (SegmentInfo.Section s : info.sections()) assertTrue(s.length() > 0, "section " + SegmentFormat.sectionName(s.id()) + " has bytes to damage"); + } + + private int footerOff() { + return good.length - SegmentFormat.LOCATOR_LEN - (SegmentFormat.F_META_LEN + SegmentFormat.ENTRY_LEN * info.sections().size()); + } + + /** flips one bit of the scratch file in place; returns the byte to give back to {@link #restore} */ + private byte flip(FileChannel ch, int pos, int bit) throws IOException { + byte old = good[pos]; + ch.write(ByteBuffer.wrap(new byte[] {(byte) (old ^ (1 << bit))}), pos); + return old; + } + + private void restore(FileChannel ch, int pos) throws IOException { + ch.write(ByteBuffer.wrap(new byte[] {good[pos]}), pos); + } + + // ---- truncation ---- + + @Test + void everyCutLengthIsRefusedWithACorruptException() throws IOException { + // the scratch file is cut one byte at a time from its full length down to nothing; a cut inside the header block leaves too little for a + // segment, every other cut leaves a file whose last 32 bytes are not the locator + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int len = good.length - 1; len >= 0; len--) { + ch.truncate(len); + CorruptException e = assertInstanceOf(CorruptException.class, refused(scratch, "cut at " + len), "cut at " + len); + assertEquals(len < SegmentFormat.HEADER_BLOCK + SegmentFormat.LOCATOR_LEN ? "HEADER" : "LOCATOR", e.section(), "cut at " + len); + } + } + assertEquals(0, Files.size(scratch)); + } + + @ParameterizedTest(name = "section id {0}") + @ValueSource(ints = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}) + void aFileCutInTheMiddleOfASectionOrAtItsLastByteFailsInConstantTime(int id) throws IOException { + SegmentInfo.Section sec = info.section(id); + for (long cut : new long[] {sec.offset() + sec.length() / 2, sec.offset() + sec.length() - 1}) { + Path p = tmp.resolve("cut-" + id + "-" + cut + ".tvs"); + Files.write(p, java.util.Arrays.copyOf(good, (int) cut)); + // inspect reads the header block and the last 32 bytes only, so no section is read + CorruptException e = assertInstanceOf(CorruptException.class, assertThrowsIo(() -> SegmentReader.inspect(p), "cut at " + cut)); + assertEquals("LOCATOR", e.section()); + assertEquals("LOCATOR", assertInstanceOf(CorruptException.class, refused(p, false, "cut at " + cut)).section()); + assertEquals("LOCATOR", assertInstanceOf(CorruptException.class, refused(p, true, "cut at " + cut)).section()); + } + } + + @Test + void aFileCutInsideTheFooterOrTheLocatorOrAtTheEndOfTheSectionsIsRefused() throws IOException { + final int footerOff = footerOff(); + final int[] cuts = {footerOff, footerOff + 1, footerOff + SegmentFormat.F_META_LEN, good.length - SegmentFormat.LOCATOR_LEN - 1, + good.length - SegmentFormat.LOCATOR_LEN, good.length - SegmentFormat.LOCATOR_LEN + 1, good.length - 8, good.length - 1}; + for (int cut : cuts) { + Path p = tmp.resolve("tail-" + cut + ".tvs"); + Files.write(p, java.util.Arrays.copyOf(good, cut)); + assertEquals("LOCATOR", assertInstanceOf(CorruptException.class, refused(p, "cut at " + cut)).section(), "cut at " + cut); + } + } + + @Test + void bytesAppendedToAGoodFileAreRefused() throws IOException { + for (int extra : new int[] {1, 7, 32, 4096}) { + byte[] b = java.util.Arrays.copyOf(good, good.length + extra); // zeros after the locator: it is no longer the last 32 bytes + Path p = tmp.resolve("appended-" + extra + ".tvs"); + Files.write(p, b); + assertEquals("LOCATOR", assertInstanceOf(CorruptException.class, refused(p, extra + " bytes appended")).section()); + } + } + + // ---- bit flips ---- + + @Test + void everyBitOfTheHeaderIsGuarded() throws IOException { + // bytes 0 to 9 are the magic and the major version: a flipped bit there names a file this reader does not understand. Bytes 10 to 51 + // (minor version, header length, probe, flags, dimensions, enum codes, alignment, the CRC itself) are covered by the header CRC. + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int pos = 0; pos < SegmentFormat.HEADER_LEN; pos++) { + for (int bit = 0; bit < 8; bit++) { + flip(ch, pos, bit); + String what = "bit " + bit + " of header byte " + pos; + for (boolean verify : new boolean[] {true, false}) { + IOException e = refused(scratch, verify, what); + if (pos < 10) assertInstanceOf(UnsupportedException.class, e, what); + else assertEquals("HEADER", assertInstanceOf(CorruptException.class, e, what).section(), what); + } + restore(ch, pos); + } + } + } + } + + @Test + void everyBitOfTheFooterIsGuarded() throws IOException { + // the footer CRC in the locator covers the metadata and every section entry. The section table, with its offsets, widths, counts and + // CRCs, therefore cannot change without notice, and the check runs before any section is read (verifyAll false below). + final int footerOff = footerOff(), footerEnd = good.length - SegmentFormat.LOCATOR_LEN; + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int pos = footerOff; pos < footerEnd; pos++) { + for (int bit = 0; bit < 8; bit++) { + flip(ch, pos, bit); + String what = "bit " + bit + " of footer byte " + (pos - footerOff); + for (boolean verify : new boolean[] {true, false}) { + assertEquals("FOOTER", assertInstanceOf(CorruptException.class, refused(scratch, verify, what), what).section(), what); + } + restore(ch, pos); + } + } + } + } + + @Test + void everyBitOfTheLocatorIsGuarded() throws IOException { + // magic (0 to 7), footer offset (8 to 15), footer length (16 to 19) and file length (24 to 31) are checked against the file itself; the + // footer CRC (20 to 23) is checked against the footer + final int locOff = good.length - SegmentFormat.LOCATOR_LEN; + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int rel = 0; rel < SegmentFormat.LOCATOR_LEN; rel++) { + for (int bit = 0; bit < 8; bit++) { + flip(ch, locOff + rel, bit); + String what = "bit " + bit + " of locator byte " + rel; + for (boolean verify : new boolean[] {true, false}) { + CorruptException e = assertInstanceOf(CorruptException.class, refused(scratch, verify, what), what); + assertEquals(rel >= 20 && rel < 24 ? "FOOTER" : "LOCATOR", e.section(), what); + } + restore(ch, locOff + rel); + } + } + } + } + + @Test + void everyByteOfEverySectionIsCoveredByItsCrc() throws IOException { + // every byte of every section has one bit flipped (the bit changes from byte to byte), and the first, the middle and the last byte of each + // section have all eight flipped. The metadata is intact in all of these, so the CRC of exactly the damaged section must be what fails. + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (SegmentInfo.Section sec : info.sections()) { + final String name = SegmentFormat.sectionName(sec.id()); + final int first = (int) sec.offset(), last = (int) (sec.offset() + sec.length() - 1), mid = (int) (sec.offset() + sec.length() / 2); + for (int pos = first; pos <= last; pos++) { + boolean all = pos == first || pos == mid || pos == last; + for (int bit = all ? 0 : pos % 8; bit < (all ? 8 : pos % 8 + 1); bit++) { + flip(ch, pos, bit); + String what = "bit " + bit + " of byte " + (pos - first) + " of section " + name; + CorruptException e = assertInstanceOf(CorruptException.class, refused(scratch, what), what); + assertEquals(name, e.section(), what); + restore(ch, pos); + } + } + } + } + } + + @Test + void aFlippedSectionBitIsFoundOnlyWhenTheSectionIsChecksummed() throws IOException { + // the contract of read(file, false): header, footer and locator are checked, the section bytes are not. A host that wants every byte + // checked uses SealedGraph.load or TieredIndex.open, which always do. + SegmentInfo.Section vec = info.section(SegmentFormat.S_VEC); + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + flip(ch, (int) (vec.offset() + 5), 3); + } + SegmentReader.read(scratch, false); // loads: the flip is in a float of the vectors + assertEquals("VEC", assertInstanceOf(CorruptException.class, refused(scratch, "bit in VEC")).section()); + assertEquals("VEC", assertInstanceOf(CorruptException.class, assertThrowsIo(() -> SealedGraph.load(scratch), "SealedGraph.load")).section()); + } + + // ---- the zero fill that no CRC covers ---- + + @Test + void damageInTheZeroFillIsNotDetectedAndChangesNothingThatIsServed() throws IOException { + // the doc says so: header bytes 52 to 4095 and the gaps between sections are zero and covered by no CRC. A flip there is not an + // integrity failure because no reader looks at those bytes; the test pins that the loaded graph is the same one. + List gaps = new ArrayList<>(); // [from, to) + gaps.add(new int[] {SegmentFormat.HEADER_LEN, SegmentFormat.HEADER_BLOCK}); + for (int i = 0; i + 1 < info.sections().size(); i++) { + SegmentInfo.Section s = info.sections().get(i), next = info.sections().get(i + 1); + if (s.offset() + s.length() < next.offset()) gaps.add(new int[] {(int) (s.offset() + s.length()), (int) next.offset()}); + } + SegmentInfo.Section lastSec = info.sections().get(info.sections().size() - 1); + gaps.add(new int[] {(int) (lastSec.offset() + lastSec.length()), footerOff()}); + assertTrue(gaps.size() >= 12, "the gaps found: " + gaps.size()); + float[][] qs = vectors(30, DIM, 99); + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int[] g : gaps) { + for (int pos : new int[] {g[0], (g[0] + g[1]) / 2, g[1] - 1}) { + if (pos < g[0] || pos >= g[1]) continue; + for (int bit : new int[] {0, 7}) { + flip(ch, pos, bit); + SealedGraph h = SealedGraph.load(scratch); + assertArrayEquals(orig.ids, h.ids, "zero fill byte " + pos); + assertArrayEquals(orig.vec, h.vec, "zero fill byte " + pos); + for (float[] q : qs) assertArrayEquals(orig.search(q, 10, null), h.search(q, 10, null), "zero fill byte " + pos); + restore(ch, pos); + } + } + } + } + // and the gaps really are zero in a file the writer wrote + for (int[] g : gaps) { + for (int pos = g[0]; pos < g[1]; pos++) assertEquals(0, good[pos], "byte " + pos + " of the fill"); + } + } + + // ---- wrong magic and files that are no segment ---- + + @Test + void aWrongMagicIsRefusedWhicheverByteIsWrong() throws IOException { + for (int pos = 0; pos < SegmentFormat.MAGIC.length; pos++) { + for (int x : new int[] {0x01, 0x80, 0xFF}) { + byte[] b = good.clone(); + b[pos] ^= (byte) x; + Path p = tmp.resolve("magic-" + pos + "-" + x + ".tvs"); + // the magic is checked before any CRC: the reader says it is not a segment file + assertInstanceOf(UnsupportedException.class, refused(p, b, "magic byte " + pos + " ^ " + x)); + } + } + byte[] zeroed = good.clone(); + java.util.Arrays.fill(zeroed, 0, 8, (byte) 0); + assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("zeroed-magic.tvs"), zeroed, "zeroed magic")); + // the magic of a manifest, a tombstone file and the locator are not the magic of a segment + for (String other : new String[] {"TVMAN001", "TVTOMB01", "TVSEGEND", "tvseg001", "TVSEG002"}) { + byte[] b = good.clone(); + System.arraycopy(other.getBytes(java.nio.charset.StandardCharsets.US_ASCII), 0, b, 0, 8); + assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("other-" + other + ".tvs"), b, other)); + } + } + + @Test + void filesThatAreNoSegmentAtAllAreRefused() throws IOException { + Random r = new Random(5); + byte[] noise = new byte[100_000]; + r.nextBytes(noise); + assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("noise.tvs"), noise, "random bytes")); + byte[] text = "this is a text file, not a segment, but it is longer than the header".getBytes(java.nio.charset.StandardCharsets.US_ASCII); + assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("text.tvs"), text, "text")); + // shorter than the magic: too short to be anything + for (int len : new int[] {0, 1, 7}) { + assertInstanceOf(CorruptException.class, refused(tmp.resolve("short" + len + ".tvs"), new byte[len], len + " bytes"), len + " bytes"); + } + // zeros: no magic + assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("zeros.tvs"), new byte[HEADER_AND_LOCATOR + 10], "zeros")); + // the right magic followed by nothing, and by random bytes + byte[] magicOnly = java.util.Arrays.copyOf(SegmentFormat.MAGIC, 8); + assertInstanceOf(CorruptException.class, refused(tmp.resolve("magic-only.tvs"), magicOnly, "magic only")); + byte[] magicNoise = noise.clone(); + System.arraycopy(SegmentFormat.MAGIC, 0, magicNoise, 0, 8); + refused(tmp.resolve("magic-noise.tvs"), magicNoise, "magic then random bytes"); // some named exception, never a graph + } + + private static final int HEADER_AND_LOCATOR = SegmentFormat.HEADER_BLOCK + SegmentFormat.LOCATOR_LEN; + + private interface IoCall { + void run() throws IOException; + } + + private static IOException assertThrowsIo(IoCall c, String what) { + try { + c.run(); + } catch (IOException e) { + return e; + } + throw new AssertionError(what + ": no exception"); + } +} diff --git a/src/test/java/com/humemai/vec/SegmentForge.java b/src/test/java/com/humemai/vec/SegmentForge.java new file mode 100644 index 0000000..17f4606 --- /dev/null +++ b/src/test/java/com/humemai/vec/SegmentForge.java @@ -0,0 +1,189 @@ +package com.humemai.vec; + +import static com.humemai.vec.SegmentFormat.F_COUNT; +import static com.humemai.vec.SegmentFormat.F_ENTRIES_OFF; +import static com.humemai.vec.SegmentFormat.HEADER_BLOCK; +import static com.humemai.vec.SegmentFormat.H_LEN; +import static com.humemai.vec.SegmentFormat.LOCATOR_LEN; +import static com.humemai.vec.SegmentFormat.LOCATOR_MAGIC; +import static com.humemai.vec.SegmentFormat.crc32c; +import static org.junit.jupiter.api.Assertions.fail; + +import com.humemai.vec.SegmentFormat.CorruptException; +import com.humemai.vec.SegmentFormat.UnsupportedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.opentest4j.AssertionFailedError; + +/** + * Takes a good segment file apart and puts it together again with chosen fields changed. {@link #build} recomputes every CRC and length + * that sits above the change (the header CRC, the section CRCs of changed sections, the footer CRC and the locator), so the file that comes + * out is damaged in exactly one place and a reader can only find that damage through the check that is meant to find it. A test that wants a + * CRC to stay wrong changes the bytes of the built file instead. + */ +final class SegmentForge { + /** one entry of the section table, as the footer holds it */ + static final class Entry { + int id; + int flags; + long width; + long count; + long offset; + long length; + int crc; + } + + /** the 4096-byte header block */ + final byte[] head; + /** the bytes from the end of the header block to the start of the footer: the sections and the zero fill between them */ + byte[] body; + /** the footer metadata that precedes the section entries */ + byte[] meta; + final List entries = new ArrayList<>(); + /** recompute the header CRC (at header length minus 4) in {@link #build} */ + boolean fixHeaderCrc = true; + /** write the entry count and the first entry offset into the footer metadata in {@link #build}; false keeps whatever {@link #meta} holds */ + boolean syncMeta = true; + /** values for the locator fields that replace the true ones, null keeps the true value */ + Long locFooterOff, locFooterLen, locFileLen; + + private SegmentForge(byte[] file) { + ByteBuffer b = le(file); + final int size = file.length; + final int footerOff = (int) b.getLong(size - LOCATOR_LEN + 8); + final int entriesOff = b.getInt(footerOff + F_ENTRIES_OFF); + final int count = b.getInt(footerOff + F_COUNT); + head = Arrays.copyOfRange(file, 0, HEADER_BLOCK); + body = Arrays.copyOfRange(file, HEADER_BLOCK, footerOff); + meta = Arrays.copyOfRange(file, footerOff, footerOff + entriesOff); + for (int i = 0; i < count; i++) { + int at = footerOff + entriesOff + i * SegmentFormat.ENTRY_LEN; + Entry e = new Entry(); + e.id = b.getShort(at) & 0xFFFF; + e.flags = b.getShort(at + 2) & 0xFFFF; + e.width = Integer.toUnsignedLong(b.getInt(at + 4)); + e.count = b.getLong(at + 8); + e.offset = b.getLong(at + 16); + e.length = b.getLong(at + 24); + e.crc = b.getInt(at + 32); + entries.add(e); + } + } + + /** takes apart a file that {@link SegmentWriter} wrote */ + static SegmentForge of(byte[] file) { + return new SegmentForge(file); + } + + static ByteBuffer le(byte[] a) { + return ByteBuffer.wrap(a).order(ByteOrder.LITTLE_ENDIAN); + } + + /** the header block for in-place changes of header fields */ + ByteBuffer headBuf() { + return le(head); + } + + /** the footer metadata for in-place changes */ + ByteBuffer metaBuf() { + return le(meta); + } + + Entry entry(int id) { + for (Entry e : entries) if (e.id == id) return e; + throw new IllegalArgumentException("no section " + id); + } + + byte[] sectionBytes(int id) { + Entry e = entry(id); + return Arrays.copyOfRange(body, (int) e.offset - HEADER_BLOCK, (int) (e.offset + e.length) - HEADER_BLOCK); + } + + /** replaces the bytes of a section (same length) and puts the CRC of the new bytes into its entry */ + void setSectionBytes(int id, byte[] data) { + Entry e = entry(id); + if (data.length != e.length) throw new IllegalArgumentException("length " + data.length + " != " + e.length); + System.arraycopy(data, 0, body, (int) e.offset - HEADER_BLOCK, data.length); + e.crc = crc32c(data, 0, data.length); + } + + /** appends a section after the last one, at the next aligned offset, with a correct CRC, and lists it in the table */ + Entry appendSection(int id, int flags, int width, byte[] data) { + long end = HEADER_BLOCK + body.length; + long off = SegmentFormat.alignUp(end); + byte[] nb = Arrays.copyOf(body, (int) (off - HEADER_BLOCK) + data.length); + System.arraycopy(data, 0, nb, (int) (off - HEADER_BLOCK), data.length); + body = nb; + Entry e = new Entry(); + e.id = id; + e.flags = flags; + e.width = width; + e.count = data.length / width; + e.offset = off; + e.length = data.length; + e.crc = crc32c(data, 0, data.length); + entries.add(e); + return e; + } + + /** the file, with the CRCs and the lengths above the changes recomputed */ + byte[] build() { + if (fixHeaderCrc) { + int hl = headBuf().getInt(H_LEN); + if (hl >= 8 && hl <= HEADER_BLOCK) headBuf().putInt(hl - 4, crc32c(head, 0, hl - 4)); + } + if (syncMeta) { + metaBuf().putInt(F_COUNT, entries.size()); + metaBuf().putInt(F_ENTRIES_OFF, meta.length); + } + ByteBuffer f = ByteBuffer.allocate(meta.length + SegmentFormat.ENTRY_LEN * entries.size()).order(ByteOrder.LITTLE_ENDIAN); + f.put(meta); + for (Entry e : entries) { + f.putShort((short) e.id).putShort((short) e.flags).putInt((int) e.width).putLong(e.count).putLong(e.offset).putLong(e.length); + f.putInt(e.crc).putInt(0); + } + byte[] footer = f.array(); + final long footerOff = HEADER_BLOCK + body.length; + final long fileLen = footerOff + footer.length + LOCATOR_LEN; + ByteBuffer out = ByteBuffer.allocate((int) fileLen).order(ByteOrder.LITTLE_ENDIAN); + out.put(head).put(body).put(footer); + out.put(LOCATOR_MAGIC).putLong(locFooterOff != null ? locFooterOff : footerOff).putInt((int) (locFooterLen != null ? locFooterLen : footer.length)); + out.putInt(crc32c(footer, 0, footer.length)).putLong(locFileLen != null ? locFileLen : fileLen); + return out.array(); + } + + /** + * Loads the file the way a host does ({@link SealedGraph#load}, every section checksummed) and returns the exception that refused it. + * Fails the test when a graph comes back, and when anything but a {@link CorruptException} or an {@link UnsupportedException} comes out: + * a runtime exception, an error or another IOException is an uncontrolled failure. + */ + static IOException refused(Path file, String what) { + return refused(file, true, what); + } + + static IOException refused(Path file, boolean verifyAll, String what) { + try { + SealedGraph g = SegmentReader.read(file, verifyAll); + fail(what + ": a graph with " + g.n + " rows came back from a damaged file"); + return null; + } catch (CorruptException | UnsupportedException e) { + return e; + } catch (AssertionFailedError e) { + throw e; + } catch (Throwable t) { + throw new AssertionFailedError(what + ": an uncontrolled failure instead of a named exception: " + t, t); + } + } + + /** the same as {@link #refused} for a file that is passed as bytes (written to {@code file} first) */ + static IOException refused(Path file, byte[] bytes, String what) throws IOException { + java.nio.file.Files.write(file, bytes); + return refused(file, what); + } +} From a1ea798cbd7ceb431cdc84fd6e905c1f57d6ea5d Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 10:22:03 +0900 Subject: [PATCH 2/4] Tests: forged segment files, damage with valid CRCs (#17) Version rule (larger major refused, larger minor opens, newer header and footer metadata, unknown optional section ignored, unknown required section refused), the byte-order probe (every bit, a big-endian header), header fields a reader cannot use, and the section table, footer and locator rules, all with the CRCs above the damage recomputed so that only the rule meant to find the damage can fire. Co-Authored-By: Claude Sonnet 5.5 --- .../humemai/vec/SegmentForgedFileTest.java | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 src/test/java/com/humemai/vec/SegmentForgedFileTest.java diff --git a/src/test/java/com/humemai/vec/SegmentForgedFileTest.java b/src/test/java/com/humemai/vec/SegmentForgedFileTest.java new file mode 100644 index 0000000..b68ad4e --- /dev/null +++ b/src/test/java/com/humemai/vec/SegmentForgedFileTest.java @@ -0,0 +1,397 @@ +package com.humemai.vec; + +import static com.humemai.vec.SegmentForge.refused; +import static com.humemai.vec.SegmentTestSupport.graph; +import static com.humemai.vec.SegmentTestSupport.params; +import static com.humemai.vec.SegmentTestSupport.vectors; +import static com.humemai.vec.SegmentTestSupport.versOf; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.humemai.vec.SegmentFormat.CorruptException; +import com.humemai.vec.SegmentFormat.UnsupportedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Segment files that are damaged in exactly one place with every CRC above the damage recomputed (see {@link SegmentForge}), so that a reader + * can find the damage only through the rule that is meant to find it: the version rule, the byte-order probe, the section table checks, the + * locator checks and the checks of the stored id order and the layer 0 offsets. {@link SegmentCorruptionTest} covers the bytes that a CRC + * covers; this class covers the rules that hold when the CRCs are fine. + */ +class SegmentForgedFileTest { + private static final int N = 300, DIM = 4; + private static final float[][] QS = vectors(40, DIM, 123); + + @TempDir + Path tmp; + + private SealedGraph orig; + private byte[] good; + private Path f; + + @BeforeEach + void write() throws IOException { + orig = graph(vectors(N, DIM, 17), Metric.L2, params(true)); + Path g = tmp.resolve("good.tvs"); + SegmentWriter.write(orig, versOf(N), g); + good = Files.readAllBytes(g); + f = tmp.resolve("forged.tvs"); + } + + private SegmentForge forge() { + return SegmentForge.of(good); + } + + private void assertServedLikeTheOriginal(Path p) throws IOException { + SealedGraph h = SealedGraph.load(p); + assertArrayEquals(orig.ids, h.ids); + assertArrayEquals(orig.vec, h.vec); + assertArrayEquals(orig.level, h.level); + assertArrayEquals(orig.off0, h.off0); + assertArrayEquals(orig.edges0, h.edges0); + assertArrayEquals(orig.codes, h.codes); + for (float[] q : QS) assertArrayEquals(orig.search(q, 10, null), h.search(q, 10, null)); + } + + // ---- the version rule ---- + + @ParameterizedTest(name = "major {0}") + @ValueSource(ints = {0, 2, 3, 255, 256, 4097, 65535}) + void aMajorVersionThatIsNotOneIsRefusedAsUnsupportedWithValidChecksums(int major) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putShort(SegmentFormat.H_MAJOR, (short) major); + Files.write(f, forge.build()); + assertInstanceOf(UnsupportedException.class, refused(f, "major " + major)); + assertInstanceOf(UnsupportedException.class, refused(f, false, "major " + major)); + try { + SegmentReader.inspect(f); + throw new AssertionError("inspect returned for major " + major); + } catch (UnsupportedException expected) { + assertTrue(expected.getMessage().contains("major version " + major), expected.getMessage()); + } + } + + @ParameterizedTest(name = "minor {0}") + @ValueSource(ints = {1, 2, 255, 65535}) + void aLargerMinorVersionOpensAndServesTheSameGraph(int minor) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putShort(SegmentFormat.H_MINOR, (short) minor); + Files.write(f, forge.build()); + assertEquals(minor, SegmentReader.inspect(f).minor()); + assertServedLikeTheOriginal(f); + } + + @Test + void aLargerMinorMayAppendHeaderFieldsAndFooterMetadata() throws IOException { + // the header length says where the header CRC is, the offset of the first section entry says where the table starts + SegmentForge forge = forge(); + ByteBuffer h = forge.headBuf(); + h.putShort(SegmentFormat.H_MINOR, (short) 3); + h.putInt(SegmentFormat.H_LEN, 60); + h.putInt(52, 0x5EED5EED); // a field of the newer minor; the old CRC slot (48 to 51) is a field now, the CRC moves to 56 + forge.meta = java.util.Arrays.copyOf(forge.meta, 96); // 32 bytes of newer footer metadata + for (int i = SegmentFormat.F_META_LEN; i < 96; i++) forge.meta[i] = (byte) (i * 7); + Files.write(f, forge.build()); + SegmentInfo si = SegmentReader.inspect(f); + assertEquals(3, si.minor()); + assertEquals(12, si.sections().size()); + assertServedLikeTheOriginal(f); + // and the header CRC that moved is still checked + byte[] b = Files.readAllBytes(f); + b[53] ^= 0x01; + assertEquals("HEADER", assertInstanceOf(CorruptException.class, refused(f, b, "newer header field")).section()); + } + + @ParameterizedTest(name = "unknown optional section id {0}") + @ValueSource(ints = {13, 900, 4096, 65535}) + void aLargerMinorWithAnUnknownOptionalSectionIsOpenedAndTheSectionIsIgnored(int id) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putShort(SegmentFormat.H_MINOR, (short) 1); + SegmentForge.Entry e = forge.appendSection(id, 0, 1, new byte[5000]); + Files.write(f, forge.build()); + SegmentInfo si = SegmentReader.inspect(f); + assertNotNull(si.section(id)); + assertEquals(false, si.section(id).required()); + assertEquals(13, si.sections().size()); + assertServedLikeTheOriginal(f); + // the reader does not look at a section it does not know, so damage there is not an integrity failure of what it serves + byte[] b = Files.readAllBytes(f); + b[(int) (e.offset + 2500)] ^= 0x20; + Files.write(f, b); + assertServedLikeTheOriginal(f); + } + + @ParameterizedTest(name = "unknown required section id {0}, minor {1}") + @MethodSource("requiredUnknown") + void anUnknownRequiredSectionIsRefusedAsUnsupportedEvenWithValidChecksums(int id, int minor) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putShort(SegmentFormat.H_MINOR, (short) minor); + forge.appendSection(id, SegmentFormat.FLAG_REQUIRED, 1, new byte[100]); + Files.write(f, forge.build()); + for (boolean verify : new boolean[] {true, false}) { + UnsupportedException e = assertInstanceOf(UnsupportedException.class, refused(f, verify, "required section " + id)); + assertTrue(e.getMessage().contains("required section"), e.getMessage()); + } + try { + SegmentReader.inspect(f); + throw new AssertionError("inspect returned"); + } catch (UnsupportedException expected) { + // refused in constant time, before any section is read + } + } + + static Stream requiredUnknown() { + return Stream.of(Arguments.of(13, 1), Arguments.of(901, 1), Arguments.of(901, 0), Arguments.of(901, 9), Arguments.of(65535, 1)); + } + + // ---- the byte-order probe ---- + + @Test + void aProbeThatReadsBackSwappedIsRefusedAsUnsupported() throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putInt(SegmentFormat.H_PROBE, 0x04030201); + Files.write(f, forge.build()); + UnsupportedException e = assertInstanceOf(UnsupportedException.class, refused(f, "swapped probe")); + assertTrue(e.getMessage().contains("byte-order"), e.getMessage()); + assertInstanceOf(UnsupportedException.class, refused(f, false, "swapped probe")); + } + + @ParameterizedTest(name = "probe bit {0}") + @MethodSource("thirtyTwoBits") + void everySingleBitOfTheProbeIsRefusedAsUnsupportedWithValidChecksums(int bit) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putInt(SegmentFormat.H_PROBE, SegmentFormat.ENDIAN_PROBE ^ (1 << bit)); + Files.write(f, forge.build()); + assertInstanceOf(UnsupportedException.class, refused(f, "probe bit " + bit)); + } + + static IntStream thirtyTwoBits() { + return IntStream.range(0, 32); + } + + @Test + void aHeaderWrittenInBigEndianIsRefusedAsUnsupported() throws IOException { + // every multi-byte field of the header written the other way round, and a CRC that matches the bytes + SegmentForge forge = forge(); + ByteBuffer le = forge.headBuf(); + byte[] be = new byte[SegmentFormat.HEADER_BLOCK]; + ByteBuffer b = ByteBuffer.wrap(be).order(ByteOrder.BIG_ENDIAN); + System.arraycopy(forge.head, 0, be, 0, 8); + b.putShort(8, le.getShort(8)).putShort(10, le.getShort(10)).putInt(12, le.getInt(12)).putInt(16, le.getInt(16)); + b.putInt(20, le.getInt(20)).putInt(24, le.getInt(24)).putInt(28, le.getInt(28)).putLong(32, le.getLong(32)); + System.arraycopy(forge.head, 40, be, 40, 4); + b.putInt(44, le.getInt(44)); + b.putInt(48, SegmentFormat.crc32c(be, 0, 48)); + System.arraycopy(be, 0, forge.head, 0, SegmentFormat.HEADER_LEN); + forge.fixHeaderCrc = false; + Files.write(f, forge.build()); + assertEquals(0x0100, ByteBuffer.wrap(forge.head).order(ByteOrder.LITTLE_ENDIAN).getShort(8), "the major version reads as 256 in little-endian"); + // the first thing the reader trusts is the major version, and a big-endian writer's 1 reads as 256 + UnsupportedException e = assertInstanceOf(UnsupportedException.class, refused(f, "big-endian header")); + assertTrue(e.getMessage().contains("major version 256"), e.getMessage()); + } + + // ---- the header fields, with a valid header CRC ---- + + static Stream headerFields() { + return Stream.of( + header("row stride differs from the dimension", b -> b.putInt(SegmentFormat.H_STRIDE, DIM + 4), UnsupportedException.class, null), + header("metric code 3", b -> b.put(SegmentFormat.H_METRIC, (byte) 3), UnsupportedException.class, null), + header("metric code 255", b -> b.put(SegmentFormat.H_METRIC, (byte) 255), UnsupportedException.class, null), + header("id width 4", b -> b.put(SegmentFormat.H_IDW, (byte) 4), UnsupportedException.class, null), + header("layer 0 layout 1", b -> b.put(SegmentFormat.H_LAYOUT, (byte) 1), UnsupportedException.class, null), + header("dimension zero", b -> b.putInt(SegmentFormat.H_DIM, 0), CorruptException.class, "HEADER"), + header("dimension above 2^20", b -> b.putInt(SegmentFormat.H_DIM, (1 << 20) + 1), CorruptException.class, "HEADER"), + header("dimension one above", b -> { + b.putInt(SegmentFormat.H_DIM, DIM + 1); + b.putInt(SegmentFormat.H_STRIDE, DIM + 1); + }, CorruptException.class, "VEC"), + header("row count one above", b -> b.putLong(SegmentFormat.H_N, N + 1), CorruptException.class, "IDS"), + header("row count zero", b -> b.putLong(SegmentFormat.H_N, 0), CorruptException.class, "IDS"), + header("row count negative", b -> b.putLong(SegmentFormat.H_N, -1), IOException.class, null), + header("row count 100 million", b -> b.putLong(SegmentFormat.H_N, 100_000_000L), CorruptException.class, "IDS"), + header("row count 2^31 - 9", b -> b.putLong(SegmentFormat.H_N, Integer.MAX_VALUE - 9L), IOException.class, null), + header("row count 2^62", b -> b.putLong(SegmentFormat.H_N, 1L << 62), IOException.class, null), + header("alignment 3000, not a power of two", b -> b.putInt(SegmentFormat.H_ALIGN, 3000), CorruptException.class, "HEADER"), + header("alignment 4", b -> b.putInt(SegmentFormat.H_ALIGN, 4), CorruptException.class, "HEADER"), + header("alignment 2^31", b -> b.putInt(SegmentFormat.H_ALIGN, 1 << 31), CorruptException.class, "HEADER"), + header("alignment 8192 while the sections start at multiples of 4096", b -> b.putInt(SegmentFormat.H_ALIGN, 8192), CorruptException.class, "IDS"), + header("header length 51", b -> b.putInt(SegmentFormat.H_LEN, 51), CorruptException.class, "HEADER"), + header("header length 4097", b -> b.putInt(SegmentFormat.H_LEN, 4097), CorruptException.class, "HEADER"), + header("header length 2^32 - 1", b -> b.putInt(SegmentFormat.H_LEN, -1), CorruptException.class, "HEADER")); + } + + private static Arguments header(String name, java.util.function.Consumer change, Class type, String section) { + return Arguments.of(name, change, type, section); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("headerFields") + void aHeaderFieldThatThisReaderCannotUseIsRefusedWithValidChecksums(String name, java.util.function.Consumer change, + Class type, String section) throws IOException { + SegmentForge forge = forge(); + change.accept(forge.headBuf()); + Files.write(f, forge.build()); + IOException e = refused(f, name); + assertInstanceOf(type, e, name); + if (section != null) assertEquals(section, ((CorruptException) e).section(), name); + } + + // ---- the section table, the footer metadata and the locator, with valid CRCs ---- + + static Stream structure() { + final int footerMeta = SegmentFormat.F_META_LEN; + return Stream.of( + table("section offset not aligned", fg -> fg.entry(SegmentFormat.S_IDS).offset += 8, "IDS"), + table("section offset inside the header block", fg -> fg.entry(SegmentFormat.S_IDS).offset = 0, "IDS"), + table("two sections swapped in the file order", fg -> { + long a = fg.entries.get(1).offset; + fg.entries.get(1).offset = fg.entries.get(2).offset; + fg.entries.get(2).offset = a; + }, "BYID"), + table("a section starts where the one before it starts", fg -> fg.entry(SegmentFormat.S_BYID).offset = fg.entry(SegmentFormat.S_VERS).offset, "BYID"), + table("a section runs past the footer", fg -> fg.entry(SegmentFormat.S_SQ8_PARAMS).offset += 1 << 20, "SQ8_PARAMS"), + table("length is not width times count", fg -> fg.entry(SegmentFormat.S_VEC).length += 4, "VEC"), + table("element width zero", fg -> fg.entry(SegmentFormat.S_IDS).width = 0, "IDS"), + table("element width above 8", fg -> { + SegmentForge.Entry e = fg.entry(SegmentFormat.S_IDS); + e.width = 16; + e.count = e.length / 16; + }, "IDS"), + table("element width 4 for an i64 section", fg -> { + SegmentForge.Entry e = fg.entry(SegmentFormat.S_IDS); + e.width = 4; + e.count = e.length / 4; + }, "IDS"), + table("element count one short with a matching length", fg -> { + SegmentForge.Entry e = fg.entry(SegmentFormat.S_IDS); + e.count = N - 1; + e.length = 8L * (N - 1); + }, "IDS"), + table("negative element count", fg -> fg.entry(SegmentFormat.S_IDS).count = -1, "IDS"), + table("negative section length", fg -> fg.entry(SegmentFormat.S_IDS).length = -8, "IDS"), + table("SQ8 codes without their parameters", fg -> fg.entries.remove(fg.entry(SegmentFormat.S_SQ8_PARAMS)), "SQ8_PARAMS"), + table("SQ8 parameters without their codes", fg -> fg.entries.remove(fg.entry(SegmentFormat.S_SQ8_CODES)), "SQ8_PARAMS"), + table("a section listed twice", fg -> fg.appendSection(SegmentFormat.S_IDS, SegmentFormat.FLAG_REQUIRED, 8, new byte[8 * N]), "IDS"), + footer("more entries than the footer holds", fg -> fg.metaBuf().putInt(SegmentFormat.F_COUNT, fg.entries.size() + 1), "FOOTER"), + footer("more than 4096 entries", fg -> fg.metaBuf().putInt(SegmentFormat.F_COUNT, 5000), "FOOTER"), + footer("entry count 2^32 - 1", fg -> fg.metaBuf().putInt(SegmentFormat.F_COUNT, -1), "FOOTER"), + footer("first entry inside the metadata", fg -> fg.metaBuf().putInt(SegmentFormat.F_ENTRIES_OFF, footerMeta - 4), "FOOTER"), + footer("first entry beyond the footer", fg -> fg.metaBuf().putInt(SegmentFormat.F_ENTRIES_OFF, 100_000), "FOOTER"), + footer("entry row equal to the row count", fg -> fg.metaBuf().putInt(SegmentFormat.F_ENTRY_ROW, N), "FOOTER"), + footer("negative entry row", fg -> fg.metaBuf().putInt(SegmentFormat.F_ENTRY_ROW, -1), "FOOTER"), + footer("top level 128", fg -> fg.metaBuf().putInt(SegmentFormat.F_MAX_LEVEL, 128), "FOOTER"), + footer("negative top level", fg -> fg.metaBuf().putInt(SegmentFormat.F_MAX_LEVEL, -1), "FOOTER"), + loc("footer offset one too high", fg -> fg.locFooterOff = 4096L + fg.body.length + 1, "LOCATOR"), + loc("footer offset inside the header block", fg -> fg.locFooterOff = 100L, "LOCATOR"), + loc("footer offset zero", fg -> fg.locFooterOff = 0L, "LOCATOR"), + loc("footer length one too long", fg -> fg.locFooterLen = fg.meta.length + 40L * fg.entries.size() + 1, "LOCATOR"), + loc("footer length zero", fg -> fg.locFooterLen = 0L, "LOCATOR"), + loc("footer length above one MiB", fg -> fg.locFooterLen = (1L << 20) + 1, "LOCATOR"), + loc("file length one too long", fg -> fg.locFileLen = 4096L + fg.body.length + fg.meta.length + 40L * fg.entries.size() + 33, "LOCATOR"), + loc("file length zero", fg -> fg.locFileLen = 0L, "LOCATOR"), + loc("file length 2^63 - 1", fg -> fg.locFileLen = Long.MAX_VALUE, "LOCATOR"), + data("a row listed twice in the id order", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_BYID); + System.arraycopy(a, 0, a, 4, 4); // row at position 1 := row at position 0 + fg.setSectionBytes(SegmentFormat.S_BYID, a); + }, "BYID"), + data("a row number outside the rows in the id order", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_BYID); + SegmentForge.le(a).putInt(0, N); + fg.setSectionBytes(SegmentFormat.S_BYID, a); + }, "BYID"), + data("a negative row number in the id order", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_BYID); + SegmentForge.le(a).putInt(4 * (N - 1), -1); + fg.setSectionBytes(SegmentFormat.S_BYID, a); + }, "BYID"), + data("layer 0 offsets that end above the edge count", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_L0_OFF); + ByteBuffer b = SegmentForge.le(a); + b.putInt(4 * N, b.getInt(4 * N) + 1); + fg.setSectionBytes(SegmentFormat.S_L0_OFF, a); + }, "L0_OFF"), + data("layer 0 offsets that do not start at zero", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_L0_OFF); + SegmentForge.le(a).putInt(0, 1); + fg.setSectionBytes(SegmentFormat.S_L0_OFF, a); + }, "L0_OFF")); + } + + private static Arguments table(String name, java.util.function.Consumer change, String section) { + return Arguments.of("table: " + name, change, section); + } + + private static Arguments footer(String name, java.util.function.Consumer change, String section) { + return Arguments.of("footer: " + name, (java.util.function.Consumer) fg -> { + fg.syncMeta = false; + change.accept(fg); + }, section); + } + + private static Arguments loc(String name, java.util.function.Consumer change, String section) { + return Arguments.of("locator: " + name, change, section); + } + + private static Arguments data(String name, java.util.function.Consumer change, String section) { + return Arguments.of("data: " + name, change, section); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("structure") + void aStructureThatBreaksTheDocumentedRulesIsRefusedWithValidChecksums(String name, java.util.function.Consumer change, + String section) throws IOException { + SegmentForge forge = forge(); + change.accept(forge); + Files.write(f, forge.build()); + CorruptException e = assertInstanceOf(CorruptException.class, refused(f, name), name); + assertEquals(section, e.section(), name); + // the checks of the table, the footer and the locator also hold without the section CRCs; the data checks need the sections loaded + if (!name.startsWith("data:")) assertEquals(section, assertInstanceOf(CorruptException.class, refused(f, false, name), name).section(), name); + } + + @ParameterizedTest(name = "required section id {0} missing") + @ValueSource(ints = {1, 3, 4, 5, 6, 7, 8, 9, 10}) + void aMissingRequiredSectionIsRefusedWithValidChecksums(int id) throws IOException { + SegmentForge forge = forge(); + forge.entries.remove(forge.entry(id)); + Files.write(f, forge.build()); + for (boolean verify : new boolean[] {true, false}) { + CorruptException e = assertInstanceOf(CorruptException.class, refused(f, verify, "section " + id + " missing")); + assertEquals(SegmentFormat.sectionName(id), e.section()); + assertTrue(e.getMessage().contains("required section is missing"), e.getMessage()); + } + } + + @Test + void theOptionalSectionsMayBeAbsentTogether() throws IOException { + // SQ8 codes and parameters are optional as a pair; the version section is optional too (a standalone graph has none) + SegmentForge forge = forge(); + forge.entries.remove(forge.entry(SegmentFormat.S_SQ8_CODES)); + forge.entries.remove(forge.entry(SegmentFormat.S_SQ8_PARAMS)); + forge.entries.remove(forge.entry(SegmentFormat.S_VERS)); + Files.write(f, forge.build()); + SegmentReader.Loaded l = SegmentReader.readFull(f, true); + assertEquals(null, l.vers()); + assertEquals(null, l.graph().codes); + assertArrayEquals(orig.ids, l.graph().ids); + } +} From 4c875887ffeb59b032c8b43d7281fc96ad964715 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 10:27:02 +0900 Subject: [PATCH 3/4] Tests: do not pin which named exception a damaged segment magic gets (#17) The claim is that damage is refused with a named exception. For the magic the reader currently throws UnsupportedException while the manifest throws CorruptException for the same kind of damage; SegmentFormatTest already pins the segment case, so these sweeps accept either named exception there. Co-Authored-By: Claude Sonnet 5.5 --- .../humemai/vec/SegmentCorruptionTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/java/com/humemai/vec/SegmentCorruptionTest.java b/src/test/java/com/humemai/vec/SegmentCorruptionTest.java index 733c414..d1eb59c 100644 --- a/src/test/java/com/humemai/vec/SegmentCorruptionTest.java +++ b/src/test/java/com/humemai/vec/SegmentCorruptionTest.java @@ -30,8 +30,8 @@ /** * Damage to a segment file, byte by byte. A file that is cut short, or has any bit of its header, footer, locator or sections flipped, is - * refused with a {@link CorruptException} (or an {@link UnsupportedException} where the damaged bytes are the magic or the major version), - * never loaded, and never fails in any other way. The sweeps are exhaustive over a small file: every cut length, and every byte that a + * refused with a {@link CorruptException} (an {@link UnsupportedException} where the damaged bytes are the major version; for the magic either + * of the two named exceptions), never loaded, and never fails in any other way. The sweeps are exhaustive over a small file: every cut length, and every byte that a * CRC or a length check covers. The only bytes whose damage is not detected are the zero fill that no CRC covers (the rest of the header * block and the gaps between sections), and a test below shows that damage there changes nothing a reader serves. * @@ -143,8 +143,8 @@ void everyBitOfTheHeaderIsGuarded() throws IOException { String what = "bit " + bit + " of header byte " + pos; for (boolean verify : new boolean[] {true, false}) { IOException e = refused(scratch, verify, what); - if (pos < 10) assertInstanceOf(UnsupportedException.class, e, what); - else assertEquals("HEADER", assertInstanceOf(CorruptException.class, e, what).section(), what); + if (pos >= 8 && pos < 10) assertInstanceOf(UnsupportedException.class, e, what); + else if (pos >= 10) assertEquals("HEADER", assertInstanceOf(CorruptException.class, e, what).section(), what); } restore(ch, pos); } @@ -272,18 +272,18 @@ void aWrongMagicIsRefusedWhicheverByteIsWrong() throws IOException { byte[] b = good.clone(); b[pos] ^= (byte) x; Path p = tmp.resolve("magic-" + pos + "-" + x + ".tvs"); - // the magic is checked before any CRC: the reader says it is not a segment file - assertInstanceOf(UnsupportedException.class, refused(p, b, "magic byte " + pos + " ^ " + x)); + // the magic is checked before any CRC; which of the two named exceptions says "not a segment file" is not part of the claim + refused(p, b, "magic byte " + pos + " ^ " + x); } } byte[] zeroed = good.clone(); java.util.Arrays.fill(zeroed, 0, 8, (byte) 0); - assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("zeroed-magic.tvs"), zeroed, "zeroed magic")); + refused(tmp.resolve("zeroed-magic.tvs"), zeroed, "zeroed magic"); // the magic of a manifest, a tombstone file and the locator are not the magic of a segment for (String other : new String[] {"TVMAN001", "TVTOMB01", "TVSEGEND", "tvseg001", "TVSEG002"}) { byte[] b = good.clone(); System.arraycopy(other.getBytes(java.nio.charset.StandardCharsets.US_ASCII), 0, b, 0, 8); - assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("other-" + other + ".tvs"), b, other)); + refused(tmp.resolve("other-" + other + ".tvs"), b, other); } } @@ -292,15 +292,15 @@ void filesThatAreNoSegmentAtAllAreRefused() throws IOException { Random r = new Random(5); byte[] noise = new byte[100_000]; r.nextBytes(noise); - assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("noise.tvs"), noise, "random bytes")); + refused(tmp.resolve("noise.tvs"), noise, "random bytes"); byte[] text = "this is a text file, not a segment, but it is longer than the header".getBytes(java.nio.charset.StandardCharsets.US_ASCII); - assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("text.tvs"), text, "text")); + refused(tmp.resolve("text.tvs"), text, "text"); // shorter than the magic: too short to be anything for (int len : new int[] {0, 1, 7}) { assertInstanceOf(CorruptException.class, refused(tmp.resolve("short" + len + ".tvs"), new byte[len], len + " bytes"), len + " bytes"); } // zeros: no magic - assertInstanceOf(UnsupportedException.class, refused(tmp.resolve("zeros.tvs"), new byte[HEADER_AND_LOCATOR + 10], "zeros")); + refused(tmp.resolve("zeros.tvs"), new byte[HEADER_AND_LOCATOR + 10], "zeros"); // the right magic followed by nothing, and by random bytes byte[] magicOnly = java.util.Arrays.copyOf(SegmentFormat.MAGIC, 8); assertInstanceOf(CorruptException.class, refused(tmp.resolve("magic-only.tvs"), magicOnly, "magic only")); From b52f08a0e98106d0e076edcdfccce75771562967 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 10:27:02 +0900 Subject: [PATCH 4/4] Tests: a damaged index directory is refused by open (#17) Segment damage (cuts and bit flips in every section, header, footer and locator, of either segment), every bit and every cut of the manifest and the tombstone file, tombstone and segment versions (larger major refused and not skipped, larger minor opens and serves the writer's results, unknown required section refused), the fallback to the records, a failed open leaves the directory untouched, and damage to the newest of two generations. Co-Authored-By: Claude Sonnet 5.5 --- .../com/humemai/vec/TierCorruptionTest.java | 548 ++++++++++++++++++ 1 file changed, 548 insertions(+) create mode 100644 src/test/java/com/humemai/vec/TierCorruptionTest.java diff --git a/src/test/java/com/humemai/vec/TierCorruptionTest.java b/src/test/java/com/humemai/vec/TierCorruptionTest.java new file mode 100644 index 0000000..5bb8b48 --- /dev/null +++ b/src/test/java/com/humemai/vec/TierCorruptionTest.java @@ -0,0 +1,548 @@ +package com.humemai.vec; + +import static com.humemai.vec.SegmentTestSupport.idOf; +import static com.humemai.vec.SegmentTestSupport.vectors; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.humemai.vec.SegmentFormat.CorruptException; +import com.humemai.vec.SegmentFormat.UnsupportedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.AssertionFailedError; + +/** + * Damage to an index directory, found by {@code TieredIndex.open}. The segment file rules are tested byte by byte in + * {@link SegmentCorruptionTest} and {@link SegmentForgedFileTest}; this class checks what the directory does with them: a damaged segment, manifest + * or tombstone file makes {@code open} throw a named exception (never return an index), a failed open deletes and changes nothing, the overload that + * holds the records falls back to a rebuild and serves them, a newer format is refused and not skipped, and a larger minor version opens and serves + * exactly what the writer's tier served. + */ +class TierCorruptionTest { + static final int DIM = TierReopenTest.DIM; + static final int ROWS = 400; + + /** the directory of the checkpoint of generation 1: two segments, seg-000001 and seg-000002 */ + @TempDir + static Path single; + /** a directory after a crash after the rename of generation 2: generation 1 (segments 1 and 2) and generation 2 (segments 1, 2 and 3) */ + @TempDir + static Path twoGen; + + static TieredIndex tier; + static float[][] data, qs; + static Hit[][] want1; + static List records1; + + @TempDir + Path dir; + + /** SQ8 on, so that a segment file has all twelve sections */ + static TieredIndex.Config cfg() { + return TierReopenTest.cfg().withGraph(GraphIndex.Params.defaults().withEfConstruction(64).withSq8(true)); + } + + @BeforeAll + static void build() throws IOException { + data = vectors(3 * ROWS, DIM, 91); + qs = vectors(40, DIM, 92); + tier = new TieredIndex(DIM, Metric.L2, cfg()); + batch(0); + batch(1); + tier.checkpoint(twoGen); // generation 1 + want1 = results(tier); + copy(twoGen, single); + records1 = new ArrayList<>(); + for (int i = 0; i < 2 * ROWS; i++) records1.add(new TieredIndex.IdVector(idOf(i), data[i])); + batch(2); + assertThrows(IOException.class, () -> tier.checkpoint(twoGen, step -> { + if (step == FaultHook.Step.DIR_SYNCED_AFTER_RENAME) throw new IOException("simulated crash after the rename and before the cleanup"); + })); + assertEquals(2, Manifest.readLatest(twoGen).generation()); + assertEquals(3, Manifest.readLatest(twoGen).entries().size()); + assertEquals(12, SegmentReader.inspect(single.resolve(Manifest.segName(1))).sections().size()); + assertEquals(1, Manifest.readLatest(single).generation()); + assertEquals(2, Manifest.readLatest(single).entries().size()); + assertTrue(Files.exists(twoGen.resolve(Manifest.manifestName(1))) && Files.exists(twoGen.resolve(Manifest.manifestName(2)))); + } + + @AfterAll + static void close() { + tier.close(); + } + + private static void batch(int b) { + for (int i = b * ROWS; i < (b + 1) * ROWS; i++) tier.add(idOf(i), data[i]); + tier.sealNow(); + assertTrue(tier.awaitIdle(60_000)); + } + + private static Hit[][] results(TieredIndex t) { + Hit[][] out = new Hit[qs.length][]; + for (int i = 0; i < qs.length; i++) out[i] = t.search(qs[i], 10, null); + return out; + } + + private static void assertServes(TieredIndex u, Hit[][] want) { + for (int i = 0; i < qs.length; i++) assertArrayEquals(want[i], u.search(qs[i], 10, null), "query " + i); + } + + private static void copy(Path from, Path to) throws IOException { + try (Stream s = Files.list(from)) { + for (Path p : (Iterable) s::iterator) Files.copy(p, to.resolve(p.getFileName().toString())); + } + } + + private Path seg(long id) { + return dir.resolve(Manifest.segName(id)); + } + + /** empties the test's directory so that the next copy of a template can go in */ + private void clear() throws IOException { + try (Stream s = Files.list(dir)) { + for (Path p : (Iterable) s::iterator) Files.delete(p); + } + } + + private static Map snapshot(Path d) throws IOException { + Map m = new TreeMap<>(); + try (Stream s = Files.list(d)) { + for (Path p : (Iterable) s::iterator) m.put(p.getFileName().toString(), Files.readAllBytes(p)); + } + return m; + } + + private static void assertUnchanged(Map before, Path d, String what) throws IOException { + Map after = snapshot(d); + assertEquals(before.keySet(), after.keySet(), what + ": the files in the directory"); + for (Map.Entry e : before.entrySet()) assertArrayEquals(e.getValue(), after.get(e.getKey()), what + ": " + e.getKey()); + } + + /** opens the directory; returns the named exception that refused it, and fails when an index comes back or anything else is thrown */ + private static IOException openRefused(Path d, String what) { + try { + TieredIndex u = TieredIndex.open(d, cfg()); + int size = u.size(); + u.close(); + fail(what + ": open returned an index with " + size + " rows from a damaged directory"); + return null; + } catch (CorruptException | UnsupportedException e) { + return e; + } catch (AssertionFailedError e) { + throw e; + } catch (Throwable t) { + throw new AssertionFailedError(what + ": an uncontrolled failure instead of a named exception: " + t, t); + } + } + + /** the same for {@link Manifest#readLatest} */ + private static IOException latestRefused(Path d, String what) { + try { + Manifest m = Manifest.readLatest(d); + fail(what + ": readLatest returned generation " + (m == null ? "null" : m.generation()) + " from a damaged manifest"); + return null; + } catch (CorruptException | UnsupportedException e) { + return e; + } catch (AssertionFailedError e) { + throw e; + } catch (Throwable t) { + throw new AssertionFailedError(what + ": an uncontrolled failure instead of a named exception: " + t, t); + } + } + + // ---- a damaged segment file ---- + + enum Expect { CORRUPT, UNSUPPORTED, NAMED } + + /** a way to damage the bytes of one segment file; {@code section} is the section the failure must name, when one is known */ + record Damage(String name, BiFunction apply, Expect expect, String section) { + @Override + public String toString() { + return name; + } + } + + private static byte[] cut(byte[] b, long len) { + return Arrays.copyOf(b, (int) len); + } + + private static byte[] flip(byte[] b, long pos) { + byte[] c = b.clone(); + c[(int) pos] ^= (byte) (1 << (pos % 8)); + return c; + } + + static Stream segmentDamage() { + List l = new ArrayList<>(); + for (int id = 1; id <= 12; id++) { + final int sid = id; + final String name = SegmentFormat.sectionName(id); + l.add(new Damage("cut in the middle of " + name, (b, i) -> cut(b, i.section(sid).offset() + i.section(sid).length() / 2), Expect.CORRUPT, null)); + l.add(new Damage("cut at the last byte of " + name, (b, i) -> cut(b, i.section(sid).offset() + i.section(sid).length() - 1), Expect.CORRUPT, null)); + l.add(new Damage("bit flipped in the first byte of " + name, (b, i) -> flip(b, i.section(sid).offset()), Expect.CORRUPT, name)); + l.add(new Damage("bit flipped in the middle of " + name, (b, i) -> flip(b, i.section(sid).offset() + i.section(sid).length() / 2), Expect.CORRUPT, name)); + l.add(new Damage("bit flipped in the last byte of " + name, (b, i) -> flip(b, i.section(sid).offset() + i.section(sid).length() - 1), Expect.CORRUPT, name)); + } + for (int len : new int[] {0, 7, 8, 100, SegmentFormat.HEADER_BLOCK, SegmentFormat.HEADER_BLOCK + SegmentFormat.LOCATOR_LEN - 1}) { + l.add(new Damage("cut to " + len + " bytes", (b, i) -> cut(b, len), Expect.CORRUPT, null)); + } + l.add(new Damage("cut at the start of the footer", (b, i) -> cut(b, footerOff(b)), Expect.CORRUPT, null)); + l.add(new Damage("cut in the middle of the footer", (b, i) -> cut(b, footerOff(b) + 100), Expect.CORRUPT, null)); + l.add(new Damage("cut in the middle of the locator", (b, i) -> cut(b, b.length - 16), Expect.CORRUPT, null)); + l.add(new Damage("cut at the last byte", (b, i) -> cut(b, b.length - 1), Expect.CORRUPT, null)); + l.add(new Damage("a byte appended", (b, i) -> cut(b, b.length + 1), Expect.CORRUPT, null)); + for (int pos : new int[] {10, 11, 12, 16, 24, 32, 40, 44, 48, 51}) { + l.add(new Damage("bit flipped in header byte " + pos, (b, i) -> flip(b, pos), Expect.CORRUPT, null)); + } + l.add(new Damage("bit flipped in the magic", (b, i) -> flip(b, 3), Expect.NAMED, null)); + l.add(new Damage("bit flipped in the major version", (b, i) -> flip(b, 8), Expect.UNSUPPORTED, null)); + l.add(new Damage("magic zeroed", (b, i) -> { + byte[] c = b.clone(); + Arrays.fill(c, 0, 8, (byte) 0); + return c; + }, Expect.NAMED, null)); + l.add(new Damage("all bytes zeroed", (b, i) -> new byte[b.length], Expect.NAMED, null)); + for (int rel : new int[] {0, 5, 40, 64 + 12, 64 + 40 * 11 + 39}) { + l.add(new Damage("bit flipped in footer byte " + rel, (b, i) -> flip(b, footerOff(b) + rel), Expect.CORRUPT, null)); + } + for (int rel : new int[] {0, 7, 8, 16, 20, 24, 31}) { + l.add(new Damage("bit flipped in locator byte " + rel, (b, i) -> flip(b, b.length - SegmentFormat.LOCATOR_LEN + rel), Expect.CORRUPT, null)); + } + return l.stream(); + } + + private static int footerOff(byte[] file) { + return (int) SegmentForge.le(file).getLong(file.length - SegmentFormat.LOCATOR_LEN + 8); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("segmentDamage") + void aDamagedSegmentFileIsRefusedByOpenAndNothingIsDeleted(Damage damage) throws IOException { + for (long segId : new long[] {1, 2}) { + copy(single, dir); + SegmentInfo info = SegmentReader.inspect(seg(segId)); + Files.write(seg(segId), damage.apply().apply(Files.readAllBytes(seg(segId)), info)); + Map before = snapshot(dir); + String what = damage.name() + " in segment " + segId; + IOException e = openRefused(dir, what); + switch (damage.expect()) { + case CORRUPT: assertInstanceOf(CorruptException.class, e, what); break; + case UNSUPPORTED: assertInstanceOf(UnsupportedException.class, e, what); break; + default: break; + } + // a flip inside a section is found when the section is read: the exception names it + if (damage.section() != null) assertEquals(damage.section(), ((CorruptException) e).section(), what); + assertUnchanged(before, dir, what); + clear(); + } + } + + // ---- the fallback to the records ---- + + private void assertRebuiltFromTheRecords(TieredIndex r, String what) { + assertEquals(2 * ROWS, r.size(), what); + int hits = 0; + for (int i = 0; i < 2 * ROWS; i += 20) { + Hit[] h = r.search(data[i], 1, null); + if (h.length == 1 && h[0].id() == idOf(i)) hits++; + } + assertTrue(hits >= 38, what + ": the rebuilt index finds its own records, " + hits + " of 40"); + } + + @Test + void whenOpenCannotTrustTheDirectoryTheRecordsAreServedAndTheDirectoryIsLeftAlone() throws IOException { + List picks = new ArrayList<>(); + segmentDamage().forEach(d -> { + if (d.name().equals("bit flipped in the middle of VEC") || d.name().equals("cut in the middle of L0_EDGES") || d.name().equals("bit flipped in header byte 24") + || d.name().equals("magic zeroed") || d.name().equals("bit flipped in the major version") || d.name().equals("bit flipped in locator byte 20") + || d.name().equals("cut at the last byte of SQ8_PARAMS")) picks.add(d); + }); + assertEquals(7, picks.size()); + for (Damage d : picks) { + copy(single, dir); + SegmentInfo info = SegmentReader.inspect(seg(1)); + Files.write(seg(1), d.apply().apply(Files.readAllBytes(seg(1)), info)); + Map before = snapshot(dir); + TieredIndex r = TieredIndex.open(dir, DIM, Metric.L2, cfg(), () -> records1); + try { + Throwable cause = r.openFallbackCause(); + assertTrue(cause instanceof CorruptException || cause instanceof UnsupportedException, d.name() + ": the cause is " + cause); + assertRebuiltFromTheRecords(r, d.name()); + } finally { + r.close(); + } + assertUnchanged(before, dir, d.name()); + clear(); + } + } + + // ---- the manifest and the tombstone file ---- + + /** flips bit {@code bit} of byte {@code pos} of a file in place; the old byte is returned */ + private static byte flipInPlace(FileChannel ch, long pos, int bit) throws IOException { + ByteBuffer b = ByteBuffer.allocate(1); + ch.read(b, pos); + byte old = b.get(0); + ch.write(ByteBuffer.wrap(new byte[] {(byte) (old ^ (1 << bit))}), pos); + return old; + } + + @Test + void everyBitOfTheManifestIsGuarded() throws IOException { + copy(single, dir); + Path mf = dir.resolve(Manifest.manifestName(1)); + final int len = (int) Files.size(mf); + try (FileChannel ch = FileChannel.open(mf, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + for (int pos = 0; pos < len; pos++) { + for (int bit = 0; bit < 8; bit++) { + byte old = flipInPlace(ch, pos, bit); + String what = "bit " + bit + " of manifest byte " + pos; + IOException e = latestRefused(dir, what); + // bytes 8 and 9 are the major version: a newer format is refused, not skipped. Everything from the minor version on is + // under the CRC of the body, and the length fields are checked against the file. + if (pos == 8 || pos == 9) assertInstanceOf(UnsupportedException.class, e, what); + else if (pos >= 10) assertInstanceOf(CorruptException.class, e, what); + ch.write(ByteBuffer.wrap(new byte[] {old}), pos); + } + } + } + assertEquals(1, Manifest.readLatest(dir).generation(), "the loop restored every byte"); + } + + @Test + void everyCutOfTheManifestIsRefused() throws IOException { + copy(single, dir); + Path mf = dir.resolve(Manifest.manifestName(1)); + final int len = (int) Files.size(mf); + try (FileChannel ch = FileChannel.open(mf, StandardOpenOption.WRITE)) { + for (int cut = len - 1; cut >= 0; cut--) { + ch.truncate(cut); + assertInstanceOf(CorruptException.class, latestRefused(dir, "manifest cut at " + cut), "manifest cut at " + cut); + } + } + assertInstanceOf(CorruptException.class, openRefused(dir, "an empty manifest")); + } + + @Test + void everyBitAndEveryCutOfTheTombstoneFileIsRefused() throws IOException { + copy(single, dir); + Path tf = dir.resolve(Manifest.tombName(1)); + final int len = (int) Files.size(tf); + byte[] raw = Files.readAllBytes(tf); + try (FileChannel ch = FileChannel.open(tf, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + for (int pos = 0; pos < len; pos++) { + for (int bit = 0; bit < 8; bit++) { + byte old = flipInPlace(ch, pos, bit); + String what = "bit " + bit + " of tombstone byte " + pos; + IOException e = latestRefused(dir, what); + // the manifest holds the length and the CRC of the file it names, so this is found before the file is parsed + CorruptException cause = assertInstanceOf(CorruptException.class, e.getCause(), what); + assertEquals("TOMBSTONES", cause.section(), what); + ch.write(ByteBuffer.wrap(new byte[] {old}), pos); + } + } + } + Files.write(tf, raw); + try (FileChannel ch = FileChannel.open(tf, StandardOpenOption.WRITE)) { + for (int cut = len - 1; cut >= 0; cut--) { + ch.truncate(cut); + CorruptException cause = assertInstanceOf(CorruptException.class, latestRefused(dir, "tombstone cut at " + cut).getCause(), "cut at " + cut); + assertEquals("TOMBSTONES", cause.section(), "tombstone cut at " + cut); + } + } + Files.write(tf, raw); + assertEquals(1, Manifest.readLatest(dir).generation()); + } + + /** rewrites the tombstone file by {@code change} (the CRC inside it is recomputed) and the manifest to name the new bytes, so that only the change can be found */ + private void forgeTombstones(Consumer change) throws IOException { + Manifest m = Manifest.readLatest(dir); + Path tf = dir.resolve(Manifest.tombName(m.tombGeneration())); + byte[] t = Files.readAllBytes(tf); + change.accept(t); + SegmentForge.le(t).putInt(t.length - 4, SegmentFormat.crc32c(t, 0, t.length - 4)); + Files.write(tf, t); + Manifest m2 = new Manifest(m.generation(), m.epoch(), m.hwm(), m.versionCounter(), m.nextSegId(), m.fingerprint(), m.entries(), m.tombGeneration(), + t.length, SegmentFormat.crc32c(t, 0, t.length)); + Files.write(dir.resolve(Manifest.manifestName(m.generation())), m2.encode(Manifest.FORMAT_MAJOR, Manifest.FORMAT_MINOR)); + } + + @Test + void aTombstoneFileThatIsNotUnderstoodIsRefusedWithValidChecksums() throws IOException { + // a larger major + copy(single, dir); + forgeTombstones(t -> SegmentForge.le(t).putShort(8, (short) 2)); + assertInstanceOf(UnsupportedException.class, openRefused(dir, "tombstone major 2")); + // a larger minor opens + clear(); + copy(single, dir); + forgeTombstones(t -> SegmentForge.le(t).putShort(10, (short) 9)); + try (TieredIndex u = TieredIndex.open(dir, cfg())) { + assertServes(u, want1); + } + // a wrong magic + clear(); + copy(single, dir); + forgeTombstones(t -> t[0] ^= 1); + CorruptException e = assertInstanceOf(CorruptException.class, latestRefused(dir, "tombstone magic").getCause()); + assertEquals("TOMBSTONES", e.section()); + // a deleted row: this version stores none and does not guess + clear(); + copy(single, dir); + forgeTombstones(t -> t[24 + 16] |= 1); // the first word of the bitmap of the first segment + assertInstanceOf(UnsupportedException.class, openRefused(dir, "a tombstone bit set")); + } + + // ---- versions, in the directory ---- + + /** + * rewrites a segment file with {@code change} applied (the CRCs of the file are recomputed) and the newest manifest to name the new file, so that + * the manifest is intact and agrees with the segment + */ + private void rewriteSegment(long segId, Consumer change) throws IOException { + Manifest m = Manifest.readLatest(dir); + SegmentForge forge = SegmentForge.of(Files.readAllBytes(seg(segId))); + change.accept(forge); + byte[] bytes = forge.build(); + Files.write(seg(segId), bytes); + int hl = forge.headBuf().getInt(SegmentFormat.H_LEN); + int headerCrc = forge.headBuf().getInt(hl - 4); + List entries = new ArrayList<>(); + for (Manifest.Entry e : m.entries()) { + entries.add(e.segId() != segId ? e : new Manifest.Entry(e.segId(), e.n(), bytes.length, headerCrc, e.verMin(), e.verMax())); + } + Manifest m2 = new Manifest(m.generation(), m.epoch(), m.hwm(), m.versionCounter(), m.nextSegId(), m.fingerprint(), entries, m.tombGeneration(), + m.tombLength(), m.tombCrc()); + Files.write(dir.resolve(Manifest.manifestName(m.generation())), m2.encode(Manifest.FORMAT_MAJOR, Manifest.FORMAT_MINOR)); + } + + @Test + void aLargerMinorSegmentOpensAndServesWhatTheWritersTierServed() throws IOException { + copy(single, dir); + rewriteSegment(1, f -> f.headBuf().putShort(SegmentFormat.H_MINOR, (short) 3)); + rewriteSegment(2, f -> { + f.headBuf().putShort(SegmentFormat.H_MINOR, (short) 7); + f.appendSection(900, 0, 1, new byte[5000]); // a section of the newer minor that this reader does not know + }); + try (TieredIndex u = TieredIndex.open(dir, cfg())) { + assertNull(u.openFallbackCause()); + assertEquals(2 * ROWS, u.size()); + assertServes(u, want1); + } + // the same with the header and the footer metadata grown by the newer minor + clear(); + copy(single, dir); + rewriteSegment(1, f -> { + f.headBuf().putShort(SegmentFormat.H_MINOR, (short) 2); + f.headBuf().putInt(SegmentFormat.H_LEN, 60); + f.headBuf().putInt(52, 0x5EED5EED); + f.meta = Arrays.copyOf(f.meta, 96); + }); + try (TieredIndex u = TieredIndex.open(dir, cfg())) { + assertServes(u, want1); + } + } + + @Test + void aSegmentWithAnUnknownRequiredSectionIsRefusedNotSkipped() throws IOException { + copy(twoGen, dir); + // the newest segment of generation 2; generation 1 is intact and would open + rewriteSegment(3, f -> { + f.headBuf().putShort(SegmentFormat.H_MINOR, (short) 1); + f.appendSection(901, SegmentFormat.FLAG_REQUIRED, 1, new byte[64]); + }); + Map before = snapshot(dir); + assertInstanceOf(UnsupportedException.class, openRefused(dir, "unknown required section")); + assertUnchanged(before, dir, "unknown required section"); + TieredIndex r = TieredIndex.open(dir, DIM, Metric.L2, cfg(), () -> records1); + try { + assertInstanceOf(UnsupportedException.class, r.openFallbackCause()); + assertRebuiltFromTheRecords(r, "unknown required section"); + } finally { + r.close(); + } + } + + @Test + void aSegmentOfALargerMajorInTheNewestGenerationIsRefusedNotReplacedByAnOlderGeneration() throws IOException { + copy(twoGen, dir); + rewriteSegment(3, f -> f.headBuf().putShort(SegmentFormat.H_MAJOR, (short) 2)); + Map before = snapshot(dir); + // generation 1 is intact, and opening it would serve the tier as it was before the third segment: silently stale + assertInstanceOf(UnsupportedException.class, latestRefused(dir, "major 2")); + assertInstanceOf(UnsupportedException.class, openRefused(dir, "major 2")); + assertUnchanged(before, dir, "major 2"); + TieredIndex r = TieredIndex.open(dir, DIM, Metric.L2, cfg(), () -> records1); + try { + assertInstanceOf(UnsupportedException.class, r.openFallbackCause()); + } finally { + r.close(); + } + } + + // ---- damage to the newest of two generations ---- + + @Test + void aFlippedSectionBitInTheNewestGenerationIsRefusedNotHiddenByTheOlderOne() throws IOException { + for (int id : new int[] {1, 4, 12}) { + copy(twoGen, dir); + SegmentInfo.Section s = SegmentReader.inspect(seg(3)).section(id); + byte[] b = Files.readAllBytes(seg(3)); + b[(int) (s.offset() + s.length() / 2)] ^= 0x04; + Files.write(seg(3), b); + Map before = snapshot(dir); + // the header, footer and locator are intact, so generation 2 validates; its sections are read in full at open and one fails + CorruptException e = assertInstanceOf(CorruptException.class, openRefused(dir, "section " + id)); + assertEquals(SegmentFormat.sectionName(id), e.section()); + assertUnchanged(before, dir, "section " + id); + clear(); + } + } + + @Test + void damageToTheHeaderFooterOrLocatorOfTheNewestSegmentMakesTheOlderGenerationTheNewestThatValidates() throws IOException { + // the documented rule: the newest manifest that validates is live; one whose segment fails the header, footer or locator check is skipped. + // generation 1 is served as it was, and the files of generation 2 are removed by the open. + int[] spots = {20, 40, -1, -1}; + for (int k = 0; k < spots.length; k++) { + copy(twoGen, dir); + byte[] b = Files.readAllBytes(seg(3)); + int pos = k < 2 ? spots[k] : k == 2 ? footerOff(b) + 10 : b.length - 10; + b[pos] ^= 0x01; + Files.write(seg(3), b); + try (TieredIndex u = TieredIndex.open(dir, cfg())) { + assertNull(u.openFallbackCause()); + assertEquals(2 * ROWS, u.size(), "byte " + pos); + assertServes(u, want1); + } + Manifest live = Manifest.readLatest(dir); + assertEquals(1, live.generation(), "byte " + pos); + assertTrue(!Files.exists(seg(3)) && !Files.exists(dir.resolve(Manifest.manifestName(2))), "the files of the unfinished generation were removed"); + clear(); + } + } +}