diff --git a/src/test/java/com/humemai/vec/SegmentCorruptionTest.java b/src/test/java/com/humemai/vec/SegmentCorruptionTest.java new file mode 100644 index 0000000..d1eb59c --- /dev/null +++ b/src/test/java/com/humemai/vec/SegmentCorruptionTest.java @@ -0,0 +1,326 @@ +package com.humemai.vec; + +import static com.humemai.vec.SegmentForge.refused; +import static com.humemai.vec.SegmentTestSupport.graph; +import static com.humemai.vec.SegmentTestSupport.params; +import static com.humemai.vec.SegmentTestSupport.vectors; +import static com.humemai.vec.SegmentTestSupport.versOf; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.humemai.vec.SegmentFormat.CorruptException; +import com.humemai.vec.SegmentFormat.UnsupportedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Damage to a segment file, byte by byte. A file that is cut short, or has any bit of its header, footer, locator or sections flipped, is + * refused with a {@link CorruptException} (an {@link UnsupportedException} where the damaged bytes are the major version; for the magic either + * of the two named exceptions), never loaded, and never fails in any other way. The sweeps are exhaustive over a small file: every cut length, and every byte that a + * CRC or a length check covers. The only bytes whose damage is not detected are the zero fill that no CRC covers (the rest of the header + * block and the gaps between sections), and a test below shows that damage there changes nothing a reader serves. + * + *

The code reads files with plain {@code FileChannel} reads into heap arrays (no memory mapping and no native memory in this version), + * so a damaged file cannot take the JVM down; what the tests rule out is every other uncontrolled failure: an exception that is not one of + * the two named ones, an out-of-memory error from a length read out of the file, or a graph that is returned. + */ +class SegmentCorruptionTest { + private static final int N = 300, DIM = 4; + + @TempDir + Path tmp; + + private SealedGraph orig; + private byte[] good; + private SegmentInfo info; + private Path scratch; + + @BeforeEach + void write() throws IOException { + orig = graph(vectors(N, DIM, 17), Metric.L2, params(true)); // SQ8 on: all twelve sections are in the file + Path f = tmp.resolve("good.tvs"); + info = SegmentWriter.write(orig, versOf(N), f); + good = Files.readAllBytes(f); + scratch = tmp.resolve("scratch.tvs"); + Files.write(scratch, good); + assertEquals(12, info.sections().size()); + for (SegmentInfo.Section s : info.sections()) assertTrue(s.length() > 0, "section " + SegmentFormat.sectionName(s.id()) + " has bytes to damage"); + } + + private int footerOff() { + return good.length - SegmentFormat.LOCATOR_LEN - (SegmentFormat.F_META_LEN + SegmentFormat.ENTRY_LEN * info.sections().size()); + } + + /** flips one bit of the scratch file in place; returns the byte to give back to {@link #restore} */ + private byte flip(FileChannel ch, int pos, int bit) throws IOException { + byte old = good[pos]; + ch.write(ByteBuffer.wrap(new byte[] {(byte) (old ^ (1 << bit))}), pos); + return old; + } + + private void restore(FileChannel ch, int pos) throws IOException { + ch.write(ByteBuffer.wrap(new byte[] {good[pos]}), pos); + } + + // ---- truncation ---- + + @Test + void everyCutLengthIsRefusedWithACorruptException() throws IOException { + // the scratch file is cut one byte at a time from its full length down to nothing; a cut inside the header block leaves too little for a + // segment, every other cut leaves a file whose last 32 bytes are not the locator + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int len = good.length - 1; len >= 0; len--) { + ch.truncate(len); + CorruptException e = assertInstanceOf(CorruptException.class, refused(scratch, "cut at " + len), "cut at " + len); + assertEquals(len < SegmentFormat.HEADER_BLOCK + SegmentFormat.LOCATOR_LEN ? "HEADER" : "LOCATOR", e.section(), "cut at " + len); + } + } + assertEquals(0, Files.size(scratch)); + } + + @ParameterizedTest(name = "section id {0}") + @ValueSource(ints = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}) + void aFileCutInTheMiddleOfASectionOrAtItsLastByteFailsInConstantTime(int id) throws IOException { + SegmentInfo.Section sec = info.section(id); + for (long cut : new long[] {sec.offset() + sec.length() / 2, sec.offset() + sec.length() - 1}) { + Path p = tmp.resolve("cut-" + id + "-" + cut + ".tvs"); + Files.write(p, java.util.Arrays.copyOf(good, (int) cut)); + // inspect reads the header block and the last 32 bytes only, so no section is read + CorruptException e = assertInstanceOf(CorruptException.class, assertThrowsIo(() -> SegmentReader.inspect(p), "cut at " + cut)); + assertEquals("LOCATOR", e.section()); + assertEquals("LOCATOR", assertInstanceOf(CorruptException.class, refused(p, false, "cut at " + cut)).section()); + assertEquals("LOCATOR", assertInstanceOf(CorruptException.class, refused(p, true, "cut at " + cut)).section()); + } + } + + @Test + void aFileCutInsideTheFooterOrTheLocatorOrAtTheEndOfTheSectionsIsRefused() throws IOException { + final int footerOff = footerOff(); + final int[] cuts = {footerOff, footerOff + 1, footerOff + SegmentFormat.F_META_LEN, good.length - SegmentFormat.LOCATOR_LEN - 1, + good.length - SegmentFormat.LOCATOR_LEN, good.length - SegmentFormat.LOCATOR_LEN + 1, good.length - 8, good.length - 1}; + for (int cut : cuts) { + Path p = tmp.resolve("tail-" + cut + ".tvs"); + Files.write(p, java.util.Arrays.copyOf(good, cut)); + assertEquals("LOCATOR", assertInstanceOf(CorruptException.class, refused(p, "cut at " + cut)).section(), "cut at " + cut); + } + } + + @Test + void bytesAppendedToAGoodFileAreRefused() throws IOException { + for (int extra : new int[] {1, 7, 32, 4096}) { + byte[] b = java.util.Arrays.copyOf(good, good.length + extra); // zeros after the locator: it is no longer the last 32 bytes + Path p = tmp.resolve("appended-" + extra + ".tvs"); + Files.write(p, b); + assertEquals("LOCATOR", assertInstanceOf(CorruptException.class, refused(p, extra + " bytes appended")).section()); + } + } + + // ---- bit flips ---- + + @Test + void everyBitOfTheHeaderIsGuarded() throws IOException { + // bytes 0 to 9 are the magic and the major version: a flipped bit there names a file this reader does not understand. Bytes 10 to 51 + // (minor version, header length, probe, flags, dimensions, enum codes, alignment, the CRC itself) are covered by the header CRC. + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int pos = 0; pos < SegmentFormat.HEADER_LEN; pos++) { + for (int bit = 0; bit < 8; bit++) { + flip(ch, pos, bit); + String what = "bit " + bit + " of header byte " + pos; + for (boolean verify : new boolean[] {true, false}) { + IOException e = refused(scratch, verify, what); + if (pos >= 8 && pos < 10) assertInstanceOf(UnsupportedException.class, e, what); + else if (pos >= 10) assertEquals("HEADER", assertInstanceOf(CorruptException.class, e, what).section(), what); + } + restore(ch, pos); + } + } + } + } + + @Test + void everyBitOfTheFooterIsGuarded() throws IOException { + // the footer CRC in the locator covers the metadata and every section entry. The section table, with its offsets, widths, counts and + // CRCs, therefore cannot change without notice, and the check runs before any section is read (verifyAll false below). + final int footerOff = footerOff(), footerEnd = good.length - SegmentFormat.LOCATOR_LEN; + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int pos = footerOff; pos < footerEnd; pos++) { + for (int bit = 0; bit < 8; bit++) { + flip(ch, pos, bit); + String what = "bit " + bit + " of footer byte " + (pos - footerOff); + for (boolean verify : new boolean[] {true, false}) { + assertEquals("FOOTER", assertInstanceOf(CorruptException.class, refused(scratch, verify, what), what).section(), what); + } + restore(ch, pos); + } + } + } + } + + @Test + void everyBitOfTheLocatorIsGuarded() throws IOException { + // magic (0 to 7), footer offset (8 to 15), footer length (16 to 19) and file length (24 to 31) are checked against the file itself; the + // footer CRC (20 to 23) is checked against the footer + final int locOff = good.length - SegmentFormat.LOCATOR_LEN; + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int rel = 0; rel < SegmentFormat.LOCATOR_LEN; rel++) { + for (int bit = 0; bit < 8; bit++) { + flip(ch, locOff + rel, bit); + String what = "bit " + bit + " of locator byte " + rel; + for (boolean verify : new boolean[] {true, false}) { + CorruptException e = assertInstanceOf(CorruptException.class, refused(scratch, verify, what), what); + assertEquals(rel >= 20 && rel < 24 ? "FOOTER" : "LOCATOR", e.section(), what); + } + restore(ch, locOff + rel); + } + } + } + } + + @Test + void everyByteOfEverySectionIsCoveredByItsCrc() throws IOException { + // every byte of every section has one bit flipped (the bit changes from byte to byte), and the first, the middle and the last byte of each + // section have all eight flipped. The metadata is intact in all of these, so the CRC of exactly the damaged section must be what fails. + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (SegmentInfo.Section sec : info.sections()) { + final String name = SegmentFormat.sectionName(sec.id()); + final int first = (int) sec.offset(), last = (int) (sec.offset() + sec.length() - 1), mid = (int) (sec.offset() + sec.length() / 2); + for (int pos = first; pos <= last; pos++) { + boolean all = pos == first || pos == mid || pos == last; + for (int bit = all ? 0 : pos % 8; bit < (all ? 8 : pos % 8 + 1); bit++) { + flip(ch, pos, bit); + String what = "bit " + bit + " of byte " + (pos - first) + " of section " + name; + CorruptException e = assertInstanceOf(CorruptException.class, refused(scratch, what), what); + assertEquals(name, e.section(), what); + restore(ch, pos); + } + } + } + } + } + + @Test + void aFlippedSectionBitIsFoundOnlyWhenTheSectionIsChecksummed() throws IOException { + // the contract of read(file, false): header, footer and locator are checked, the section bytes are not. A host that wants every byte + // checked uses SealedGraph.load or TieredIndex.open, which always do. + SegmentInfo.Section vec = info.section(SegmentFormat.S_VEC); + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + flip(ch, (int) (vec.offset() + 5), 3); + } + SegmentReader.read(scratch, false); // loads: the flip is in a float of the vectors + assertEquals("VEC", assertInstanceOf(CorruptException.class, refused(scratch, "bit in VEC")).section()); + assertEquals("VEC", assertInstanceOf(CorruptException.class, assertThrowsIo(() -> SealedGraph.load(scratch), "SealedGraph.load")).section()); + } + + // ---- the zero fill that no CRC covers ---- + + @Test + void damageInTheZeroFillIsNotDetectedAndChangesNothingThatIsServed() throws IOException { + // the doc says so: header bytes 52 to 4095 and the gaps between sections are zero and covered by no CRC. A flip there is not an + // integrity failure because no reader looks at those bytes; the test pins that the loaded graph is the same one. + List gaps = new ArrayList<>(); // [from, to) + gaps.add(new int[] {SegmentFormat.HEADER_LEN, SegmentFormat.HEADER_BLOCK}); + for (int i = 0; i + 1 < info.sections().size(); i++) { + SegmentInfo.Section s = info.sections().get(i), next = info.sections().get(i + 1); + if (s.offset() + s.length() < next.offset()) gaps.add(new int[] {(int) (s.offset() + s.length()), (int) next.offset()}); + } + SegmentInfo.Section lastSec = info.sections().get(info.sections().size() - 1); + gaps.add(new int[] {(int) (lastSec.offset() + lastSec.length()), footerOff()}); + assertTrue(gaps.size() >= 12, "the gaps found: " + gaps.size()); + float[][] qs = vectors(30, DIM, 99); + try (FileChannel ch = FileChannel.open(scratch, StandardOpenOption.WRITE)) { + for (int[] g : gaps) { + for (int pos : new int[] {g[0], (g[0] + g[1]) / 2, g[1] - 1}) { + if (pos < g[0] || pos >= g[1]) continue; + for (int bit : new int[] {0, 7}) { + flip(ch, pos, bit); + SealedGraph h = SealedGraph.load(scratch); + assertArrayEquals(orig.ids, h.ids, "zero fill byte " + pos); + assertArrayEquals(orig.vec, h.vec, "zero fill byte " + pos); + for (float[] q : qs) assertArrayEquals(orig.search(q, 10, null), h.search(q, 10, null), "zero fill byte " + pos); + restore(ch, pos); + } + } + } + } + // and the gaps really are zero in a file the writer wrote + for (int[] g : gaps) { + for (int pos = g[0]; pos < g[1]; pos++) assertEquals(0, good[pos], "byte " + pos + " of the fill"); + } + } + + // ---- wrong magic and files that are no segment ---- + + @Test + void aWrongMagicIsRefusedWhicheverByteIsWrong() throws IOException { + for (int pos = 0; pos < SegmentFormat.MAGIC.length; pos++) { + for (int x : new int[] {0x01, 0x80, 0xFF}) { + byte[] b = good.clone(); + b[pos] ^= (byte) x; + Path p = tmp.resolve("magic-" + pos + "-" + x + ".tvs"); + // the magic is checked before any CRC; which of the two named exceptions says "not a segment file" is not part of the claim + refused(p, b, "magic byte " + pos + " ^ " + x); + } + } + byte[] zeroed = good.clone(); + java.util.Arrays.fill(zeroed, 0, 8, (byte) 0); + refused(tmp.resolve("zeroed-magic.tvs"), zeroed, "zeroed magic"); + // the magic of a manifest, a tombstone file and the locator are not the magic of a segment + for (String other : new String[] {"TVMAN001", "TVTOMB01", "TVSEGEND", "tvseg001", "TVSEG002"}) { + byte[] b = good.clone(); + System.arraycopy(other.getBytes(java.nio.charset.StandardCharsets.US_ASCII), 0, b, 0, 8); + refused(tmp.resolve("other-" + other + ".tvs"), b, other); + } + } + + @Test + void filesThatAreNoSegmentAtAllAreRefused() throws IOException { + Random r = new Random(5); + byte[] noise = new byte[100_000]; + r.nextBytes(noise); + refused(tmp.resolve("noise.tvs"), noise, "random bytes"); + byte[] text = "this is a text file, not a segment, but it is longer than the header".getBytes(java.nio.charset.StandardCharsets.US_ASCII); + refused(tmp.resolve("text.tvs"), text, "text"); + // shorter than the magic: too short to be anything + for (int len : new int[] {0, 1, 7}) { + assertInstanceOf(CorruptException.class, refused(tmp.resolve("short" + len + ".tvs"), new byte[len], len + " bytes"), len + " bytes"); + } + // zeros: no magic + refused(tmp.resolve("zeros.tvs"), new byte[HEADER_AND_LOCATOR + 10], "zeros"); + // the right magic followed by nothing, and by random bytes + byte[] magicOnly = java.util.Arrays.copyOf(SegmentFormat.MAGIC, 8); + assertInstanceOf(CorruptException.class, refused(tmp.resolve("magic-only.tvs"), magicOnly, "magic only")); + byte[] magicNoise = noise.clone(); + System.arraycopy(SegmentFormat.MAGIC, 0, magicNoise, 0, 8); + refused(tmp.resolve("magic-noise.tvs"), magicNoise, "magic then random bytes"); // some named exception, never a graph + } + + private static final int HEADER_AND_LOCATOR = SegmentFormat.HEADER_BLOCK + SegmentFormat.LOCATOR_LEN; + + private interface IoCall { + void run() throws IOException; + } + + private static IOException assertThrowsIo(IoCall c, String what) { + try { + c.run(); + } catch (IOException e) { + return e; + } + throw new AssertionError(what + ": no exception"); + } +} diff --git a/src/test/java/com/humemai/vec/SegmentForge.java b/src/test/java/com/humemai/vec/SegmentForge.java new file mode 100644 index 0000000..17f4606 --- /dev/null +++ b/src/test/java/com/humemai/vec/SegmentForge.java @@ -0,0 +1,189 @@ +package com.humemai.vec; + +import static com.humemai.vec.SegmentFormat.F_COUNT; +import static com.humemai.vec.SegmentFormat.F_ENTRIES_OFF; +import static com.humemai.vec.SegmentFormat.HEADER_BLOCK; +import static com.humemai.vec.SegmentFormat.H_LEN; +import static com.humemai.vec.SegmentFormat.LOCATOR_LEN; +import static com.humemai.vec.SegmentFormat.LOCATOR_MAGIC; +import static com.humemai.vec.SegmentFormat.crc32c; +import static org.junit.jupiter.api.Assertions.fail; + +import com.humemai.vec.SegmentFormat.CorruptException; +import com.humemai.vec.SegmentFormat.UnsupportedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.opentest4j.AssertionFailedError; + +/** + * Takes a good segment file apart and puts it together again with chosen fields changed. {@link #build} recomputes every CRC and length + * that sits above the change (the header CRC, the section CRCs of changed sections, the footer CRC and the locator), so the file that comes + * out is damaged in exactly one place and a reader can only find that damage through the check that is meant to find it. A test that wants a + * CRC to stay wrong changes the bytes of the built file instead. + */ +final class SegmentForge { + /** one entry of the section table, as the footer holds it */ + static final class Entry { + int id; + int flags; + long width; + long count; + long offset; + long length; + int crc; + } + + /** the 4096-byte header block */ + final byte[] head; + /** the bytes from the end of the header block to the start of the footer: the sections and the zero fill between them */ + byte[] body; + /** the footer metadata that precedes the section entries */ + byte[] meta; + final List entries = new ArrayList<>(); + /** recompute the header CRC (at header length minus 4) in {@link #build} */ + boolean fixHeaderCrc = true; + /** write the entry count and the first entry offset into the footer metadata in {@link #build}; false keeps whatever {@link #meta} holds */ + boolean syncMeta = true; + /** values for the locator fields that replace the true ones, null keeps the true value */ + Long locFooterOff, locFooterLen, locFileLen; + + private SegmentForge(byte[] file) { + ByteBuffer b = le(file); + final int size = file.length; + final int footerOff = (int) b.getLong(size - LOCATOR_LEN + 8); + final int entriesOff = b.getInt(footerOff + F_ENTRIES_OFF); + final int count = b.getInt(footerOff + F_COUNT); + head = Arrays.copyOfRange(file, 0, HEADER_BLOCK); + body = Arrays.copyOfRange(file, HEADER_BLOCK, footerOff); + meta = Arrays.copyOfRange(file, footerOff, footerOff + entriesOff); + for (int i = 0; i < count; i++) { + int at = footerOff + entriesOff + i * SegmentFormat.ENTRY_LEN; + Entry e = new Entry(); + e.id = b.getShort(at) & 0xFFFF; + e.flags = b.getShort(at + 2) & 0xFFFF; + e.width = Integer.toUnsignedLong(b.getInt(at + 4)); + e.count = b.getLong(at + 8); + e.offset = b.getLong(at + 16); + e.length = b.getLong(at + 24); + e.crc = b.getInt(at + 32); + entries.add(e); + } + } + + /** takes apart a file that {@link SegmentWriter} wrote */ + static SegmentForge of(byte[] file) { + return new SegmentForge(file); + } + + static ByteBuffer le(byte[] a) { + return ByteBuffer.wrap(a).order(ByteOrder.LITTLE_ENDIAN); + } + + /** the header block for in-place changes of header fields */ + ByteBuffer headBuf() { + return le(head); + } + + /** the footer metadata for in-place changes */ + ByteBuffer metaBuf() { + return le(meta); + } + + Entry entry(int id) { + for (Entry e : entries) if (e.id == id) return e; + throw new IllegalArgumentException("no section " + id); + } + + byte[] sectionBytes(int id) { + Entry e = entry(id); + return Arrays.copyOfRange(body, (int) e.offset - HEADER_BLOCK, (int) (e.offset + e.length) - HEADER_BLOCK); + } + + /** replaces the bytes of a section (same length) and puts the CRC of the new bytes into its entry */ + void setSectionBytes(int id, byte[] data) { + Entry e = entry(id); + if (data.length != e.length) throw new IllegalArgumentException("length " + data.length + " != " + e.length); + System.arraycopy(data, 0, body, (int) e.offset - HEADER_BLOCK, data.length); + e.crc = crc32c(data, 0, data.length); + } + + /** appends a section after the last one, at the next aligned offset, with a correct CRC, and lists it in the table */ + Entry appendSection(int id, int flags, int width, byte[] data) { + long end = HEADER_BLOCK + body.length; + long off = SegmentFormat.alignUp(end); + byte[] nb = Arrays.copyOf(body, (int) (off - HEADER_BLOCK) + data.length); + System.arraycopy(data, 0, nb, (int) (off - HEADER_BLOCK), data.length); + body = nb; + Entry e = new Entry(); + e.id = id; + e.flags = flags; + e.width = width; + e.count = data.length / width; + e.offset = off; + e.length = data.length; + e.crc = crc32c(data, 0, data.length); + entries.add(e); + return e; + } + + /** the file, with the CRCs and the lengths above the changes recomputed */ + byte[] build() { + if (fixHeaderCrc) { + int hl = headBuf().getInt(H_LEN); + if (hl >= 8 && hl <= HEADER_BLOCK) headBuf().putInt(hl - 4, crc32c(head, 0, hl - 4)); + } + if (syncMeta) { + metaBuf().putInt(F_COUNT, entries.size()); + metaBuf().putInt(F_ENTRIES_OFF, meta.length); + } + ByteBuffer f = ByteBuffer.allocate(meta.length + SegmentFormat.ENTRY_LEN * entries.size()).order(ByteOrder.LITTLE_ENDIAN); + f.put(meta); + for (Entry e : entries) { + f.putShort((short) e.id).putShort((short) e.flags).putInt((int) e.width).putLong(e.count).putLong(e.offset).putLong(e.length); + f.putInt(e.crc).putInt(0); + } + byte[] footer = f.array(); + final long footerOff = HEADER_BLOCK + body.length; + final long fileLen = footerOff + footer.length + LOCATOR_LEN; + ByteBuffer out = ByteBuffer.allocate((int) fileLen).order(ByteOrder.LITTLE_ENDIAN); + out.put(head).put(body).put(footer); + out.put(LOCATOR_MAGIC).putLong(locFooterOff != null ? locFooterOff : footerOff).putInt((int) (locFooterLen != null ? locFooterLen : footer.length)); + out.putInt(crc32c(footer, 0, footer.length)).putLong(locFileLen != null ? locFileLen : fileLen); + return out.array(); + } + + /** + * Loads the file the way a host does ({@link SealedGraph#load}, every section checksummed) and returns the exception that refused it. + * Fails the test when a graph comes back, and when anything but a {@link CorruptException} or an {@link UnsupportedException} comes out: + * a runtime exception, an error or another IOException is an uncontrolled failure. + */ + static IOException refused(Path file, String what) { + return refused(file, true, what); + } + + static IOException refused(Path file, boolean verifyAll, String what) { + try { + SealedGraph g = SegmentReader.read(file, verifyAll); + fail(what + ": a graph with " + g.n + " rows came back from a damaged file"); + return null; + } catch (CorruptException | UnsupportedException e) { + return e; + } catch (AssertionFailedError e) { + throw e; + } catch (Throwable t) { + throw new AssertionFailedError(what + ": an uncontrolled failure instead of a named exception: " + t, t); + } + } + + /** the same as {@link #refused} for a file that is passed as bytes (written to {@code file} first) */ + static IOException refused(Path file, byte[] bytes, String what) throws IOException { + java.nio.file.Files.write(file, bytes); + return refused(file, what); + } +} diff --git a/src/test/java/com/humemai/vec/SegmentForgedFileTest.java b/src/test/java/com/humemai/vec/SegmentForgedFileTest.java new file mode 100644 index 0000000..b68ad4e --- /dev/null +++ b/src/test/java/com/humemai/vec/SegmentForgedFileTest.java @@ -0,0 +1,397 @@ +package com.humemai.vec; + +import static com.humemai.vec.SegmentForge.refused; +import static com.humemai.vec.SegmentTestSupport.graph; +import static com.humemai.vec.SegmentTestSupport.params; +import static com.humemai.vec.SegmentTestSupport.vectors; +import static com.humemai.vec.SegmentTestSupport.versOf; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.humemai.vec.SegmentFormat.CorruptException; +import com.humemai.vec.SegmentFormat.UnsupportedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Segment files that are damaged in exactly one place with every CRC above the damage recomputed (see {@link SegmentForge}), so that a reader + * can find the damage only through the rule that is meant to find it: the version rule, the byte-order probe, the section table checks, the + * locator checks and the checks of the stored id order and the layer 0 offsets. {@link SegmentCorruptionTest} covers the bytes that a CRC + * covers; this class covers the rules that hold when the CRCs are fine. + */ +class SegmentForgedFileTest { + private static final int N = 300, DIM = 4; + private static final float[][] QS = vectors(40, DIM, 123); + + @TempDir + Path tmp; + + private SealedGraph orig; + private byte[] good; + private Path f; + + @BeforeEach + void write() throws IOException { + orig = graph(vectors(N, DIM, 17), Metric.L2, params(true)); + Path g = tmp.resolve("good.tvs"); + SegmentWriter.write(orig, versOf(N), g); + good = Files.readAllBytes(g); + f = tmp.resolve("forged.tvs"); + } + + private SegmentForge forge() { + return SegmentForge.of(good); + } + + private void assertServedLikeTheOriginal(Path p) throws IOException { + SealedGraph h = SealedGraph.load(p); + assertArrayEquals(orig.ids, h.ids); + assertArrayEquals(orig.vec, h.vec); + assertArrayEquals(orig.level, h.level); + assertArrayEquals(orig.off0, h.off0); + assertArrayEquals(orig.edges0, h.edges0); + assertArrayEquals(orig.codes, h.codes); + for (float[] q : QS) assertArrayEquals(orig.search(q, 10, null), h.search(q, 10, null)); + } + + // ---- the version rule ---- + + @ParameterizedTest(name = "major {0}") + @ValueSource(ints = {0, 2, 3, 255, 256, 4097, 65535}) + void aMajorVersionThatIsNotOneIsRefusedAsUnsupportedWithValidChecksums(int major) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putShort(SegmentFormat.H_MAJOR, (short) major); + Files.write(f, forge.build()); + assertInstanceOf(UnsupportedException.class, refused(f, "major " + major)); + assertInstanceOf(UnsupportedException.class, refused(f, false, "major " + major)); + try { + SegmentReader.inspect(f); + throw new AssertionError("inspect returned for major " + major); + } catch (UnsupportedException expected) { + assertTrue(expected.getMessage().contains("major version " + major), expected.getMessage()); + } + } + + @ParameterizedTest(name = "minor {0}") + @ValueSource(ints = {1, 2, 255, 65535}) + void aLargerMinorVersionOpensAndServesTheSameGraph(int minor) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putShort(SegmentFormat.H_MINOR, (short) minor); + Files.write(f, forge.build()); + assertEquals(minor, SegmentReader.inspect(f).minor()); + assertServedLikeTheOriginal(f); + } + + @Test + void aLargerMinorMayAppendHeaderFieldsAndFooterMetadata() throws IOException { + // the header length says where the header CRC is, the offset of the first section entry says where the table starts + SegmentForge forge = forge(); + ByteBuffer h = forge.headBuf(); + h.putShort(SegmentFormat.H_MINOR, (short) 3); + h.putInt(SegmentFormat.H_LEN, 60); + h.putInt(52, 0x5EED5EED); // a field of the newer minor; the old CRC slot (48 to 51) is a field now, the CRC moves to 56 + forge.meta = java.util.Arrays.copyOf(forge.meta, 96); // 32 bytes of newer footer metadata + for (int i = SegmentFormat.F_META_LEN; i < 96; i++) forge.meta[i] = (byte) (i * 7); + Files.write(f, forge.build()); + SegmentInfo si = SegmentReader.inspect(f); + assertEquals(3, si.minor()); + assertEquals(12, si.sections().size()); + assertServedLikeTheOriginal(f); + // and the header CRC that moved is still checked + byte[] b = Files.readAllBytes(f); + b[53] ^= 0x01; + assertEquals("HEADER", assertInstanceOf(CorruptException.class, refused(f, b, "newer header field")).section()); + } + + @ParameterizedTest(name = "unknown optional section id {0}") + @ValueSource(ints = {13, 900, 4096, 65535}) + void aLargerMinorWithAnUnknownOptionalSectionIsOpenedAndTheSectionIsIgnored(int id) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putShort(SegmentFormat.H_MINOR, (short) 1); + SegmentForge.Entry e = forge.appendSection(id, 0, 1, new byte[5000]); + Files.write(f, forge.build()); + SegmentInfo si = SegmentReader.inspect(f); + assertNotNull(si.section(id)); + assertEquals(false, si.section(id).required()); + assertEquals(13, si.sections().size()); + assertServedLikeTheOriginal(f); + // the reader does not look at a section it does not know, so damage there is not an integrity failure of what it serves + byte[] b = Files.readAllBytes(f); + b[(int) (e.offset + 2500)] ^= 0x20; + Files.write(f, b); + assertServedLikeTheOriginal(f); + } + + @ParameterizedTest(name = "unknown required section id {0}, minor {1}") + @MethodSource("requiredUnknown") + void anUnknownRequiredSectionIsRefusedAsUnsupportedEvenWithValidChecksums(int id, int minor) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putShort(SegmentFormat.H_MINOR, (short) minor); + forge.appendSection(id, SegmentFormat.FLAG_REQUIRED, 1, new byte[100]); + Files.write(f, forge.build()); + for (boolean verify : new boolean[] {true, false}) { + UnsupportedException e = assertInstanceOf(UnsupportedException.class, refused(f, verify, "required section " + id)); + assertTrue(e.getMessage().contains("required section"), e.getMessage()); + } + try { + SegmentReader.inspect(f); + throw new AssertionError("inspect returned"); + } catch (UnsupportedException expected) { + // refused in constant time, before any section is read + } + } + + static Stream requiredUnknown() { + return Stream.of(Arguments.of(13, 1), Arguments.of(901, 1), Arguments.of(901, 0), Arguments.of(901, 9), Arguments.of(65535, 1)); + } + + // ---- the byte-order probe ---- + + @Test + void aProbeThatReadsBackSwappedIsRefusedAsUnsupported() throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putInt(SegmentFormat.H_PROBE, 0x04030201); + Files.write(f, forge.build()); + UnsupportedException e = assertInstanceOf(UnsupportedException.class, refused(f, "swapped probe")); + assertTrue(e.getMessage().contains("byte-order"), e.getMessage()); + assertInstanceOf(UnsupportedException.class, refused(f, false, "swapped probe")); + } + + @ParameterizedTest(name = "probe bit {0}") + @MethodSource("thirtyTwoBits") + void everySingleBitOfTheProbeIsRefusedAsUnsupportedWithValidChecksums(int bit) throws IOException { + SegmentForge forge = forge(); + forge.headBuf().putInt(SegmentFormat.H_PROBE, SegmentFormat.ENDIAN_PROBE ^ (1 << bit)); + Files.write(f, forge.build()); + assertInstanceOf(UnsupportedException.class, refused(f, "probe bit " + bit)); + } + + static IntStream thirtyTwoBits() { + return IntStream.range(0, 32); + } + + @Test + void aHeaderWrittenInBigEndianIsRefusedAsUnsupported() throws IOException { + // every multi-byte field of the header written the other way round, and a CRC that matches the bytes + SegmentForge forge = forge(); + ByteBuffer le = forge.headBuf(); + byte[] be = new byte[SegmentFormat.HEADER_BLOCK]; + ByteBuffer b = ByteBuffer.wrap(be).order(ByteOrder.BIG_ENDIAN); + System.arraycopy(forge.head, 0, be, 0, 8); + b.putShort(8, le.getShort(8)).putShort(10, le.getShort(10)).putInt(12, le.getInt(12)).putInt(16, le.getInt(16)); + b.putInt(20, le.getInt(20)).putInt(24, le.getInt(24)).putInt(28, le.getInt(28)).putLong(32, le.getLong(32)); + System.arraycopy(forge.head, 40, be, 40, 4); + b.putInt(44, le.getInt(44)); + b.putInt(48, SegmentFormat.crc32c(be, 0, 48)); + System.arraycopy(be, 0, forge.head, 0, SegmentFormat.HEADER_LEN); + forge.fixHeaderCrc = false; + Files.write(f, forge.build()); + assertEquals(0x0100, ByteBuffer.wrap(forge.head).order(ByteOrder.LITTLE_ENDIAN).getShort(8), "the major version reads as 256 in little-endian"); + // the first thing the reader trusts is the major version, and a big-endian writer's 1 reads as 256 + UnsupportedException e = assertInstanceOf(UnsupportedException.class, refused(f, "big-endian header")); + assertTrue(e.getMessage().contains("major version 256"), e.getMessage()); + } + + // ---- the header fields, with a valid header CRC ---- + + static Stream headerFields() { + return Stream.of( + header("row stride differs from the dimension", b -> b.putInt(SegmentFormat.H_STRIDE, DIM + 4), UnsupportedException.class, null), + header("metric code 3", b -> b.put(SegmentFormat.H_METRIC, (byte) 3), UnsupportedException.class, null), + header("metric code 255", b -> b.put(SegmentFormat.H_METRIC, (byte) 255), UnsupportedException.class, null), + header("id width 4", b -> b.put(SegmentFormat.H_IDW, (byte) 4), UnsupportedException.class, null), + header("layer 0 layout 1", b -> b.put(SegmentFormat.H_LAYOUT, (byte) 1), UnsupportedException.class, null), + header("dimension zero", b -> b.putInt(SegmentFormat.H_DIM, 0), CorruptException.class, "HEADER"), + header("dimension above 2^20", b -> b.putInt(SegmentFormat.H_DIM, (1 << 20) + 1), CorruptException.class, "HEADER"), + header("dimension one above", b -> { + b.putInt(SegmentFormat.H_DIM, DIM + 1); + b.putInt(SegmentFormat.H_STRIDE, DIM + 1); + }, CorruptException.class, "VEC"), + header("row count one above", b -> b.putLong(SegmentFormat.H_N, N + 1), CorruptException.class, "IDS"), + header("row count zero", b -> b.putLong(SegmentFormat.H_N, 0), CorruptException.class, "IDS"), + header("row count negative", b -> b.putLong(SegmentFormat.H_N, -1), IOException.class, null), + header("row count 100 million", b -> b.putLong(SegmentFormat.H_N, 100_000_000L), CorruptException.class, "IDS"), + header("row count 2^31 - 9", b -> b.putLong(SegmentFormat.H_N, Integer.MAX_VALUE - 9L), IOException.class, null), + header("row count 2^62", b -> b.putLong(SegmentFormat.H_N, 1L << 62), IOException.class, null), + header("alignment 3000, not a power of two", b -> b.putInt(SegmentFormat.H_ALIGN, 3000), CorruptException.class, "HEADER"), + header("alignment 4", b -> b.putInt(SegmentFormat.H_ALIGN, 4), CorruptException.class, "HEADER"), + header("alignment 2^31", b -> b.putInt(SegmentFormat.H_ALIGN, 1 << 31), CorruptException.class, "HEADER"), + header("alignment 8192 while the sections start at multiples of 4096", b -> b.putInt(SegmentFormat.H_ALIGN, 8192), CorruptException.class, "IDS"), + header("header length 51", b -> b.putInt(SegmentFormat.H_LEN, 51), CorruptException.class, "HEADER"), + header("header length 4097", b -> b.putInt(SegmentFormat.H_LEN, 4097), CorruptException.class, "HEADER"), + header("header length 2^32 - 1", b -> b.putInt(SegmentFormat.H_LEN, -1), CorruptException.class, "HEADER")); + } + + private static Arguments header(String name, java.util.function.Consumer change, Class type, String section) { + return Arguments.of(name, change, type, section); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("headerFields") + void aHeaderFieldThatThisReaderCannotUseIsRefusedWithValidChecksums(String name, java.util.function.Consumer change, + Class type, String section) throws IOException { + SegmentForge forge = forge(); + change.accept(forge.headBuf()); + Files.write(f, forge.build()); + IOException e = refused(f, name); + assertInstanceOf(type, e, name); + if (section != null) assertEquals(section, ((CorruptException) e).section(), name); + } + + // ---- the section table, the footer metadata and the locator, with valid CRCs ---- + + static Stream structure() { + final int footerMeta = SegmentFormat.F_META_LEN; + return Stream.of( + table("section offset not aligned", fg -> fg.entry(SegmentFormat.S_IDS).offset += 8, "IDS"), + table("section offset inside the header block", fg -> fg.entry(SegmentFormat.S_IDS).offset = 0, "IDS"), + table("two sections swapped in the file order", fg -> { + long a = fg.entries.get(1).offset; + fg.entries.get(1).offset = fg.entries.get(2).offset; + fg.entries.get(2).offset = a; + }, "BYID"), + table("a section starts where the one before it starts", fg -> fg.entry(SegmentFormat.S_BYID).offset = fg.entry(SegmentFormat.S_VERS).offset, "BYID"), + table("a section runs past the footer", fg -> fg.entry(SegmentFormat.S_SQ8_PARAMS).offset += 1 << 20, "SQ8_PARAMS"), + table("length is not width times count", fg -> fg.entry(SegmentFormat.S_VEC).length += 4, "VEC"), + table("element width zero", fg -> fg.entry(SegmentFormat.S_IDS).width = 0, "IDS"), + table("element width above 8", fg -> { + SegmentForge.Entry e = fg.entry(SegmentFormat.S_IDS); + e.width = 16; + e.count = e.length / 16; + }, "IDS"), + table("element width 4 for an i64 section", fg -> { + SegmentForge.Entry e = fg.entry(SegmentFormat.S_IDS); + e.width = 4; + e.count = e.length / 4; + }, "IDS"), + table("element count one short with a matching length", fg -> { + SegmentForge.Entry e = fg.entry(SegmentFormat.S_IDS); + e.count = N - 1; + e.length = 8L * (N - 1); + }, "IDS"), + table("negative element count", fg -> fg.entry(SegmentFormat.S_IDS).count = -1, "IDS"), + table("negative section length", fg -> fg.entry(SegmentFormat.S_IDS).length = -8, "IDS"), + table("SQ8 codes without their parameters", fg -> fg.entries.remove(fg.entry(SegmentFormat.S_SQ8_PARAMS)), "SQ8_PARAMS"), + table("SQ8 parameters without their codes", fg -> fg.entries.remove(fg.entry(SegmentFormat.S_SQ8_CODES)), "SQ8_PARAMS"), + table("a section listed twice", fg -> fg.appendSection(SegmentFormat.S_IDS, SegmentFormat.FLAG_REQUIRED, 8, new byte[8 * N]), "IDS"), + footer("more entries than the footer holds", fg -> fg.metaBuf().putInt(SegmentFormat.F_COUNT, fg.entries.size() + 1), "FOOTER"), + footer("more than 4096 entries", fg -> fg.metaBuf().putInt(SegmentFormat.F_COUNT, 5000), "FOOTER"), + footer("entry count 2^32 - 1", fg -> fg.metaBuf().putInt(SegmentFormat.F_COUNT, -1), "FOOTER"), + footer("first entry inside the metadata", fg -> fg.metaBuf().putInt(SegmentFormat.F_ENTRIES_OFF, footerMeta - 4), "FOOTER"), + footer("first entry beyond the footer", fg -> fg.metaBuf().putInt(SegmentFormat.F_ENTRIES_OFF, 100_000), "FOOTER"), + footer("entry row equal to the row count", fg -> fg.metaBuf().putInt(SegmentFormat.F_ENTRY_ROW, N), "FOOTER"), + footer("negative entry row", fg -> fg.metaBuf().putInt(SegmentFormat.F_ENTRY_ROW, -1), "FOOTER"), + footer("top level 128", fg -> fg.metaBuf().putInt(SegmentFormat.F_MAX_LEVEL, 128), "FOOTER"), + footer("negative top level", fg -> fg.metaBuf().putInt(SegmentFormat.F_MAX_LEVEL, -1), "FOOTER"), + loc("footer offset one too high", fg -> fg.locFooterOff = 4096L + fg.body.length + 1, "LOCATOR"), + loc("footer offset inside the header block", fg -> fg.locFooterOff = 100L, "LOCATOR"), + loc("footer offset zero", fg -> fg.locFooterOff = 0L, "LOCATOR"), + loc("footer length one too long", fg -> fg.locFooterLen = fg.meta.length + 40L * fg.entries.size() + 1, "LOCATOR"), + loc("footer length zero", fg -> fg.locFooterLen = 0L, "LOCATOR"), + loc("footer length above one MiB", fg -> fg.locFooterLen = (1L << 20) + 1, "LOCATOR"), + loc("file length one too long", fg -> fg.locFileLen = 4096L + fg.body.length + fg.meta.length + 40L * fg.entries.size() + 33, "LOCATOR"), + loc("file length zero", fg -> fg.locFileLen = 0L, "LOCATOR"), + loc("file length 2^63 - 1", fg -> fg.locFileLen = Long.MAX_VALUE, "LOCATOR"), + data("a row listed twice in the id order", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_BYID); + System.arraycopy(a, 0, a, 4, 4); // row at position 1 := row at position 0 + fg.setSectionBytes(SegmentFormat.S_BYID, a); + }, "BYID"), + data("a row number outside the rows in the id order", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_BYID); + SegmentForge.le(a).putInt(0, N); + fg.setSectionBytes(SegmentFormat.S_BYID, a); + }, "BYID"), + data("a negative row number in the id order", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_BYID); + SegmentForge.le(a).putInt(4 * (N - 1), -1); + fg.setSectionBytes(SegmentFormat.S_BYID, a); + }, "BYID"), + data("layer 0 offsets that end above the edge count", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_L0_OFF); + ByteBuffer b = SegmentForge.le(a); + b.putInt(4 * N, b.getInt(4 * N) + 1); + fg.setSectionBytes(SegmentFormat.S_L0_OFF, a); + }, "L0_OFF"), + data("layer 0 offsets that do not start at zero", fg -> { + byte[] a = fg.sectionBytes(SegmentFormat.S_L0_OFF); + SegmentForge.le(a).putInt(0, 1); + fg.setSectionBytes(SegmentFormat.S_L0_OFF, a); + }, "L0_OFF")); + } + + private static Arguments table(String name, java.util.function.Consumer change, String section) { + return Arguments.of("table: " + name, change, section); + } + + private static Arguments footer(String name, java.util.function.Consumer change, String section) { + return Arguments.of("footer: " + name, (java.util.function.Consumer) fg -> { + fg.syncMeta = false; + change.accept(fg); + }, section); + } + + private static Arguments loc(String name, java.util.function.Consumer change, String section) { + return Arguments.of("locator: " + name, change, section); + } + + private static Arguments data(String name, java.util.function.Consumer change, String section) { + return Arguments.of("data: " + name, change, section); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("structure") + void aStructureThatBreaksTheDocumentedRulesIsRefusedWithValidChecksums(String name, java.util.function.Consumer change, + String section) throws IOException { + SegmentForge forge = forge(); + change.accept(forge); + Files.write(f, forge.build()); + CorruptException e = assertInstanceOf(CorruptException.class, refused(f, name), name); + assertEquals(section, e.section(), name); + // the checks of the table, the footer and the locator also hold without the section CRCs; the data checks need the sections loaded + if (!name.startsWith("data:")) assertEquals(section, assertInstanceOf(CorruptException.class, refused(f, false, name), name).section(), name); + } + + @ParameterizedTest(name = "required section id {0} missing") + @ValueSource(ints = {1, 3, 4, 5, 6, 7, 8, 9, 10}) + void aMissingRequiredSectionIsRefusedWithValidChecksums(int id) throws IOException { + SegmentForge forge = forge(); + forge.entries.remove(forge.entry(id)); + Files.write(f, forge.build()); + for (boolean verify : new boolean[] {true, false}) { + CorruptException e = assertInstanceOf(CorruptException.class, refused(f, verify, "section " + id + " missing")); + assertEquals(SegmentFormat.sectionName(id), e.section()); + assertTrue(e.getMessage().contains("required section is missing"), e.getMessage()); + } + } + + @Test + void theOptionalSectionsMayBeAbsentTogether() throws IOException { + // SQ8 codes and parameters are optional as a pair; the version section is optional too (a standalone graph has none) + SegmentForge forge = forge(); + forge.entries.remove(forge.entry(SegmentFormat.S_SQ8_CODES)); + forge.entries.remove(forge.entry(SegmentFormat.S_SQ8_PARAMS)); + forge.entries.remove(forge.entry(SegmentFormat.S_VERS)); + Files.write(f, forge.build()); + SegmentReader.Loaded l = SegmentReader.readFull(f, true); + assertEquals(null, l.vers()); + assertEquals(null, l.graph().codes); + assertArrayEquals(orig.ids, l.graph().ids); + } +} diff --git a/src/test/java/com/humemai/vec/TierCorruptionTest.java b/src/test/java/com/humemai/vec/TierCorruptionTest.java new file mode 100644 index 0000000..5bb8b48 --- /dev/null +++ b/src/test/java/com/humemai/vec/TierCorruptionTest.java @@ -0,0 +1,548 @@ +package com.humemai.vec; + +import static com.humemai.vec.SegmentTestSupport.idOf; +import static com.humemai.vec.SegmentTestSupport.vectors; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.humemai.vec.SegmentFormat.CorruptException; +import com.humemai.vec.SegmentFormat.UnsupportedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.AssertionFailedError; + +/** + * Damage to an index directory, found by {@code TieredIndex.open}. The segment file rules are tested byte by byte in + * {@link SegmentCorruptionTest} and {@link SegmentForgedFileTest}; this class checks what the directory does with them: a damaged segment, manifest + * or tombstone file makes {@code open} throw a named exception (never return an index), a failed open deletes and changes nothing, the overload that + * holds the records falls back to a rebuild and serves them, a newer format is refused and not skipped, and a larger minor version opens and serves + * exactly what the writer's tier served. + */ +class TierCorruptionTest { + static final int DIM = TierReopenTest.DIM; + static final int ROWS = 400; + + /** the directory of the checkpoint of generation 1: two segments, seg-000001 and seg-000002 */ + @TempDir + static Path single; + /** a directory after a crash after the rename of generation 2: generation 1 (segments 1 and 2) and generation 2 (segments 1, 2 and 3) */ + @TempDir + static Path twoGen; + + static TieredIndex tier; + static float[][] data, qs; + static Hit[][] want1; + static List records1; + + @TempDir + Path dir; + + /** SQ8 on, so that a segment file has all twelve sections */ + static TieredIndex.Config cfg() { + return TierReopenTest.cfg().withGraph(GraphIndex.Params.defaults().withEfConstruction(64).withSq8(true)); + } + + @BeforeAll + static void build() throws IOException { + data = vectors(3 * ROWS, DIM, 91); + qs = vectors(40, DIM, 92); + tier = new TieredIndex(DIM, Metric.L2, cfg()); + batch(0); + batch(1); + tier.checkpoint(twoGen); // generation 1 + want1 = results(tier); + copy(twoGen, single); + records1 = new ArrayList<>(); + for (int i = 0; i < 2 * ROWS; i++) records1.add(new TieredIndex.IdVector(idOf(i), data[i])); + batch(2); + assertThrows(IOException.class, () -> tier.checkpoint(twoGen, step -> { + if (step == FaultHook.Step.DIR_SYNCED_AFTER_RENAME) throw new IOException("simulated crash after the rename and before the cleanup"); + })); + assertEquals(2, Manifest.readLatest(twoGen).generation()); + assertEquals(3, Manifest.readLatest(twoGen).entries().size()); + assertEquals(12, SegmentReader.inspect(single.resolve(Manifest.segName(1))).sections().size()); + assertEquals(1, Manifest.readLatest(single).generation()); + assertEquals(2, Manifest.readLatest(single).entries().size()); + assertTrue(Files.exists(twoGen.resolve(Manifest.manifestName(1))) && Files.exists(twoGen.resolve(Manifest.manifestName(2)))); + } + + @AfterAll + static void close() { + tier.close(); + } + + private static void batch(int b) { + for (int i = b * ROWS; i < (b + 1) * ROWS; i++) tier.add(idOf(i), data[i]); + tier.sealNow(); + assertTrue(tier.awaitIdle(60_000)); + } + + private static Hit[][] results(TieredIndex t) { + Hit[][] out = new Hit[qs.length][]; + for (int i = 0; i < qs.length; i++) out[i] = t.search(qs[i], 10, null); + return out; + } + + private static void assertServes(TieredIndex u, Hit[][] want) { + for (int i = 0; i < qs.length; i++) assertArrayEquals(want[i], u.search(qs[i], 10, null), "query " + i); + } + + private static void copy(Path from, Path to) throws IOException { + try (Stream s = Files.list(from)) { + for (Path p : (Iterable) s::iterator) Files.copy(p, to.resolve(p.getFileName().toString())); + } + } + + private Path seg(long id) { + return dir.resolve(Manifest.segName(id)); + } + + /** empties the test's directory so that the next copy of a template can go in */ + private void clear() throws IOException { + try (Stream s = Files.list(dir)) { + for (Path p : (Iterable) s::iterator) Files.delete(p); + } + } + + private static Map snapshot(Path d) throws IOException { + Map m = new TreeMap<>(); + try (Stream s = Files.list(d)) { + for (Path p : (Iterable) s::iterator) m.put(p.getFileName().toString(), Files.readAllBytes(p)); + } + return m; + } + + private static void assertUnchanged(Map before, Path d, String what) throws IOException { + Map after = snapshot(d); + assertEquals(before.keySet(), after.keySet(), what + ": the files in the directory"); + for (Map.Entry e : before.entrySet()) assertArrayEquals(e.getValue(), after.get(e.getKey()), what + ": " + e.getKey()); + } + + /** opens the directory; returns the named exception that refused it, and fails when an index comes back or anything else is thrown */ + private static IOException openRefused(Path d, String what) { + try { + TieredIndex u = TieredIndex.open(d, cfg()); + int size = u.size(); + u.close(); + fail(what + ": open returned an index with " + size + " rows from a damaged directory"); + return null; + } catch (CorruptException | UnsupportedException e) { + return e; + } catch (AssertionFailedError e) { + throw e; + } catch (Throwable t) { + throw new AssertionFailedError(what + ": an uncontrolled failure instead of a named exception: " + t, t); + } + } + + /** the same for {@link Manifest#readLatest} */ + private static IOException latestRefused(Path d, String what) { + try { + Manifest m = Manifest.readLatest(d); + fail(what + ": readLatest returned generation " + (m == null ? "null" : m.generation()) + " from a damaged manifest"); + return null; + } catch (CorruptException | UnsupportedException e) { + return e; + } catch (AssertionFailedError e) { + throw e; + } catch (Throwable t) { + throw new AssertionFailedError(what + ": an uncontrolled failure instead of a named exception: " + t, t); + } + } + + // ---- a damaged segment file ---- + + enum Expect { CORRUPT, UNSUPPORTED, NAMED } + + /** a way to damage the bytes of one segment file; {@code section} is the section the failure must name, when one is known */ + record Damage(String name, BiFunction apply, Expect expect, String section) { + @Override + public String toString() { + return name; + } + } + + private static byte[] cut(byte[] b, long len) { + return Arrays.copyOf(b, (int) len); + } + + private static byte[] flip(byte[] b, long pos) { + byte[] c = b.clone(); + c[(int) pos] ^= (byte) (1 << (pos % 8)); + return c; + } + + static Stream segmentDamage() { + List l = new ArrayList<>(); + for (int id = 1; id <= 12; id++) { + final int sid = id; + final String name = SegmentFormat.sectionName(id); + l.add(new Damage("cut in the middle of " + name, (b, i) -> cut(b, i.section(sid).offset() + i.section(sid).length() / 2), Expect.CORRUPT, null)); + l.add(new Damage("cut at the last byte of " + name, (b, i) -> cut(b, i.section(sid).offset() + i.section(sid).length() - 1), Expect.CORRUPT, null)); + l.add(new Damage("bit flipped in the first byte of " + name, (b, i) -> flip(b, i.section(sid).offset()), Expect.CORRUPT, name)); + l.add(new Damage("bit flipped in the middle of " + name, (b, i) -> flip(b, i.section(sid).offset() + i.section(sid).length() / 2), Expect.CORRUPT, name)); + l.add(new Damage("bit flipped in the last byte of " + name, (b, i) -> flip(b, i.section(sid).offset() + i.section(sid).length() - 1), Expect.CORRUPT, name)); + } + for (int len : new int[] {0, 7, 8, 100, SegmentFormat.HEADER_BLOCK, SegmentFormat.HEADER_BLOCK + SegmentFormat.LOCATOR_LEN - 1}) { + l.add(new Damage("cut to " + len + " bytes", (b, i) -> cut(b, len), Expect.CORRUPT, null)); + } + l.add(new Damage("cut at the start of the footer", (b, i) -> cut(b, footerOff(b)), Expect.CORRUPT, null)); + l.add(new Damage("cut in the middle of the footer", (b, i) -> cut(b, footerOff(b) + 100), Expect.CORRUPT, null)); + l.add(new Damage("cut in the middle of the locator", (b, i) -> cut(b, b.length - 16), Expect.CORRUPT, null)); + l.add(new Damage("cut at the last byte", (b, i) -> cut(b, b.length - 1), Expect.CORRUPT, null)); + l.add(new Damage("a byte appended", (b, i) -> cut(b, b.length + 1), Expect.CORRUPT, null)); + for (int pos : new int[] {10, 11, 12, 16, 24, 32, 40, 44, 48, 51}) { + l.add(new Damage("bit flipped in header byte " + pos, (b, i) -> flip(b, pos), Expect.CORRUPT, null)); + } + l.add(new Damage("bit flipped in the magic", (b, i) -> flip(b, 3), Expect.NAMED, null)); + l.add(new Damage("bit flipped in the major version", (b, i) -> flip(b, 8), Expect.UNSUPPORTED, null)); + l.add(new Damage("magic zeroed", (b, i) -> { + byte[] c = b.clone(); + Arrays.fill(c, 0, 8, (byte) 0); + return c; + }, Expect.NAMED, null)); + l.add(new Damage("all bytes zeroed", (b, i) -> new byte[b.length], Expect.NAMED, null)); + for (int rel : new int[] {0, 5, 40, 64 + 12, 64 + 40 * 11 + 39}) { + l.add(new Damage("bit flipped in footer byte " + rel, (b, i) -> flip(b, footerOff(b) + rel), Expect.CORRUPT, null)); + } + for (int rel : new int[] {0, 7, 8, 16, 20, 24, 31}) { + l.add(new Damage("bit flipped in locator byte " + rel, (b, i) -> flip(b, b.length - SegmentFormat.LOCATOR_LEN + rel), Expect.CORRUPT, null)); + } + return l.stream(); + } + + private static int footerOff(byte[] file) { + return (int) SegmentForge.le(file).getLong(file.length - SegmentFormat.LOCATOR_LEN + 8); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("segmentDamage") + void aDamagedSegmentFileIsRefusedByOpenAndNothingIsDeleted(Damage damage) throws IOException { + for (long segId : new long[] {1, 2}) { + copy(single, dir); + SegmentInfo info = SegmentReader.inspect(seg(segId)); + Files.write(seg(segId), damage.apply().apply(Files.readAllBytes(seg(segId)), info)); + Map before = snapshot(dir); + String what = damage.name() + " in segment " + segId; + IOException e = openRefused(dir, what); + switch (damage.expect()) { + case CORRUPT: assertInstanceOf(CorruptException.class, e, what); break; + case UNSUPPORTED: assertInstanceOf(UnsupportedException.class, e, what); break; + default: break; + } + // a flip inside a section is found when the section is read: the exception names it + if (damage.section() != null) assertEquals(damage.section(), ((CorruptException) e).section(), what); + assertUnchanged(before, dir, what); + clear(); + } + } + + // ---- the fallback to the records ---- + + private void assertRebuiltFromTheRecords(TieredIndex r, String what) { + assertEquals(2 * ROWS, r.size(), what); + int hits = 0; + for (int i = 0; i < 2 * ROWS; i += 20) { + Hit[] h = r.search(data[i], 1, null); + if (h.length == 1 && h[0].id() == idOf(i)) hits++; + } + assertTrue(hits >= 38, what + ": the rebuilt index finds its own records, " + hits + " of 40"); + } + + @Test + void whenOpenCannotTrustTheDirectoryTheRecordsAreServedAndTheDirectoryIsLeftAlone() throws IOException { + List picks = new ArrayList<>(); + segmentDamage().forEach(d -> { + if (d.name().equals("bit flipped in the middle of VEC") || d.name().equals("cut in the middle of L0_EDGES") || d.name().equals("bit flipped in header byte 24") + || d.name().equals("magic zeroed") || d.name().equals("bit flipped in the major version") || d.name().equals("bit flipped in locator byte 20") + || d.name().equals("cut at the last byte of SQ8_PARAMS")) picks.add(d); + }); + assertEquals(7, picks.size()); + for (Damage d : picks) { + copy(single, dir); + SegmentInfo info = SegmentReader.inspect(seg(1)); + Files.write(seg(1), d.apply().apply(Files.readAllBytes(seg(1)), info)); + Map before = snapshot(dir); + TieredIndex r = TieredIndex.open(dir, DIM, Metric.L2, cfg(), () -> records1); + try { + Throwable cause = r.openFallbackCause(); + assertTrue(cause instanceof CorruptException || cause instanceof UnsupportedException, d.name() + ": the cause is " + cause); + assertRebuiltFromTheRecords(r, d.name()); + } finally { + r.close(); + } + assertUnchanged(before, dir, d.name()); + clear(); + } + } + + // ---- the manifest and the tombstone file ---- + + /** flips bit {@code bit} of byte {@code pos} of a file in place; the old byte is returned */ + private static byte flipInPlace(FileChannel ch, long pos, int bit) throws IOException { + ByteBuffer b = ByteBuffer.allocate(1); + ch.read(b, pos); + byte old = b.get(0); + ch.write(ByteBuffer.wrap(new byte[] {(byte) (old ^ (1 << bit))}), pos); + return old; + } + + @Test + void everyBitOfTheManifestIsGuarded() throws IOException { + copy(single, dir); + Path mf = dir.resolve(Manifest.manifestName(1)); + final int len = (int) Files.size(mf); + try (FileChannel ch = FileChannel.open(mf, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + for (int pos = 0; pos < len; pos++) { + for (int bit = 0; bit < 8; bit++) { + byte old = flipInPlace(ch, pos, bit); + String what = "bit " + bit + " of manifest byte " + pos; + IOException e = latestRefused(dir, what); + // bytes 8 and 9 are the major version: a newer format is refused, not skipped. Everything from the minor version on is + // under the CRC of the body, and the length fields are checked against the file. + if (pos == 8 || pos == 9) assertInstanceOf(UnsupportedException.class, e, what); + else if (pos >= 10) assertInstanceOf(CorruptException.class, e, what); + ch.write(ByteBuffer.wrap(new byte[] {old}), pos); + } + } + } + assertEquals(1, Manifest.readLatest(dir).generation(), "the loop restored every byte"); + } + + @Test + void everyCutOfTheManifestIsRefused() throws IOException { + copy(single, dir); + Path mf = dir.resolve(Manifest.manifestName(1)); + final int len = (int) Files.size(mf); + try (FileChannel ch = FileChannel.open(mf, StandardOpenOption.WRITE)) { + for (int cut = len - 1; cut >= 0; cut--) { + ch.truncate(cut); + assertInstanceOf(CorruptException.class, latestRefused(dir, "manifest cut at " + cut), "manifest cut at " + cut); + } + } + assertInstanceOf(CorruptException.class, openRefused(dir, "an empty manifest")); + } + + @Test + void everyBitAndEveryCutOfTheTombstoneFileIsRefused() throws IOException { + copy(single, dir); + Path tf = dir.resolve(Manifest.tombName(1)); + final int len = (int) Files.size(tf); + byte[] raw = Files.readAllBytes(tf); + try (FileChannel ch = FileChannel.open(tf, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + for (int pos = 0; pos < len; pos++) { + for (int bit = 0; bit < 8; bit++) { + byte old = flipInPlace(ch, pos, bit); + String what = "bit " + bit + " of tombstone byte " + pos; + IOException e = latestRefused(dir, what); + // the manifest holds the length and the CRC of the file it names, so this is found before the file is parsed + CorruptException cause = assertInstanceOf(CorruptException.class, e.getCause(), what); + assertEquals("TOMBSTONES", cause.section(), what); + ch.write(ByteBuffer.wrap(new byte[] {old}), pos); + } + } + } + Files.write(tf, raw); + try (FileChannel ch = FileChannel.open(tf, StandardOpenOption.WRITE)) { + for (int cut = len - 1; cut >= 0; cut--) { + ch.truncate(cut); + CorruptException cause = assertInstanceOf(CorruptException.class, latestRefused(dir, "tombstone cut at " + cut).getCause(), "cut at " + cut); + assertEquals("TOMBSTONES", cause.section(), "tombstone cut at " + cut); + } + } + Files.write(tf, raw); + assertEquals(1, Manifest.readLatest(dir).generation()); + } + + /** rewrites the tombstone file by {@code change} (the CRC inside it is recomputed) and the manifest to name the new bytes, so that only the change can be found */ + private void forgeTombstones(Consumer change) throws IOException { + Manifest m = Manifest.readLatest(dir); + Path tf = dir.resolve(Manifest.tombName(m.tombGeneration())); + byte[] t = Files.readAllBytes(tf); + change.accept(t); + SegmentForge.le(t).putInt(t.length - 4, SegmentFormat.crc32c(t, 0, t.length - 4)); + Files.write(tf, t); + Manifest m2 = new Manifest(m.generation(), m.epoch(), m.hwm(), m.versionCounter(), m.nextSegId(), m.fingerprint(), m.entries(), m.tombGeneration(), + t.length, SegmentFormat.crc32c(t, 0, t.length)); + Files.write(dir.resolve(Manifest.manifestName(m.generation())), m2.encode(Manifest.FORMAT_MAJOR, Manifest.FORMAT_MINOR)); + } + + @Test + void aTombstoneFileThatIsNotUnderstoodIsRefusedWithValidChecksums() throws IOException { + // a larger major + copy(single, dir); + forgeTombstones(t -> SegmentForge.le(t).putShort(8, (short) 2)); + assertInstanceOf(UnsupportedException.class, openRefused(dir, "tombstone major 2")); + // a larger minor opens + clear(); + copy(single, dir); + forgeTombstones(t -> SegmentForge.le(t).putShort(10, (short) 9)); + try (TieredIndex u = TieredIndex.open(dir, cfg())) { + assertServes(u, want1); + } + // a wrong magic + clear(); + copy(single, dir); + forgeTombstones(t -> t[0] ^= 1); + CorruptException e = assertInstanceOf(CorruptException.class, latestRefused(dir, "tombstone magic").getCause()); + assertEquals("TOMBSTONES", e.section()); + // a deleted row: this version stores none and does not guess + clear(); + copy(single, dir); + forgeTombstones(t -> t[24 + 16] |= 1); // the first word of the bitmap of the first segment + assertInstanceOf(UnsupportedException.class, openRefused(dir, "a tombstone bit set")); + } + + // ---- versions, in the directory ---- + + /** + * rewrites a segment file with {@code change} applied (the CRCs of the file are recomputed) and the newest manifest to name the new file, so that + * the manifest is intact and agrees with the segment + */ + private void rewriteSegment(long segId, Consumer change) throws IOException { + Manifest m = Manifest.readLatest(dir); + SegmentForge forge = SegmentForge.of(Files.readAllBytes(seg(segId))); + change.accept(forge); + byte[] bytes = forge.build(); + Files.write(seg(segId), bytes); + int hl = forge.headBuf().getInt(SegmentFormat.H_LEN); + int headerCrc = forge.headBuf().getInt(hl - 4); + List entries = new ArrayList<>(); + for (Manifest.Entry e : m.entries()) { + entries.add(e.segId() != segId ? e : new Manifest.Entry(e.segId(), e.n(), bytes.length, headerCrc, e.verMin(), e.verMax())); + } + Manifest m2 = new Manifest(m.generation(), m.epoch(), m.hwm(), m.versionCounter(), m.nextSegId(), m.fingerprint(), entries, m.tombGeneration(), + m.tombLength(), m.tombCrc()); + Files.write(dir.resolve(Manifest.manifestName(m.generation())), m2.encode(Manifest.FORMAT_MAJOR, Manifest.FORMAT_MINOR)); + } + + @Test + void aLargerMinorSegmentOpensAndServesWhatTheWritersTierServed() throws IOException { + copy(single, dir); + rewriteSegment(1, f -> f.headBuf().putShort(SegmentFormat.H_MINOR, (short) 3)); + rewriteSegment(2, f -> { + f.headBuf().putShort(SegmentFormat.H_MINOR, (short) 7); + f.appendSection(900, 0, 1, new byte[5000]); // a section of the newer minor that this reader does not know + }); + try (TieredIndex u = TieredIndex.open(dir, cfg())) { + assertNull(u.openFallbackCause()); + assertEquals(2 * ROWS, u.size()); + assertServes(u, want1); + } + // the same with the header and the footer metadata grown by the newer minor + clear(); + copy(single, dir); + rewriteSegment(1, f -> { + f.headBuf().putShort(SegmentFormat.H_MINOR, (short) 2); + f.headBuf().putInt(SegmentFormat.H_LEN, 60); + f.headBuf().putInt(52, 0x5EED5EED); + f.meta = Arrays.copyOf(f.meta, 96); + }); + try (TieredIndex u = TieredIndex.open(dir, cfg())) { + assertServes(u, want1); + } + } + + @Test + void aSegmentWithAnUnknownRequiredSectionIsRefusedNotSkipped() throws IOException { + copy(twoGen, dir); + // the newest segment of generation 2; generation 1 is intact and would open + rewriteSegment(3, f -> { + f.headBuf().putShort(SegmentFormat.H_MINOR, (short) 1); + f.appendSection(901, SegmentFormat.FLAG_REQUIRED, 1, new byte[64]); + }); + Map before = snapshot(dir); + assertInstanceOf(UnsupportedException.class, openRefused(dir, "unknown required section")); + assertUnchanged(before, dir, "unknown required section"); + TieredIndex r = TieredIndex.open(dir, DIM, Metric.L2, cfg(), () -> records1); + try { + assertInstanceOf(UnsupportedException.class, r.openFallbackCause()); + assertRebuiltFromTheRecords(r, "unknown required section"); + } finally { + r.close(); + } + } + + @Test + void aSegmentOfALargerMajorInTheNewestGenerationIsRefusedNotReplacedByAnOlderGeneration() throws IOException { + copy(twoGen, dir); + rewriteSegment(3, f -> f.headBuf().putShort(SegmentFormat.H_MAJOR, (short) 2)); + Map before = snapshot(dir); + // generation 1 is intact, and opening it would serve the tier as it was before the third segment: silently stale + assertInstanceOf(UnsupportedException.class, latestRefused(dir, "major 2")); + assertInstanceOf(UnsupportedException.class, openRefused(dir, "major 2")); + assertUnchanged(before, dir, "major 2"); + TieredIndex r = TieredIndex.open(dir, DIM, Metric.L2, cfg(), () -> records1); + try { + assertInstanceOf(UnsupportedException.class, r.openFallbackCause()); + } finally { + r.close(); + } + } + + // ---- damage to the newest of two generations ---- + + @Test + void aFlippedSectionBitInTheNewestGenerationIsRefusedNotHiddenByTheOlderOne() throws IOException { + for (int id : new int[] {1, 4, 12}) { + copy(twoGen, dir); + SegmentInfo.Section s = SegmentReader.inspect(seg(3)).section(id); + byte[] b = Files.readAllBytes(seg(3)); + b[(int) (s.offset() + s.length() / 2)] ^= 0x04; + Files.write(seg(3), b); + Map before = snapshot(dir); + // the header, footer and locator are intact, so generation 2 validates; its sections are read in full at open and one fails + CorruptException e = assertInstanceOf(CorruptException.class, openRefused(dir, "section " + id)); + assertEquals(SegmentFormat.sectionName(id), e.section()); + assertUnchanged(before, dir, "section " + id); + clear(); + } + } + + @Test + void damageToTheHeaderFooterOrLocatorOfTheNewestSegmentMakesTheOlderGenerationTheNewestThatValidates() throws IOException { + // the documented rule: the newest manifest that validates is live; one whose segment fails the header, footer or locator check is skipped. + // generation 1 is served as it was, and the files of generation 2 are removed by the open. + int[] spots = {20, 40, -1, -1}; + for (int k = 0; k < spots.length; k++) { + copy(twoGen, dir); + byte[] b = Files.readAllBytes(seg(3)); + int pos = k < 2 ? spots[k] : k == 2 ? footerOff(b) + 10 : b.length - 10; + b[pos] ^= 0x01; + Files.write(seg(3), b); + try (TieredIndex u = TieredIndex.open(dir, cfg())) { + assertNull(u.openFallbackCause()); + assertEquals(2 * ROWS, u.size(), "byte " + pos); + assertServes(u, want1); + } + Manifest live = Manifest.readLatest(dir); + assertEquals(1, live.generation(), "byte " + pos); + assertTrue(!Files.exists(seg(3)) && !Files.exists(dir.resolve(Manifest.manifestName(2))), "the files of the unfinished generation were removed"); + clear(); + } + } +}