From 7c899af9a90a6aee64522392608b7887449ebf19 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Tue, 4 Aug 2026 18:02:53 -0400 Subject: [PATCH 1/7] fix(wasm): bound the sizes read out of compressed headers Adding unit tests to the WASM crate turned up a crash: blosc_decompress panics on a corrupt header instead of returning the empty Vec the TypeScript layer expects. blosc_decompress(&[0xff; 64]) -> panicked at blusc-0.0.6/src/internal/mod.rs:597: range end index 4294967311 out of range for slice of length 64 blusc reads nbytes/cbytes straight out of the header and, on the BLOSC_MEMCPYED path, copies that many bytes out of the source without first checking the source is that long. It allocates the declared nbytes up front too, so the same header asks for a 4 GB buffer on the way past. This matters more in WASM than the stack trace suggests. A panic traps the instance and every later call into the module fails with it, so a single corrupt chunk does not fail one read - it breaks every subsequent read for the life of the page. The other two decoders have the same shape without the panic: - lz4_flex::block::decompress zero-fills the declared original size before it looks at a byte of the block, so a header claiming u32::MAX is a 4 GB calloc from an 8-byte input. - zstd::bulk::decompress reserves the frame's declared content size, which the frame header carries as a u64. On wasm32 all three end the same way: the allocation fails, and an allocation failure aborts exactly like a panic does. Each decoder now validates before delegating. The Blosc guard checks the version byte, that cbytes fits in the buffer we were handed, that nbytes is within BLOSC2_MAX_BUFFERSIZE, and - for memcpy'd frames - the bound blusc itself omits. LZ4 and Zstd bound the declared size against their own format's maximum expansion. Those expansion bounds are measured rather than guessed, because a bound that is too tight silently rejects valid data: lz4_flex 16 MB of one byte -> 255.0:1 (a match-length extension byte adds 255; guard uses 256) zstd 64 MB of one byte -> 32498:1 (a 4-byte RLE block regenerates a 128 KB block, so 32768:1 is the ceiling; guard uses 32768) Tests compress a constant buffer - which sits on that ceiling - and assert the guard accepts it, so tightening either bound past what real data produces fails here rather than in the field. Also replaces an `as usize` on the 64-bit frame content size field, which on wasm32 would wrap a >4 GiB declaration into a small plausible-looking one instead of rejecting it. The crate had no tests at all before this, despite rust-ci.yml running cargo test -- --test-threads=1: the suite was passing vacuously. 28 tests now cover the decoders' error paths, the framing each codec emits, and the branches of the zstd frame header parser. Verified by mutation: the blosc case was red before the fix and green after, and reverting either size guard fails its own test. Co-Authored-By: Claude Opus 5 (1M context) --- crates/rumcodecs-wasm/src/lib.rs | 503 ++++++++++++++++++++++++++++++- 1 file changed, 497 insertions(+), 6 deletions(-) diff --git a/crates/rumcodecs-wasm/src/lib.rs b/crates/rumcodecs-wasm/src/lib.rs index 4b2b619..eda6d89 100644 --- a/crates/rumcodecs-wasm/src/lib.rs +++ b/crates/rumcodecs-wasm/src/lib.rs @@ -4,8 +4,8 @@ use blusc::api::{ blosc2_cbuffer_sizes as blusc_cbuffer_sizes, blosc2_compress_ctx, blosc2_create_cctx, Blosc2Cparams, BLOSC2_CPARAMS_DEFAULTS, }; -use blusc::internal::constants::*; use blusc::convenience::blosc1_decompress as blusc_decompress; +use blusc::internal::constants::*; fn cname_to_compcode(cname: &str) -> u8 { match cname { @@ -59,8 +59,60 @@ pub fn blosc_compress( dest } +/// A Blosc header is untrusted input: `blusc` reads the buffer sizes straight out +/// of it and then indexes the source with them, so a corrupt chunk can panic +/// inside the decompressor. A panic traps the whole WASM instance and takes every +/// later call down with it, so implausible headers are rejected here instead and +/// `src/blosc.ts` turns the empty result into a thrown Error. +fn blosc_header_is_sane(data: &[u8]) -> bool { + if data.len() < BLOSC_MIN_HEADER_LENGTH { + return false; + } + + let version = data[0]; + if version == 0 || version > BLOSC2_VERSION_FORMAT_STABLE { + return false; + } + + let header_len = if version == BLOSC2_VERSION_FORMAT_STABLE + || version == BLOSC2_VERSION_FORMAT_BETA1 + || version == BLOSC2_VERSION_FORMAT_ALPHA + { + BLOSC_EXTENDED_HEADER_LENGTH + } else { + BLOSC_MIN_HEADER_LENGTH + }; + if data.len() < header_len { + return false; + } + + let (nbytes, cbytes, _) = blusc_cbuffer_sizes(data); + + // The frame records its own compressed length; it cannot exceed what we hold. + if cbytes < header_len || cbytes > data.len() { + return false; + } + + // A header claiming more than Blosc allows only gets us an allocation + // failure, which aborts the instance just like a panic does. + if nbytes > BLOSC2_MAX_BUFFERSIZE { + return false; + } + + // An uncompressed (memcpy'd) frame stores nbytes verbatim after the header. + // This is the bound blusc itself omits before copying out of the source. + if data[2] & BLOSC_MEMCPYED != 0 && header_len.saturating_add(nbytes) > data.len() { + return false; + } + + true +} + #[wasm_bindgen] pub fn blosc_decompress(data: &[u8]) -> Vec { + if !blosc_header_is_sane(data) { + return vec![]; + } match blusc_decompress(data) { Ok(result) => result, Err(_) => vec![], @@ -84,14 +136,36 @@ pub fn lz4_compress(data: &[u8], _acceleration: i32) -> Vec { result } +/// numcodecs' ceiling on a single buffer, mirrored in `src/lz4.ts`. +const MAX_DECODED_SIZE: usize = 0x7e00_0000; + +/// Max output bytes an LZ4 block can produce per input byte: each match-length +/// extension byte adds 255 to the copy. A constant buffer measures 255.0:1. +const LZ4_MAX_EXPANSION: usize = 256; + +/// The size in an LZ4 header is untrusted, and `lz4_flex` zero-fills a buffer +/// that large before it reads a single byte of the block — so a corrupt header +/// asking for 4 GB aborts the WASM instance rather than failing cleanly. Reject +/// sizes no LZ4 block could actually expand to. +fn lz4_declared_size_is_plausible(orig_size: usize, block_len: usize) -> bool { + orig_size <= MAX_DECODED_SIZE + && orig_size + <= block_len + .saturating_mul(LZ4_MAX_EXPANSION) + .saturating_add(1024) +} + #[wasm_bindgen] pub fn lz4_decompress(data: &[u8]) -> Vec { if data.len() < 4 { return vec![]; } - let orig_size = - u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; - lz4_flex::block::decompress(&data[4..], orig_size).unwrap_or_default() + let orig_size = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; + let block = &data[4..]; + if !lz4_declared_size_is_plausible(orig_size, block.len()) { + return vec![]; + } + lz4_flex::block::decompress(block, orig_size).unwrap_or_default() } // Standalone Zstd codec — standard Zstd frame format @@ -100,9 +174,30 @@ pub fn zstd_compress(data: &[u8], level: i32) -> Vec { zstd::bulk::compress(data, level).unwrap_or_default() } +/// Max output bytes a zstd frame can produce per input byte: a 4-byte RLE block +/// regenerates a full 128 KB block. A constant buffer measures ~32500:1. +const ZSTD_MAX_EXPANSION: usize = 131_072 / 4; + +/// Same hazard as the LZ4 header, one size class up: `zstd::bulk::decompress` +/// reserves the capacity we hand it up front, and that capacity comes straight +/// out of the frame's declared content size. +fn zstd_declared_size_is_plausible(declared: usize, frame_len: usize) -> bool { + declared <= MAX_DECODED_SIZE + && declared + <= frame_len + .saturating_mul(ZSTD_MAX_EXPANSION) + .saturating_add(131_072) +} + #[wasm_bindgen] pub fn zstd_decompress(data: &[u8]) -> Vec { - let capacity = zstd_frame_content_size(data).unwrap_or(data.len() * 4); + // Frames written by `zstd_compress` always declare their content size; the + // multiple-of-input fallback only covers frames from other producers that + // omit it, and will fail cleanly if it guesses low. + let capacity = zstd_frame_content_size(data).unwrap_or_else(|| data.len().saturating_mul(4)); + if !zstd_declared_size_is_plausible(capacity, data.len()) { + return vec![]; + } zstd::bulk::decompress(data, capacity).unwrap_or_default() } @@ -179,8 +274,404 @@ fn zstd_frame_content_size(data: &[u8]) -> Option { data[offset + 6], data[offset + 7], ]); - Some(val as usize) + // `usize` is 32-bit on wasm32, so a declared size past 4 GiB would + // wrap to a small plausible-looking one rather than being rejected. + usize::try_from(val).ok() } _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + + const COMPRESSORS: [&str; 6] = ["blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd"]; + + /// Mildly compressible bytes, so a round trip exercises a real codec path + /// rather than an all-zero fast path. + fn sample() -> Vec { + (0..8192u32) + .flat_map(|i| (i as u16).to_le_bytes()) + .collect() + } + + // The TypeScript layer always passes typesize 1; see `src/blosc.ts`. + fn compress_blosc(data: &[u8], cname: &str, clevel: u8, shuffle: i32) -> Vec { + blosc_compress(data, cname, clevel, shuffle, 1, 0) + } + + #[test] + fn cname_maps_to_blosc_compcode() { + assert_eq!(cname_to_compcode("lz4"), BLOSC_LZ4); + assert_eq!(cname_to_compcode("lz4hc"), BLOSC_LZ4HC); + assert_eq!(cname_to_compcode("snappy"), BLOSC_SNAPPY); + assert_eq!(cname_to_compcode("zlib"), BLOSC_ZLIB); + assert_eq!(cname_to_compcode("zstd"), BLOSC_ZSTD); + assert_eq!(cname_to_compcode("blosclz"), BLOSC_BLOSCLZ); + // numcodecs validates the name in JS; an unknown one still must not panic. + assert_eq!(cname_to_compcode("not-a-compressor"), BLOSC_BLOSCLZ); + } + + #[test] + fn blosc_round_trips_every_compressor() { + let data = sample(); + for cname in COMPRESSORS { + let compressed = compress_blosc(&data, cname, 5, BLOSC_SHUFFLE as i32); + assert!(!compressed.is_empty(), "{cname} produced no output"); + assert_eq!(blosc_decompress(&compressed), data, "{cname} round trip"); + } + } + + #[test] + fn blosc_round_trips_every_shuffle_mode() { + let data = sample(); + let modes = [ + BLOSC_NOSHUFFLE as i32, + BLOSC_SHUFFLE as i32, + BLOSC_BITSHUFFLE as i32, + ]; + for shuffle in modes { + let compressed = compress_blosc(&data, "lz4", 5, shuffle); + assert!( + !compressed.is_empty(), + "shuffle {shuffle} produced no output" + ); + assert_eq!(blosc_decompress(&compressed), data, "shuffle {shuffle}"); + } + } + + #[test] + fn blosc_round_trips_at_clevel_zero() { + // clevel 0 stores the buffer uncompressed but still emits a Blosc frame. + let data = sample(); + let compressed = compress_blosc(&data, "lz4", 0, BLOSC_NOSHUFFLE as i32); + assert!(!compressed.is_empty()); + assert_eq!(blosc_decompress(&compressed), data); + } + + #[test] + fn blosc_round_trips_an_empty_buffer() { + let compressed = compress_blosc(&[], "lz4", 5, BLOSC_SHUFFLE as i32); + // An empty input may legitimately yield no output; what must not happen + // is a panic across the wasm_bindgen boundary. + assert!(blosc_decompress(&compressed).is_empty()); + } + + #[test] + fn blosc_header_is_self_describing() { + // Blosc1-compatible framing: the header carries the sizes back out, which + // is what lets Python numcodecs read a chunk we wrote. + let data = sample(); + let compressed = compress_blosc(&data, "zstd", 5, BLOSC_SHUFFLE as i32); + let sizes = blosc_cbuffer_sizes(&compressed); + assert_eq!(sizes.len(), 3); + assert_eq!(sizes[0] as usize, data.len(), "nbytes"); + assert_eq!(sizes[1] as usize, compressed.len(), "cbytes"); + assert!(sizes[2] > 0, "blocksize"); + } + + #[test] + fn blosc_decompress_returns_empty_on_garbage() { + // The error convention across the boundary is an empty Vec, never a panic; + // src/blosc.ts turns that into a thrown Error. A panic here would trap the + // WASM instance, so a single corrupt Zarr chunk would break every later read. + assert!(blosc_decompress(&[]).is_empty(), "empty"); + assert!( + blosc_decompress(&[0u8; 3]).is_empty(), + "shorter than a header" + ); + assert!(blosc_decompress(&[0u8; 64]).is_empty(), "zeroed header"); + assert!(blosc_decompress(&[0xff; 64]).is_empty(), "all-ones header"); + assert!( + blosc_decompress(&[0x02; 64]).is_empty(), + "plausible version byte" + ); + } + + #[test] + fn blosc_decompress_rejects_a_lying_header() { + let data = sample(); + let compressed = compress_blosc(&data, "lz4", 5, BLOSC_SHUFFLE as i32); + + // cbytes larger than the buffer we were actually handed. + let mut overlong = compressed.clone(); + overlong[12..16].copy_from_slice(&(compressed.len() as u32 + 1).to_le_bytes()); + assert!( + blosc_decompress(&overlong).is_empty(), + "cbytes past the buffer" + ); + + // A memcpy'd frame promising far more payload than it carries. This is the + // bound blusc skips before copying, and the shape that used to panic. + let mut memcpyed = compressed.clone(); + memcpyed[2] |= BLOSC_MEMCPYED; + memcpyed[4..8].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!( + blosc_decompress(&memcpyed).is_empty(), + "memcpy past the buffer" + ); + + // Truncation must not be mistaken for a decodable frame. + assert!( + blosc_decompress(&compressed[..compressed.len() / 2]).is_empty(), + "truncated" + ); + } + + #[test] + fn blosc_header_check_accepts_what_we_emit() { + // The guard must not reject our own frames, in any of the shapes we emit: + // compressed, stored at clevel 0 (which sets BLOSC_MEMCPYED), and empty. + let data = sample(); + for cname in COMPRESSORS { + let compressed = compress_blosc(&data, cname, 5, BLOSC_SHUFFLE as i32); + assert!(blosc_header_is_sane(&compressed), "{cname}"); + } + let stored = compress_blosc(&data, "lz4", 0, BLOSC_NOSHUFFLE as i32); + assert!(blosc_header_is_sane(&stored), "clevel 0"); + assert_ne!(stored[2] & BLOSC_MEMCPYED, 0, "clevel 0 should be memcpy'd"); + } + + #[test] + fn lz4_uses_numcodecs_framing() { + // 4-byte little-endian original size, then a raw LZ4 block. Changing this + // silently would break interchange with Python numcodecs. + let data = sample(); + let compressed = lz4_compress(&data, 1); + assert!(compressed.len() > 4); + assert_eq!(&compressed[..4], &(data.len() as u32).to_le_bytes()); + + let block = lz4_flex::block::decompress(&compressed[4..], data.len()).unwrap(); + assert_eq!(block, data); + } + + #[test] + fn lz4_round_trips() { + let data = sample(); + assert_eq!(lz4_decompress(&lz4_compress(&data, 1)), data); + } + + #[test] + fn lz4_round_trips_an_empty_buffer() { + let compressed = lz4_compress(&[], 1); + assert_eq!(&compressed[..4], &0u32.to_le_bytes()); + assert!(lz4_decompress(&compressed).is_empty()); + } + + #[test] + fn lz4_decompress_returns_empty_on_garbage() { + assert!(lz4_decompress(&[]).is_empty(), "no header"); + assert!(lz4_decompress(&[0, 0, 0]).is_empty(), "truncated header"); + // Well-formed header, unreadable block. + let mut bad = 32u32.to_le_bytes().to_vec(); + bad.extend_from_slice(&[0xff; 16]); + assert!(lz4_decompress(&bad).is_empty(), "corrupt block"); + } + + #[test] + fn lz4_rejects_an_implausible_declared_size() { + // A declared size is a request to zero-fill that many bytes before the + // block is even read, so it has to be bounded by what the block could + // possibly expand to. 4 GB from an 8-byte chunk is an OOM, not an error. + assert!( + !lz4_declared_size_is_plausible(u32::MAX as usize, 8), + "u32::MAX" + ); + assert!( + !lz4_declared_size_is_plausible(MAX_DECODED_SIZE + 1, usize::MAX), + "over cap" + ); + assert!(!lz4_declared_size_is_plausible(16 << 20, 16), "past 255:1"); + + let mut hostile = u32::MAX.to_le_bytes().to_vec(); + hostile.extend_from_slice(&[0u8; 8]); + assert!(lz4_decompress(&hostile).is_empty()); + } + + #[test] + fn lz4_accepts_the_maximum_real_expansion() { + // The guard must not reject genuinely well-compressed data. A constant + // buffer sits right on the format's 255:1 ceiling, so if anything real + // trips this bound, it is this. + let data = vec![0u8; 1 << 20]; + let compressed = lz4_compress(&data, 1); + let ratio = data.len() / (compressed.len() - 4); + assert!(ratio >= 250, "expected a near-ceiling ratio, got {ratio}:1"); + assert!(lz4_declared_size_is_plausible( + data.len(), + compressed.len() - 4 + )); + assert_eq!(lz4_decompress(&compressed), data); + } + + #[test] + fn zstd_emits_a_standard_frame() { + let data = sample(); + let compressed = zstd_compress(&data, 1); + assert!(compressed.len() > 4); + assert_eq!( + u32::from_le_bytes([compressed[0], compressed[1], compressed[2], compressed[3]]), + 0xFD2FB528, + "zstd frame magic" + ); + } + + #[test] + fn zstd_round_trips_across_levels() { + let data = sample(); + for level in [1, 10, 22] { + let compressed = zstd_compress(&data, level); + assert!(!compressed.is_empty(), "level {level} produced no output"); + assert_eq!(zstd_decompress(&compressed), data, "level {level}"); + } + } + + #[test] + fn zstd_round_trips_highly_compressible_data() { + // Guards the `data.len() * 4` capacity fallback in zstd_decompress: this + // buffer compresses far past 4:1, so decoding only fits if the frame's + // declared content size is used instead. + let data = vec![7u8; 1 << 16]; + assert_eq!(zstd_decompress(&zstd_compress(&data, 1)), data); + } + + #[test] + fn zstd_decompress_returns_empty_on_garbage() { + assert!(zstd_decompress(&[]).is_empty()); + assert!(zstd_decompress(&[0xff; 64]).is_empty()); + } + + #[test] + fn zstd_rejects_an_implausible_declared_size() { + assert!( + !zstd_declared_size_is_plausible(usize::MAX, 64), + "usize::MAX" + ); + assert!( + !zstd_declared_size_is_plausible(MAX_DECODED_SIZE + 1, usize::MAX), + "over cap" + ); + assert!( + !zstd_declared_size_is_plausible(1 << 30, 16), + "past 32768:1" + ); + + // A frame header is enough to trigger the allocation; the block never + // has to be readable, so the guard has to run before decompression. + let mut hostile = frame_header(0xE0, &[0u8; 8]); + hostile[5..13].copy_from_slice(&u64::MAX.to_le_bytes()); + assert!(zstd_decompress(&hostile).is_empty()); + } + + #[test] + fn zstd_accepts_the_maximum_real_expansion() { + // 16 MB of one byte lands near the format's 32768:1 RLE ceiling, so this + // is the shape most likely to be rejected by an over-tight bound. + let data = vec![0u8; 16 << 20]; + let compressed = zstd_compress(&data, 1); + let ratio = data.len() / compressed.len(); + assert!( + ratio >= 30_000, + "expected a near-ceiling ratio, got {ratio}:1" + ); + assert!(zstd_declared_size_is_plausible( + data.len(), + compressed.len() + )); + assert_eq!(zstd_decompress(&compressed), data); + } + + /// Builds a synthetic zstd frame header: magic, frame descriptor, then the + /// trailing bytes the descriptor says to expect. + fn frame_header(descriptor: u8, rest: &[u8]) -> Vec { + let mut frame = vec![0x28, 0xB5, 0x2F, 0xFD, descriptor]; + frame.extend_from_slice(rest); + frame + } + + #[test] + fn frame_content_size_rejects_non_zstd_input() { + assert_eq!(zstd_frame_content_size(&[]), None, "empty"); + assert_eq!( + zstd_frame_content_size(&[0x28, 0xB5, 0x2F, 0xFD]), + None, + "no descriptor" + ); + assert_eq!(zstd_frame_content_size(&[0u8; 16]), None, "wrong magic"); + } + + #[test] + fn frame_content_size_reads_single_byte_field() { + // fcs_flag 0 + single_segment: one byte of size, no window descriptor. + assert_eq!( + zstd_frame_content_size(&frame_header(0x20, &[0x42])), + Some(0x42) + ); + // fcs_flag 0 without single_segment means the field is absent entirely. + assert_eq!(zstd_frame_content_size(&frame_header(0x00, &[0x42])), None); + } + + #[test] + fn frame_content_size_reads_two_byte_field() { + // fcs_flag 1: u16 biased by 256, and a window descriptor byte to skip. + let frame = frame_header(0x40, &[0x00, 0x10, 0x00]); + assert_eq!(zstd_frame_content_size(&frame), Some(16 + 256)); + } + + #[test] + fn frame_content_size_skips_the_dictionary_id() { + // fcs_flag 2 + single_segment + 1-byte dict id. + let frame = frame_header(0xA1, &[0x07, 0x00, 0x10, 0x00, 0x00]); + assert_eq!(zstd_frame_content_size(&frame), Some(0x1000)); + // fcs_flag 3 + single_segment + 4-byte dict id. + let mut rest = vec![0xDE, 0xAD, 0xBE, 0xEF]; + rest.extend_from_slice(&1_000_000u64.to_le_bytes()); + assert_eq!( + zstd_frame_content_size(&frame_header(0xE3, &rest)), + Some(1_000_000) + ); + } + + #[test] + fn frame_content_size_does_not_wrap_a_64_bit_field() { + // `usize` is 32-bit on wasm32, where an `as usize` cast would turn a + // declared size past 4 GiB into a small, plausible-looking one. + let frame = frame_header(0xE0, &u64::MAX.to_le_bytes()); + assert_eq!( + zstd_frame_content_size(&frame), + usize::try_from(u64::MAX).ok() + ); + } + + #[test] + fn frame_content_size_rejects_a_truncated_field() { + assert_eq!( + zstd_frame_content_size(&frame_header(0x20, &[])), + None, + "1-byte" + ); + assert_eq!( + zstd_frame_content_size(&frame_header(0x40, &[0x00, 0x10])), + None, + "2-byte" + ); + assert_eq!( + zstd_frame_content_size(&frame_header(0x80, &[0x00, 0x10])), + None, + "4-byte" + ); + assert_eq!( + zstd_frame_content_size(&frame_header(0xE0, &[0x00; 4])), + None, + "8-byte" + ); + } + + #[test] + fn frame_content_size_matches_a_real_frame() { + let data = sample(); + let compressed = zstd_compress(&data, 1); + assert_eq!(zstd_frame_content_size(&compressed), Some(data.len())); + } +} From 7a90b0e41286840ae8151c5016323e079b4be427 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Tue, 4 Aug 2026 18:03:07 -0400 Subject: [PATCH 2/7] test: pin numcodecs interop with fixtures written by Python The round-trip suites cannot show that this package is format-compatible with numcodecs. Encode and decode could both drift off the format together and every assertion would still pass, because nothing in CI read a buffer this package did not also write. test/fixtures now holds chunks encoded by Python numcodecs 0.16.5, and interop.test.ts decodes each one and compares it to the source array. Coverage is every codec this package exports plus four Blosc compressor/shuffle combinations, including bitshuffle - the filter most likely to break silently, since a wrong element width still yields output of exactly the right length. Each codec is built from the get_config() numcodecs itself recorded, which is the path a Zarr reader takes from stored chunk metadata rather than a hand-written config that could drift from it. A final test asserts the fixture set covers every exported codec, so a new codec cannot land without one. Only the decode direction is pinned. Encoded output is deliberately not compared byte for byte: a compressor may emit different bytes across versions while staying readable, so that assertion would fail for reasons that say nothing about compatibility. fixtures/generate.py regenerates the set through `uv run --with numcodecs`, so reproducing them needs no local install. fixtures/README.md records why refreshing existing fixtures to clear a failure defeats the purpose - it discards the evidence of the break. Verified by mutation: flipping one byte in lz4.bin fails that fixture's test and nothing else. Also covers two contracts the codebase states but never asserted: - decode(data, out) writing into the caller's buffer and returning that same object, for the three WASM-backed codecs. GZip and Zlib return fflate's own buffer instead; that matches numcodecs.js, so it is pinned as deliberate rather than corrected. - the package.json export map agreeing with both src/index.ts and the vite entry list. A codec missing from vite.config.ts advertises a subpath that resolves to a file that was never built, which otherwise only surfaces after publishing. Suite goes 24 -> 40 tests. Co-Authored-By: Claude Opus 5 (1M context) --- test/compat.test.ts | 44 ++++++++++++ test/fixtures/README.md | 32 +++++++++ test/fixtures/blosc-blosclz-noshuffle.bin | Bin 0 -> 528 bytes test/fixtures/blosc-lz4-shuffle.bin | Bin 0 -> 528 bytes test/fixtures/blosc-zlib-shuffle.bin | Bin 0 -> 377 bytes test/fixtures/blosc-zstd-bitshuffle.bin | Bin 0 -> 73 bytes test/fixtures/fixtures.json | 79 ++++++++++++++++++++++ test/fixtures/generate.py | 63 +++++++++++++++++ test/fixtures/gzip.bin | Bin 0 -> 365 bytes test/fixtures/lz4.bin | Bin 0 -> 519 bytes test/fixtures/source.u2.bin | Bin 0 -> 512 bytes test/fixtures/zlib.bin | Bin 0 -> 353 bytes test/fixtures/zstd.bin | Bin 0 -> 353 bytes test/index.test.ts | 30 ++++++++ test/interop.test.ts | 49 ++++++++++++++ 15 files changed, 297 insertions(+) create mode 100644 test/fixtures/README.md create mode 100644 test/fixtures/blosc-blosclz-noshuffle.bin create mode 100644 test/fixtures/blosc-lz4-shuffle.bin create mode 100644 test/fixtures/blosc-zlib-shuffle.bin create mode 100644 test/fixtures/blosc-zstd-bitshuffle.bin create mode 100644 test/fixtures/fixtures.json create mode 100644 test/fixtures/generate.py create mode 100644 test/fixtures/gzip.bin create mode 100644 test/fixtures/lz4.bin create mode 100644 test/fixtures/source.u2.bin create mode 100644 test/fixtures/zlib.bin create mode 100644 test/fixtures/zstd.bin create mode 100644 test/interop.test.ts diff --git a/test/compat.test.ts b/test/compat.test.ts index fccfa44..8a5a533 100644 --- a/test/compat.test.ts +++ b/test/compat.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from "vitest"; import { Blosc, GZip, Zlib, LZ4, Zstd } from "../src/index.js"; +import { range } from "./helpers.js"; describe("numcodecs compatibility", () => { it("Blosc has all numcodecs static members", () => { @@ -39,4 +40,47 @@ describe("numcodecs compatibility", () => { expect(Zstd.DEFAULT_CLEVEL).toBe(1); expect(Zstd.MAX_CLEVEL).toBe(22); }); + + it("LZ4 and Zstd fromConfig round-trip their options", () => { + const l = LZ4.fromConfig({ id: "lz4", acceleration: 10 }); + expect((l as any).acceleration).toBe(10); + const z = Zstd.fromConfig({ id: "zstd", level: 9 }); + expect((z as any).level).toBe(9); + }); + + describe("decode(data, out)", () => { + const arr = range(1000, " { + const encoded = await codec.encode(bytes); + const out = new Uint8Array(bytes.length); + const decoded = await codec.decode(encoded, out); + expect(decoded).toBe(out); + expect(Array.from(out)).toEqual(Array.from(bytes)); + }); + } + + // fflate hands back its own buffer; numcodecs.js does the same, so consumers + // that pass `out` to GZip/Zlib must keep using the return value. + for (const [name, codec] of [ + ["GZip", GZip.fromConfig({ id: "gzip" })], + ["Zlib", Zlib.fromConfig({ id: "zlib" })], + ] as const) { + it(`${name} still returns the decoded bytes when given an out buffer`, async () => { + const encoded = await codec.encode(bytes); + const decoded = await codec.decode(encoded, new Uint8Array(bytes.length)); + expect(Array.from(decoded)).toEqual(Array.from(bytes)); + }); + } + }); }); diff --git a/test/fixtures/README.md b/test/fixtures/README.md new file mode 100644 index 0000000..369ad0a --- /dev/null +++ b/test/fixtures/README.md @@ -0,0 +1,32 @@ +# numcodecs interop fixtures + +Compressed buffers produced by **Python** `numcodecs`, used by +[`../interop.test.ts`](../interop.test.ts) to prove that rumcodecs decodes what +the reference implementation writes. The round-trip suites cannot show this: +they would keep passing if the encoder and decoder drifted off the numcodecs +format together. + +- `source.u2.bin` — the uncompressed input every fixture decodes back to: + `numpy.arange(256, dtype=".bin` — that input encoded by the codec described under `` in + `fixtures.json`. +- `fixtures.json` — the exact `get_config()` for each codec, plus the + `numcodecs` version that wrote them. The test builds each codec from this + config, the way a Zarr reader builds one from stored chunk metadata. + +Only the decode direction is pinned here. Encoded output is not compared byte +for byte, because a compressor is free to emit different bytes across versions +while staying readable — what has to hold is that both sides read each other. + +## Regenerating + +Requires no local install; `uv` fetches `numcodecs` on demand: + +```sh +cd test/fixtures +uv run --with numcodecs python generate.py +``` + +Regenerate only to add a codec or a new option combination. Refreshing the +existing files against a newer `numcodecs` weakens the test — it would hide a +format break by replacing the fixture that should have caught it. diff --git a/test/fixtures/blosc-blosclz-noshuffle.bin b/test/fixtures/blosc-blosclz-noshuffle.bin new file mode 100644 index 0000000000000000000000000000000000000000..d6835f8b5b492b223c87a99c0e41e2f483309ca4 GIT binary patch literal 528 zcmWm1W3&(k0D#fk%YNCeW!v_)Y}>YNyVl~CZQHhO+wQy{-?`@t2o?}b06_%?DQJTW zA*4`33nQ#>!iylHNFoaqMO4v57eh?3#1=4(n=@2 z3^K|jvn;a8Cc7MR$|biv^2#T_0tzamup){ornnMHDy6hC%KFn^{#H(T6;xD7WmQyF zO?5TYR7-7j)KyP?4K&n9V@)*GOmi)?)Jkh@wAD^~9dy)5XI*sFO?N%?)Jt!D^wm#) z0}M3CU_%Tw%y1)&G|FgWj5W@96HGM8WK&Ev&2%&TW2S%2GTR(;%`@MB{Gw#HiPthd2Nn{2klR@-d1!%n;Gw#Q!k?03LHha7gqQO6v2!bzu`cE(xf zoOi)Rmt1zmRo7g1!%er`cE?@!-1opkk39CoQ_np2!b`8b_QqT9y!XLJpM3VkSKoa1 I!%x5b29(Wzod5s; literal 0 HcmV?d00001 diff --git a/test/fixtures/blosc-lz4-shuffle.bin b/test/fixtures/blosc-lz4-shuffle.bin new file mode 100644 index 0000000000000000000000000000000000000000..6d51abb28fe6b89eeca431b041480ac8151830f6 GIT binary patch literal 528 zcmWm1W3&(k0D#f^Ubbz!W!tW`Y}>YN+qG)h_R^MZ+cwYp@tu3Vz<^`{0tqT|kb*Xt z;6ex~l+eNmE1d8mh$xcCqKGP*=wgT|me}HmE1vigNGOrSl1M6<^$Y_-jHJM6T}ZhP#t&wdB|>!3ppJL0JS{O_3KPB`h5)6O{S zobxWY=#tB>xaykgZn)`|+wQpQp8Fnn=#j^scckMElS?NAX{}s3G0f^^>V!bG znW+;)&2B!O5UjWJ>BMlmpP~T)imjr7A%;h%1_Ws?of;Ts`7|^jP<3i(V5sTUs{z5f zTdxL&+kQ2@5TM*^dNIWK?A8lG+H1F746}Y)dLdAKZt2BP^SfU!1nckpdNJJoFR#G` zg=SvED+Y&q4K8Uc?lrt-@z~ekqRM1n!>cBj&l+6T*?iXUy3J?Z4HuNUbvIryI=yzo zC9TzKH(s-Pox9Git>mvy(_Exc~~-S)!;<$l|bSB%f^{cuTp{oaq)tl!suxTrqA_TyFa V`+q-N*5Cj4<8}N0jM2aA8v$DivX}q> literal 0 HcmV?d00001 diff --git a/test/fixtures/blosc-zstd-bitshuffle.bin b/test/fixtures/blosc-zstd-bitshuffle.bin new file mode 100644 index 0000000000000000000000000000000000000000..5bdcdecca2c65f6225a4ccb90209a6a65321469f GIT binary patch literal 73 zcmZQ#oWjV!#J~W;o`Y W!F7t;l!Dgmn4I9p)%ZkZ12X`t(GbA^ literal 0 HcmV?d00001 diff --git a/test/fixtures/fixtures.json b/test/fixtures/fixtures.json new file mode 100644 index 0000000..69588ca --- /dev/null +++ b/test/fixtures/fixtures.json @@ -0,0 +1,79 @@ +{ + "numcodecsVersion": "0.16.5", + "source": { + "dtype": " None: + here = pathlib.Path(__file__).parent + raw = SOURCE.tobytes() + (here / "source.u2.bin").write_bytes(raw) + + manifest = {} + for name, codec in CODECS.items(): + encoded = bytes(codec.encode(raw)) + (here / f"{name}.bin").write_bytes(encoded) + manifest[name] = {"config": codec.get_config(), "bytes": len(encoded)} + print(f"{name:26} {len(encoded):5} bytes") + + (here / "fixtures.json").write_text( + json.dumps( + { + "numcodecsVersion": numcodecs.__version__, + "source": { + "dtype": "J005x2ZQHhO+qP}nwr$(CZQHg{0`mUfM*;{akidcnDwyCx2q~1%!U!vz z@FIvPlE|WnDw^nGh$)uX;)pAr_!3Ack;IZnDw*U`NGX-n(nu?v^fJgOlgzTnDx2(b z$SIfH^2jTn{0b6s+i(RD5;dv$|$Rx@+zpPlFF*6s+#I*sHv9P>Zq%p`Wk4c zk;a;6s+s0mXsMOf+Gwkt_B!aOlg_&6s+;b5=&6_9`sk~l{stImkimu+YM9|h7-^Kz z#u#gy@g|sPlF6o+YMSY0m}!>T=9p`q`4(7ck;Rr+YMJF$SZS5j)>vzu^)}dOlg+l+ zYMbqL*lCyD_SkEm{SG+jki(8R>X_qBIO&wr&N%Cw^DemPlFP2R>YD3rxapSL?zroo s`yP1ck;k5R>Y3+WcYML=`01D5{`l*k|0pPeWB>pF literal 0 HcmV?d00001 diff --git a/test/fixtures/source.u2.bin b/test/fixtures/source.u2.bin new file mode 100644 index 0000000000000000000000000000000000000000..a0d7369e4551faa649af1ab1e5ee00f5d5e7d60f GIT binary patch literal 512 zcmV~$0{{>J005x2ZQHhO+qP}nwr$(CZQHg{1Q1Xlfdvs%Fu{coQYfK?5mq?iMG#RW zkwps(N;U{b7(kP>iG1fTa zO)$|UlT9(zG}Fy6(=4;iG1olvEwIoci!HI#GRv*7(kiR1vDP~4ZLrZMn{BbxHrws6 z(=NO1vDZHP9dOVghaGX$F~^;7(kZ8%an?EKU2xGQmtAqyHP_v6(=E5%ao0WfJ@C*Y ok3I3!Gta&7(krjM@zy)_d4?E;=Lk%;;@InhKq;NtBBZTll2ODH?K?M^; z@IV6#Byd0h0|fBz`{Z*^J@dr#PCM(Qb51$qg!4@|+hlW1HPb}%Of$!o*IdE + typeof v === "function" && typeof (v as any).codecId === "string", +); describe("rumcodecs exports", () => { it("exports all codec classes", () => { @@ -25,4 +34,25 @@ describe("rumcodecs exports", () => { expect(typeof rumcodecs.LZ4.fromConfig).toBe("function"); expect(typeof rumcodecs.Zstd.fromConfig).toBe("function"); }); + + // Adding a codec means touching src/index.ts, the package.json export map and + // the vite entry list together. Miss either one and the advertised subpath + // resolves to a file that was never built, which only shows up once published. + it("every codec has a matching subpath export", () => { + expect(CODECS.length).toBe(5); + for (const { codecId } of CODECS) { + expect(pkg.exports[`./${codecId}`]).toEqual({ + types: `./dist/${codecId}.d.ts`, + import: `./dist/${codecId}.js`, + }); + } + }); + + it("every subpath export is built by vite", () => { + const entries = (viteConfig as any).build.lib.entry; + for (const { codecId } of CODECS) { + expect(Object.keys(entries)).toContain(codecId); + expect(entries[codecId]).toMatch(new RegExp(`src/${codecId}\\.ts$`)); + } + }); }); diff --git a/test/interop.test.ts b/test/interop.test.ts new file mode 100644 index 0000000..85badac --- /dev/null +++ b/test/interop.test.ts @@ -0,0 +1,49 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { Blosc, GZip, Zlib, LZ4, Zstd } from "../src/index.js"; +import type { Codec, CodecConstructor } from "../src/types.js"; + +// Buffers in ./fixtures were written by Python numcodecs, not by this package. +// They are the only thing standing behind the claim that a Zarr chunk written by +// Python decodes here byte for byte — a round-trip suite would keep passing even +// if both directions drifted off the numcodecs format together. +// See ./fixtures/README.md to regenerate them. +const fixture = (name: string) => + new Uint8Array(readFileSync(new URL(`./fixtures/${name}`, import.meta.url))); + +const manifest = JSON.parse( + readFileSync(new URL("./fixtures/fixtures.json", import.meta.url), "utf8"), +) as { + numcodecsVersion: string; + codecs: Record }>; +}; + +const REGISTRY: Record> = { + blosc: Blosc, + gzip: GZip, + zlib: Zlib, + lz4: LZ4, + zstd: Zstd, +}; + +describe(`decodes Python numcodecs ${manifest.numcodecsVersion} output`, () => { + const expected = Array.from(fixture("source.u2.bin")); + + for (const [name, { config }] of Object.entries(manifest.codecs)) { + it(name, async () => { + const constructor = REGISTRY[config.id]; + expect(constructor, `no codec registered for id '${config.id}'`).toBeDefined(); + + // Built from the config numcodecs itself recorded, the way a Zarr reader + // builds a codec out of the stored chunk metadata. + const codec: Codec = constructor.fromConfig(config as any); + const decoded = await codec.decode(fixture(`${name}.bin`)); + expect(Array.from(decoded)).toEqual(expected); + }); + } + + it("covers every codec this package exports", () => { + const ids = new Set(Object.values(manifest.codecs).map((c) => c.config.id)); + expect([...ids].sort()).toEqual(Object.keys(REGISTRY).sort()); + }); +}); From cd40507b5eae4df093f87067ac3327b497d5a66b Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Tue, 4 Aug 2026 18:03:21 -0400 Subject: [PATCH 3/7] docs: add README, contributor guide, and code of conduct Gives the repository a front door: what the package is, the five codecs and the backend behind each, install and quick start, how the WASM gets built, and a command table. AGENTS.md is the map for anyone working in the repo - module layout, what to read for a given task, and the conventions that are not visible from the code alone: the drop-in contract with numcodecs.js, the byte formats that must not change silently, the lazy WASM load that keeps the pure-JS codecs free of it, the empty-Vec error convention across wasm_bindgen, and the rule that sizes read out of a compressed header are untrusted. CODE_OF_CONDUCT.md is Builder's Code v1.0, dedicated to the public domain under CC0. Claims were checked against the code rather than assumed, which turned up four that would have shipped wrong: - "the round-trip suites assert the formats" - they assert symmetry, which survives both directions drifting off the format together. Now points at the interop fixtures, which do assert interchange. - "decode(data, out?) must honor the out buffer" - written as a blanket rule, but GZip and Zlib ignore out and hand back fflate's own buffer. That matches numcodecs.js exactly, so it is now documented as deliberate parity rather than quietly changed. - "a new codec touches four places" - it is five; the list omitted vite.config.ts, without which the advertised subpath export resolves to a file that was never built. - the command table advertised Rust unit tests for the WASM crate that did not exist. They do as of the fix two commits back, which is what made the claim worth keeping. The logo is a compressing-buffer motif in two variants, selected by GitHub's colour-scheme switch; both were rendered and checked rather than assumed to work. Note the paths are repo-relative, so the logo resolves on GitHub but not necessarily on npmjs.com, where docs/ is outside the published `files` allowlist. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 125 ++++++++++++++++++++++++ CODE_OF_CONDUCT.md | 37 +++++++ README.md | 144 ++++++++++++++++++++++++++++ docs/assets/rumcodecs-logo-dark.svg | 23 +++++ docs/assets/rumcodecs-logo.svg | 19 ++++ 5 files changed, 348 insertions(+) create mode 100644 AGENTS.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 README.md create mode 100644 docs/assets/rumcodecs-logo-dark.svg create mode 100644 docs/assets/rumcodecs-logo.svg diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..dd50191 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,125 @@ +# AGENTS.md — rumcodecs + +## Overview + +rumcodecs is a pnpm workspace providing **buffer compression codecs for +JavaScript** — a Rust/WASM reimplementation of +[numcodecs.js](https://github.com/manzt/numcodecs.js) built for Zarr. Five +codec classes are exported from one npm package, +`@fideus-labs/rumcodecs`: + +- **`Blosc`** — the Blosc meta-compressor via the pure-Rust + [blusc](https://crates.io/crates/blusc) crate, compiled to WASM (SIMD128). +- **`LZ4`** — [lz4_flex](https://crates.io/crates/lz4_flex) via WASM, using + numcodecs framing (4-byte little-endian original size + LZ4 block). +- **`Zstd`** — the [zstd](https://crates.io/crates/zstd) crate via WASM, + standard Zstd frames. +- **`GZip`** / **`Zlib`** — pure JS via + [fflate](https://github.com/101arrowz/fflate) (a peer dependency); no WASM + involved. + +The contract is **drop-in compatibility**: identical API surface to +numcodecs.js and identical byte formats to Python numcodecs, so chunks are +interchangeable across ecosystems. Three layers hold that up: +[`test/compat.test.ts`](./test/compat.test.ts) asserts the API surface, +[`test/interop.test.ts`](./test/interop.test.ts) decodes buffers actually +written by Python `numcodecs`, and the Rust unit tests in +[`lib.rs`](./crates/rumcodecs-wasm/src/lib.rs) pin the framing we emit. Note +what the per-codec suites do _not_ prove: they only show encode/decode symmetry, +which would survive both directions drifting off the format together. The +interop fixtures are the ones that fail when that happens. + +## Modules + +| Module | Read | +| ---------------------------------------------- | ------------------------------------------------------------------------ | +| TypeScript codec classes | [./src/](./src/) | +| Shared `Codec` / `CodecConstructor` interfaces | [./src/types.ts](./src/types.ts) | +| Rust WASM bindings (blusc, lz4_flex, zstd) | [./crates/rumcodecs-wasm/src/lib.rs](./crates/rumcodecs-wasm/src/lib.rs) | +| Test suites (round-trip, compat, interop) | [./test/](./test/) | +| Benchmark package (vs numcodecs.js) | [./benchmark/](./benchmark/) | +| CI / release workflows | [./.github/workflows/](./.github/workflows/) | + +## Context to load on demand + +| Task | Read | +| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Add or change a codec class | [./src/types.ts](./src/types.ts), the sibling codec in [./src/](./src/), its test in [./test/](./test/) | +| numcodecs API-compatibility rules | [./test/compat.test.ts](./test/compat.test.ts) | +| Byte-format / interop questions | [./test/interop.test.ts](./test/interop.test.ts), [./test/fixtures/README.md](./test/fixtures/README.md) | +| Change or add WASM exports | [./crates/rumcodecs-wasm/src/lib.rs](./crates/rumcodecs-wasm/src/lib.rs), the `build:wasm*` scripts in [./package.json](./package.json) | +| Run or extend benchmarks | [./benchmark/src/run.ts](./benchmark/src/run.ts), [./benchmark/src/data.ts](./benchmark/src/data.ts), [./benchmark/src/results.ts](./benchmark/src/results.ts) | +| CI matrix and gates | [./.github/workflows/ci.yml](./.github/workflows/ci.yml), [./.github/workflows/rust-ci.yml](./.github/workflows/rust-ci.yml) | +| Release / publish flow | [./.github/workflows/release.yml](./.github/workflows/release.yml) | + +## Conventions + +- Conventional Commits with optional scopes: `feat: …`, `fix(blosc): …`, + `ci: …`, `build: …` — match the existing `git log` style. +- Tooling is [vite-plus](https://www.npmjs.com/package/vite-plus) (`vp`): + `pnpm build` / `pnpm test` / `pnpm check` wrap `vp build` / `vp test` / + `vp check` (oxlint + oxfmt + typecheck). Keep `pnpm check` clean before + committing — CI runs it as a gate. +- **Every codec class is a drop-in for numcodecs.js**: static `codecId`, + static `fromConfig(config)`, the same constructor signature, and the same + static members (`Blosc.SHUFFLE`, `LZ4.max_buffer_size`, + `Zstd.MAX_CLEVEL`, …). Any new surface must be asserted in + [./test/compat.test.ts](./test/compat.test.ts). +- **Byte formats are Python-numcodecs compatible** and must never change + silently: LZ4 uses a 4-byte little-endian original-size header before the + block; Zstd emits standard frames; Blosc emits Blosc1-compatible frames + (`BLOSC_FORWARD_COMPAT_SPLIT`). A new codec or option combination needs a + fixture in [./test/fixtures/](./test/fixtures/) — regenerate with its + `generate.py`, and do not refresh existing fixtures to make a failure go + away, since that discards the evidence of the break. +- WASM-backed codecs load the module lazily through a cached dynamic import + (the `getModule()` pattern in [./src/blosc.ts](./src/blosc.ts)); new + WASM-backed codecs must reuse it so pure-JS codecs never pay the WASM load. +- `GZip` and `Zlib` stay pure-JS on fflate, and fflate stays a + `peerDependency` — do not move it to `dependencies` or reimplement it in + the WASM crate. +- The Rust functions return an empty `Vec` on failure; the TypeScript layer + converts that into a thrown `Error`. Keep error handling on that boundary — + **no panics across `wasm_bindgen`**, because a panic traps the WASM instance + and every later call fails with it, so one corrupt chunk breaks every + subsequent read. +- **Sizes read out of a compressed header are untrusted input.** Every decoder + we wrap trusts them to index or to size an allocation up front — `blusc` + indexes the source with them, `lz4_flex` zero-fills the declared size before + reading a byte, `zstd` reserves the declared content size — so a corrupt + header turns into an out-of-bounds panic or a multi-gigabyte allocation that + aborts the module. Validate before handing anything over: + `blosc_header_is_sane`, `lz4_declared_size_is_plausible`, + `zstd_declared_size_is_plausible`. The expansion bounds come from each + format's own ceiling (LZ4 255:1, zstd 32768:1) and are pinned by tests that + compress a constant buffer, which sits right on that ceiling — tighten them + and real data starts failing to decode. +- `decode(data, out?)` on the WASM-backed codecs must honor the optional `out` + buffer (write into it and return it) — zarr consumers rely on this. `GZip` + and `Zlib` ignore `out` and return fflate's own buffer, matching numcodecs.js + exactly; both behaviors are pinned in + [./test/compat.test.ts](./test/compat.test.ts). +- WASM builds require `RUSTFLAGS="-C target-feature=+simd128"` and build both + `web` and `nodejs` targets into `pkg/` (see the `build:wasm*` scripts); + `pkg/` is generated output, never hand-edited. +- A new codec touches five places: `src/.ts`, `src/index.ts`, the export + map in `package.json` (subpath export), the `build.lib.entry` list in + [./vite.config.ts](./vite.config.ts), and a `test/.test.ts` — plus the + codec table in [./README.md](./README.md). Miss the vite entry and the + advertised subpath resolves to a file that was never built; + [./test/index.test.ts](./test/index.test.ts) checks the three lists agree. +- Node ≥ 18 is supported (`engines`); CI tests Node 22 and 24. + +## Key dependencies + +| Dependency | Version | Notes | +| ------------------------- | --------------------------- | -------------------------------------------------------------------------- | +| `blusc` | 0.0.6 | Pure-Rust Blosc (Blosc2 API, Blosc1-compatible output) — the `Blosc` codec | +| `lz4_flex` | ~0.12 | Pure-Rust LZ4 block compression — the `LZ4` codec | +| `zstd` | ~0.13 (no default features) | Zstd — the `Zstd` codec | +| `wasm-bindgen` / `js-sys` | 0.2 / 0.3 | WASM boundary | +| `fflate` | ^0.8 (peer) | Pure-JS gzip/zlib — the `GZip` and `Zlib` codecs | +| `vite-plus` (`vp`) | 0.2 | Build / test / check driver | +| `vitest` | 4 | Test runner | +| `tinybench` | 6 | Benchmark timing (benchmark package) | +| `numcodecs` | 0.3 (dev) | Head-to-head baseline in [./benchmark/](./benchmark/) | diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f396188 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,37 @@ +# Builder's Code v1.0 + +A Code of Conduct for people who build things. + +--- + +## The Rule + +**Stay professional. Stay technical.** + +## Expected + +- Contribute constructively. +- Respect others' time and work. +- Focus on the work and its technical merit. + +## Not Welcome + +- Harassment, name-calling, or personal attacks. +- Trolling, spamming, or derailing discussions. +- Discussions about contributors rather than their contributions. + +## Enforcement + +Violations result in: + +1. **Warning** - First offense. +2. **Temporary suspension** - Repeated or serious violations. +3. **Permanent ban** - Continued violations. + +Maintainers can remove, block, or ban anyone who disrupts the project. + +--- + +## License + +This work is dedicated to the public domain under [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/). diff --git a/README.md b/README.md new file mode 100644 index 0000000..148fac0 --- /dev/null +++ b/README.md @@ -0,0 +1,144 @@ +

+ + + rumcodecs + +

+ +

rumcodecs

+ +

+ CI + Rust CI + npm + MIT License +

+ +

+ Buffer compression codecs for the browser and Node.js — numcodecs, reimplemented in Rust and WebAssembly. +

+ +

+ A drop-in replacement for + numcodecs.js: the same + codec classes with the same API and the same byte formats, backed by Rust + compiled to WASM (SIMD128) via + blusc. Built for + Zarr — chunks written by Python + numcodecs decode in the browser, and vice versa. +

+ +## 📦 The codecs + +| Codec | Backend | Format | +| ----------- | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| **`Blosc`** | [blusc](https://crates.io/crates/blusc) (Rust → WASM) | Blosc meta-compressor: `blosclz` / `lz4` / `lz4hc` / `snappy` / `zlib` / `zstd` with shuffle and bitshuffle filters | +| **`LZ4`** | [lz4_flex](https://crates.io/crates/lz4_flex) (Rust → WASM) | numcodecs framing — 4-byte little-endian original size + LZ4 block | +| **`Zstd`** | [zstd](https://crates.io/crates/zstd) (Rust → WASM) | Standard Zstd frames | +| **`GZip`** | [fflate](https://github.com/101arrowz/fflate) (pure JS) | gzip | +| **`Zlib`** | [fflate](https://github.com/101arrowz/fflate) (pure JS) | zlib | + +Every class carries the numcodecs.js surface — static `codecId`, static +`fromConfig`, the same constructor signatures and static members +(`Blosc.SHUFFLE`, `Zstd.MAX_CLEVEL`, …) — and +[`test/compat.test.ts`](test/compat.test.ts) asserts it stays that way. +Compatibility with the _Python_ side is checked against real chunks: the +buffers in [`test/fixtures/`](test/fixtures/) were written by Python +`numcodecs`, and [`test/interop.test.ts`](test/interop.test.ts) decodes every +one of them. The WASM module loads lazily on first encode/decode, so the +pure-JS codecs never pay for it. + +## 🚀 Quick start + +```sh +npm install @fideus-labs/rumcodecs +# fflate is a peer dependency, needed only for GZip and Zlib +npm install fflate +``` + +```ts +import { Blosc } from "@fideus-labs/rumcodecs"; + +const codec = new Blosc(5, "zstd", Blosc.SHUFFLE); +const compressed = await codec.encode(data); +const restored = await codec.decode(compressed); +``` + +Each codec is also a subpath export, so bundlers ship only what you use: + +```ts +import Blosc from "@fideus-labs/rumcodecs/blosc"; + +// The numcodecs config shape works unchanged — e.g. straight from Zarr metadata. +const codec = Blosc.fromConfig({ id: "blosc", cname: "lz4", clevel: 5, shuffle: 1 }); +``` + +## 📊 Benchmarks + +The [`benchmark/`](benchmark/) workspace package runs rumcodecs head-to-head +against numcodecs.js on the same generated arrays +([tinybench](https://github.com/tinylibs/tinybench), multiple sizes and +dtypes): + +```sh +pnpm bench # console table with speedups +pnpm bench:report # write JSON + markdown report +``` + +## 🛠️ Development + +### Prerequisites + +- Rust 1.91+ with [wasm-pack](https://rustwasm.github.io/wasm-pack/) +- Node 18+ (CI tests 22 and 24) and [pnpm](https://pnpm.io/) + +### Setup + +```sh +git clone https://github.com/fideus-labs/rumcodecs +cd rumcodecs +pnpm install # runs prepare: builds WASM (web + node) and the bundle +pnpm test +``` + +### Repository structure + +``` +rumcodecs/ +├── src/ # TypeScript codec classes (Blosc, GZip, Zlib, LZ4, Zstd) +├── crates/rumcodecs-wasm/ # Rust WASM bindings: blusc, lz4_flex, zstd +├── pkg/ # wasm-pack output (generated) +├── test/ # vitest suites: round-trip, numcodecs API, interop fixtures +├── benchmark/ # head-to-head benchmark vs numcodecs.js +└── .github/workflows/ # ci.yml, rust-ci.yml, release.yml +``` + +### Commands + +| Command | Purpose | +| ------------------------------------------ | ------------------------------------------------------------------------------------------- | +| `pnpm test` | Run all vitest suites (round-trip, per-codec, compat, interop) | +| `pnpm check` | Lint, format, and typecheck (oxlint / oxfmt / tsc via `vp`) | +| `pnpm build` | Bundle with [vite-plus](https://www.npmjs.com/package/vite-plus) and emit type declarations | +| `pnpm build:wasm` / `pnpm build:wasm:node` | Rebuild the WASM package for web / Node (SIMD128) | +| `pnpm bench` | Benchmark against numcodecs.js | +| `cargo test -- --test-threads=1` | Rust unit tests for the WASM crate | + +## 🤝 Contributing + +Start with [AGENTS.md](AGENTS.md) for the repository map and conventions, and +the [open issues](https://github.com/fideus-labs/rumcodecs/issues) for what to +pick up next. All participation is governed by our +[Code of Conduct](CODE_OF_CONDUCT.md). + +## 📄 License + +MIT — see [LICENSE.txt](LICENSE.txt). Copyright (c) Fideus Labs LLC. + +rumcodecs reimplements the API of +[numcodecs.js](https://github.com/manzt/numcodecs.js) (MIT, © Trevor Manz) so +existing Zarr tooling works unchanged. Blosc support comes from the pure-Rust +[blusc](https://crates.io/crates/blusc) implementation of the +[Blosc](https://www.blosc.org/) meta-compressor; LZ4 from +[lz4_flex](https://crates.io/crates/lz4_flex); Zstd from the +[zstd](https://crates.io/crates/zstd) crate. diff --git a/docs/assets/rumcodecs-logo-dark.svg b/docs/assets/rumcodecs-logo-dark.svg new file mode 100644 index 0000000..aacefee --- /dev/null +++ b/docs/assets/rumcodecs-logo-dark.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/rumcodecs-logo.svg b/docs/assets/rumcodecs-logo.svg new file mode 100644 index 0000000..2e3b303 --- /dev/null +++ b/docs/assets/rumcodecs-logo.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + From eacdb6a2d0f9e9cc80f8111e081f03d605969979 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Thu, 6 Aug 2026 10:35:12 -0400 Subject: [PATCH 4/7] fix(codecs): make decode failures distinguishable from empty frames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every decoder here reports failure the same way — an empty Vec, which the TypeScript layer turned into a thrown Error. A frame that legitimately holds zero bytes decodes to exactly the same thing, so the two were never actually distinguishable, and the failure mode was silent: with `out` supplied, a failed decode wrote nothing and handed the caller back their own untouched buffer, which reads as data rather than as an error. Each wrapper now compares the result against the length the frame itself declares — `blosc_declared_size`, the 4-byte LE header in src/lz4.ts, and `zstd_declared_content_size` (the latter two added here). Only a mismatch is an error. This also fixes a live bug in Blosc: `encode` of an empty buffer emits a valid 32-byte frame declaring nbytes=0, and `decode` of that frame threw "decompression failed" on a correct result. Reading the raw `blosc_cbuffer_sizes` would not have been enough, since it reports zeroes for garbage too — the sanity check has to run first. The scope of the check is truncation and short decodes. A Blosc1 frame carries no checksum over its payload, so corruption that still parses can decode to the declared length and be returned as valid; that limit is pinned in a test rather than left as an assumption. Separately, the size guards added earlier bound what a header may claim but not what the allocator can supply, and an infallible `vec![0; n]` turns that shortfall into `handle_alloc_error` — which aborts the module exactly like the panic this all exists to prevent. LZ4 and Zstd now reserve through `try_reserve_exact`. Co-Authored-By: Claude Opus 5 (1M context) --- crates/rumcodecs-wasm/src/lib.rs | 131 ++++++++++++++++++++++++++++++- src/blosc.ts | 8 +- src/lz4.ts | 12 +++ src/zstd.ts | 8 ++ test/compat.test.ts | 85 +++++++++++++++++++- 5 files changed, 240 insertions(+), 4 deletions(-) diff --git a/crates/rumcodecs-wasm/src/lib.rs b/crates/rumcodecs-wasm/src/lib.rs index eda6d89..34d0e3c 100644 --- a/crates/rumcodecs-wasm/src/lib.rs +++ b/crates/rumcodecs-wasm/src/lib.rs @@ -125,6 +125,24 @@ pub fn blosc_cbuffer_sizes(data: &[u8]) -> Vec { vec![nb as u32, cb as u32, bs as u32] } +/// The uncompressed size a well-formed Blosc frame declares, or `undefined` if +/// the header is not one we would decode. +/// +/// `blosc_decompress` signals failure with an empty `Vec`, and a frame that +/// really does hold zero bytes decodes to exactly the same thing — +/// `blosc_compress(&[])` emits a valid 32-byte frame declaring `nbytes = 0`. +/// `src/blosc.ts` uses this to tell them apart. Reading `blosc_cbuffer_sizes` +/// directly would not do: it reports zeroes for garbage too, which is the +/// answer that makes a corrupt chunk look like an empty one. +#[wasm_bindgen] +pub fn blosc_declared_size(data: &[u8]) -> Option { + if !blosc_header_is_sane(data) { + return None; + } + let (nbytes, _, _) = blusc_cbuffer_sizes(data); + Some(nbytes) +} + // Standalone LZ4 codec — numcodecs format: 4-byte LE original size + LZ4 block #[wasm_bindgen] pub fn lz4_compress(data: &[u8], _acceleration: i32) -> Vec { @@ -155,6 +173,21 @@ fn lz4_declared_size_is_plausible(orig_size: usize, block_len: usize) -> bool { .saturating_add(1024) } +/// A zeroed buffer of `len` bytes, or `None` if the allocator cannot supply one. +/// +/// The plausibility guards above reject sizes no real block could produce, but +/// what they cannot bound is how much memory this instance still has. An +/// infallible `vec![0; n]` turns that shortfall into `handle_alloc_error`, which +/// aborts the module exactly like a panic and takes every later decode with it — +/// so the reservation is made fallibly and a shortfall becomes an ordinary +/// decode failure. +fn try_zeroed(len: usize) -> Option> { + let mut buf = Vec::new(); + buf.try_reserve_exact(len).ok()?; + buf.resize(len, 0); + Some(buf) +} + #[wasm_bindgen] pub fn lz4_decompress(data: &[u8]) -> Vec { if data.len() < 4 { @@ -165,7 +198,15 @@ pub fn lz4_decompress(data: &[u8]) -> Vec { if !lz4_declared_size_is_plausible(orig_size, block.len()) { return vec![]; } - lz4_flex::block::decompress(block, orig_size).unwrap_or_default() + let Some(mut out) = try_zeroed(orig_size) else { + return vec![]; + }; + match lz4_flex::block::decompress_into(block, &mut out) { + // A block that decodes short of its declared size is corrupt; returning + // the zero-padded remainder would hand back silent garbage. + Ok(n) if n == orig_size => out, + _ => vec![], + } } // Standalone Zstd codec — standard Zstd frame format @@ -198,7 +239,32 @@ pub fn zstd_decompress(data: &[u8]) -> Vec { if !zstd_declared_size_is_plausible(capacity, data.len()) { return vec![]; } - zstd::bulk::decompress(data, capacity).unwrap_or_default() + // `zstd::bulk::decompress` would reserve `capacity` infallibly; reserve it + // ourselves so a request this instance cannot satisfy fails as a decode + // error rather than aborting the module. See `try_zeroed`. + let mut out = Vec::new(); + if out.try_reserve_exact(capacity).is_err() { + return vec![]; + } + match zstd::bulk::Decompressor::new() { + Ok(mut decoder) => match decoder.decompress_to_buffer(data, &mut out) { + Ok(_) => out, + Err(_) => vec![], + }, + Err(_) => vec![], + } +} + +/// The content size a zstd frame declares in its header, or `undefined` when it +/// declares none (or is not a zstd frame at all). +/// +/// `zstd_decompress` reports failure the same way every decoder here does — an +/// empty `Vec` — which on its own cannot be told apart from a frame that really +/// does hold nothing. `src/zstd.ts` uses this to make that call before deciding +/// whether to throw. Exposed for that reason, not as part of the codec API. +#[wasm_bindgen] +pub fn zstd_declared_content_size(data: &[u8]) -> Option { + zstd_frame_content_size(data) } fn zstd_frame_content_size(data: &[u8]) -> Option { @@ -674,4 +740,65 @@ mod tests { let compressed = zstd_compress(&data, 1); assert_eq!(zstd_frame_content_size(&compressed), Some(data.len())); } + + #[test] + fn blosc_declared_size_separates_an_empty_frame_from_a_bad_one() { + // The empty frame is a real frame: 32 bytes of header declaring nothing. + // Without this distinction src/blosc.ts reports it as a decode failure. + let empty = compress_blosc(&[], "lz4", 5, BLOSC_SHUFFLE as i32); + assert!(!empty.is_empty(), "empty input still yields a frame"); + assert_eq!(blosc_declared_size(&empty), Some(0)); + assert!(blosc_decompress(&empty).is_empty()); + + let data = sample(); + let compressed = compress_blosc(&data, "lz4", 5, BLOSC_SHUFFLE as i32); + assert_eq!(blosc_declared_size(&compressed), Some(data.len())); + + // Garbage must not read as "a frame holding zero bytes" — which is + // exactly what blosc_cbuffer_sizes reports for a zeroed buffer, and why + // the raw sizes cannot be used as the empty-vs-failed signal. + assert_eq!(blusc_cbuffer_sizes(&[0u8; 64]).0, 0, "the trap"); + assert_eq!(blosc_declared_size(&[0u8; 64]), None); + assert_eq!(blosc_declared_size(&[0xff; 64]), None); + assert_eq!(blosc_declared_size(&[]), None); + assert_eq!( + blosc_declared_size(&compressed[..compressed.len() / 2]), + None, + "truncated" + ); + } + + #[test] + fn declared_content_size_is_visible_to_the_wrapper() { + // src/zstd.ts needs this to tell a frame that holds nothing from one that + // failed to decode — both come back as an empty Vec. + let data = sample(); + assert_eq!( + zstd_declared_content_size(&zstd_compress(&data, 1)), + Some(data.len()) + ); + assert_eq!(zstd_declared_content_size(&zstd_compress(&[], 1)), Some(0)); + assert_eq!(zstd_declared_content_size(&[0xff; 64]), None, "not a frame"); + } + + #[test] + fn lz4_rejects_a_block_that_stops_short_of_its_declared_size() { + // Decoding into a preallocated buffer means a block that ends early + // leaves the tail zeroed. Handing that back would be silent corruption, + // so a short decode is a failure like any other. + let data = sample(); + let compressed = lz4_compress(&data, 1); + let truncated = &compressed[..compressed.len() - 8]; + assert!(lz4_decompress(truncated).is_empty()); + } + + #[test] + fn try_zeroed_reports_a_reservation_it_cannot_make() { + // The plausibility guards bound what a header may claim; this bounds what + // the allocator can actually supply. Without it the shortfall is an abort, + // which on wasm32 takes the whole instance down. + assert_eq!(try_zeroed(0).map(|b| b.len()), Some(0)); + assert_eq!(try_zeroed(64).map(|b| b.len()), Some(64)); + assert!(try_zeroed(usize::MAX).is_none()); + } } diff --git a/src/blosc.ts b/src/blosc.ts index 2e50fba..a1593a7 100644 --- a/src/blosc.ts +++ b/src/blosc.ts @@ -83,7 +83,13 @@ const Blosc: CodecConstructor = class Blosc implements Codec { async decode(data: Uint8Array, out?: Uint8Array): Promise { const m = await getModule(); const result = m.blosc_decompress(data); - if (result.length === 0) { + // The frame header states how many bytes it holds, so compare against that + // rather than treating any empty result as failure: `encode` of an empty + // buffer produces a valid frame declaring zero bytes, which decodes to + // nothing and is not an error. Checking the full length also catches a + // frame that decoded short of what it promised. + const declared = m.blosc_declared_size(data); + if (declared === undefined || result.length !== declared) { throw new Error("Blosc decompression failed"); } if (out !== undefined) { diff --git a/src/lz4.ts b/src/lz4.ts index 345f7c0..5133e73 100644 --- a/src/lz4.ts +++ b/src/lz4.ts @@ -48,8 +48,20 @@ const LZ4: CodecConstructor = class LZ4 implements Codec { if (data.length > MAX_BUFFER_SIZE) { throw Error(`Codec does not support buffers of > ${MAX_BUFFER_SIZE} bytes.`); } + // The header is the numcodecs framing: a 4-byte LE original size. It gives + // us the exact length a successful decode has to produce, which is what + // separates a genuine empty payload from the empty Vec the WASM side + // returns on failure. Without this check a corrupt chunk decodes to a + // zero-filled `out` and reaches the caller as data. + if (data.length < 4) { + throw new Error("LZ4 decompression failed: input is shorter than the size header"); + } + const declared = new DataView(data.buffer, data.byteOffset, 4).getUint32(0, true); const m = await getModule(); const result = m.lz4_decompress(data); + if (result.length !== declared) { + throw new Error("LZ4 decompression failed"); + } if (out !== undefined) { out.set(result); return out; diff --git a/src/zstd.ts b/src/zstd.ts index d008b1b..0653336 100644 --- a/src/zstd.ts +++ b/src/zstd.ts @@ -50,6 +50,14 @@ const Zstd: CodecConstructor = class Zstd implements Codec { async decode(data: Uint8Array, out?: Uint8Array): Promise { const m = await getModule(); const result = m.zstd_decompress(data); + // An empty result is how the WASM side reports failure, and a frame that + // really does hold nothing looks identical. The frame header settles it: + // only a frame declaring zero bytes may legitimately decode to nothing. + // A frame carrying no declared size at all (`undefined`) cannot be told + // apart, and erring toward an Error beats returning an untouched `out`. + if (result.length === 0 && m.zstd_declared_content_size(data) !== 0) { + throw new Error("Zstd decompression failed"); + } if (out !== undefined) { out.set(result); return out; diff --git a/test/compat.test.ts b/test/compat.test.ts index 8a5a533..ae9558a 100644 --- a/test/compat.test.ts +++ b/test/compat.test.ts @@ -78,8 +78,91 @@ describe("numcodecs compatibility", () => { ] as const) { it(`${name} still returns the decoded bytes when given an out buffer`, async () => { const encoded = await codec.encode(bytes); - const decoded = await codec.decode(encoded, new Uint8Array(bytes.length)); + const out = new Uint8Array(bytes.length); + const decoded = await codec.decode(encoded, out); + // Asserting the bytes alone would pass under either behavior. Pinning + // the identity is what documents which one callers actually get: the + // divergence from Blosc/LZ4/Zstd above is deliberate parity with + // numcodecs.js, so it should fail here if it ever changes silently. + expect(decoded).not.toBe(out); expect(Array.from(decoded)).toEqual(Array.from(bytes)); + expect(Array.from(out)).toEqual(Array.from(new Uint8Array(bytes.length))); + }); + } + }); + + // A decoder that reports failure by returning nothing is indistinguishable + // from one that succeeded on an empty payload — and with `out` supplied, the + // caller gets their own untouched buffer back and reads it as data. Corrupt + // chunks have to throw, in both call shapes. + describe("corrupt input", () => { + const corrupt = (encoded: Uint8Array) => { + const bad = new Uint8Array(encoded); + // Damage the payload, not the header: a header the guards reject is the + // easy case. This is the one that reaches the decompressor. + for (let i = Math.floor(bad.length / 2); i < bad.length; i++) { + bad[i] ^= 0xff; + } + return bad; + }; + + const arr = range(1000, " { + const encoded = await codec.encode(bytes); + const truncated = encoded.slice(0, Math.floor(encoded.length / 2)); + await expect(codec.decode(truncated)).rejects.toThrow(); + }); + } + + // Corruption *within* the payload is a different matter, and only LZ4 and + // Zstd are asserted here. A Blosc1 frame carries no checksum over its + // compressed body, so flipped bytes that still parse can decode to the + // declared length and be returned as valid data — see the note below. That + // is a property of the format, not something this layer can detect. + for (const [name, codec] of codecs.filter(([n]) => n !== "Blosc")) { + it(`${name} throws on a corrupt chunk`, async () => { + const bad = corrupt(await codec.encode(bytes)); + await expect(codec.decode(bad)).rejects.toThrow(); + }); + + it(`${name} throws on a corrupt chunk rather than returning an untouched out`, async () => { + const bad = corrupt(await codec.encode(bytes)); + const out = new Uint8Array(bytes.length); + await expect(codec.decode(bad, out)).rejects.toThrow(); + }); + } + + // Pinned so the limitation above is a recorded fact rather than an + // assumption: if blusc ever does start rejecting this, that is a behavior + // change worth noticing here rather than discovering in the field. + it("Blosc cannot detect payload corruption its format does not checksum", async () => { + const codec = Blosc.fromConfig({ id: "blosc", cname: "lz4", clevel: 5, shuffle: 1 }); + const bad = corrupt(await codec.encode(bytes)); + const decoded = await codec.decode(bad).catch(() => null); + if (decoded !== null) { + expect(decoded.length).toBe(bytes.length); + expect(Array.from(decoded)).not.toEqual(Array.from(bytes)); + } + }); + + // The flip side: an empty payload is a legitimate thing to encode, and must + // not be mistaken for the empty buffer a failed decode returns. + for (const [name, codec] of codecs) { + it(`${name} round-trips an empty buffer without reporting failure`, async () => { + const encoded = await codec.encode(new Uint8Array(0)); + const decoded = await codec.decode(encoded); + expect(decoded.length).toBe(0); }); } }); From c04b667cc1debf87fc5c750aca6ef2b2387ea774 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Thu, 6 Aug 2026 10:35:32 -0400 Subject: [PATCH 5/7] test: compare the codec lists as sets rather than one way MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both checks walked the source list and looked for a matching entry elsewhere. That catches a codec that was added and never exported, but not the reverse: a `./old` export and an `old` vite entry left behind after a codec is removed keep advertising a subpath nothing backs, and every one-way check still passes. Verified by adding exactly that pair — green before, red now. The interop registry had the same shape of gap for a different reason. It was written out by hand and compared against the fixture manifest, so a codec missing from both agreed with itself and passed. It is now derived from the package's own exports, which makes a new codec fail until it has a fixture. Both derivations are themselves pinned against the expected five ids, since a scan that silently found nothing would make every assertion built on it vacuously true. Co-Authored-By: Claude Opus 5 (1M context) --- test/index.test.ts | 30 ++++++++++++++++++++++++++---- test/interop.test.ts | 28 +++++++++++++++++++--------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/test/index.test.ts b/test/index.test.ts index f63bbcd..028da6e 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -38,9 +38,31 @@ describe("rumcodecs exports", () => { // Adding a codec means touching src/index.ts, the package.json export map and // the vite entry list together. Miss either one and the advertised subpath // resolves to a file that was never built, which only shows up once published. + // + // The three lists are compared as sets, in both directions. Walking only the + // source list catches a codec that was added and never exported, but not the + // reverse: a `./old` export and an `old` vite entry left behind after a codec + // is removed keep advertising a subpath nothing backs, and every one-way + // check still passes. + const sourceIds = CODECS.map((c) => c.codecId).sort(); + + // "." is the package root, not a codec, and `index` is its vite entry. + const exportedIds = Object.keys(pkg.exports) + .filter((k) => k !== ".") + .map((k) => k.replace(/^\.\//, "")) + .sort(); + const viteIds = Object.keys((viteConfig as any).build.lib.entry) + .filter((k) => k !== "index") + .sort(); + + it("the source, package export and vite entry lists name the same codecs", () => { + expect(sourceIds).toEqual(["blosc", "gzip", "lz4", "zlib", "zstd"]); + expect(exportedIds).toEqual(sourceIds); + expect(viteIds).toEqual(sourceIds); + }); + it("every codec has a matching subpath export", () => { - expect(CODECS.length).toBe(5); - for (const { codecId } of CODECS) { + for (const codecId of sourceIds) { expect(pkg.exports[`./${codecId}`]).toEqual({ types: `./dist/${codecId}.d.ts`, import: `./dist/${codecId}.js`, @@ -50,9 +72,9 @@ describe("rumcodecs exports", () => { it("every subpath export is built by vite", () => { const entries = (viteConfig as any).build.lib.entry; - for (const { codecId } of CODECS) { - expect(Object.keys(entries)).toContain(codecId); + for (const codecId of sourceIds) { expect(entries[codecId]).toMatch(new RegExp(`src/${codecId}\\.ts$`)); } + expect(entries.index).toMatch(/src\/index\.ts$/); }); }); diff --git a/test/interop.test.ts b/test/interop.test.ts index 85badac..8165e46 100644 --- a/test/interop.test.ts +++ b/test/interop.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; import { readFileSync } from "node:fs"; -import { Blosc, GZip, Zlib, LZ4, Zstd } from "../src/index.js"; +import * as rumcodecs from "../src/index.js"; import type { Codec, CodecConstructor } from "../src/types.js"; // Buffers in ./fixtures were written by Python numcodecs, not by this package. @@ -18,13 +18,19 @@ const manifest = JSON.parse( codecs: Record }>; }; -const REGISTRY: Record> = { - blosc: Blosc, - gzip: GZip, - zlib: Zlib, - lz4: LZ4, - zstd: Zstd, -}; +// Derived from the package's own exports rather than written out by hand. A +// hand-kept list is only ever as complete as the last person to edit it: add a +// codec, forget both this list and a fixture, and the coverage check below +// compares two lists that are wrong in the same way and passes. Reading the +// exports means a new codec has to be given a fixture or fail here. +const REGISTRY: Record> = Object.fromEntries( + Object.values(rumcodecs) + .filter( + (v): v is CodecConstructor => + typeof v === "function" && typeof (v as any).codecId === "string", + ) + .map((c) => [(c as any).codecId as string, c]), +); describe(`decodes Python numcodecs ${manifest.numcodecsVersion} output`, () => { const expected = Array.from(fixture("source.u2.bin")); @@ -44,6 +50,10 @@ describe(`decodes Python numcodecs ${manifest.numcodecsVersion} output`, () => { it("covers every codec this package exports", () => { const ids = new Set(Object.values(manifest.codecs).map((c) => c.config.id)); - expect([...ids].sort()).toEqual(Object.keys(REGISTRY).sort()); + const exported = Object.keys(REGISTRY).sort(); + // Guards the derivation itself: if the export scan ever silently found + // nothing, every other assertion here would vacuously agree with it. + expect(exported).toEqual(["blosc", "gzip", "lz4", "zlib", "zstd"]); + expect([...ids].sort()).toEqual(exported); }); }); From d182c73bc87571c843d98ffb1c48d4d19351b01b Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Thu, 6 Aug 2026 10:35:32 -0400 Subject: [PATCH 6/7] test(fixtures): pin the generator's numcodecs and make its output reproducible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `uv run --with numcodecs` resolved whatever release was current, so adding a single fixture would rewrite every existing buffer against a newer numcodecs — the silent refresh the README warns against, performed as a side effect. The version is now pinned to 0.16.5, and the generator refuses to run under a different one so bumping it is a deliberate edit. Pinning alone was not enough to make regeneration verifiable: a gzip member stores the current time in its header, so `gzip.bin` differed on every run regardless of whether anything had changed. That buries a real format change in noise and means the buffer cannot be re-derived to check it. The field is advisory and decoders ignore it, so it is zeroed the way `gzip -n` does — the compressed payload is untouched Python output. Two consecutive runs now produce byte-identical fixtures. Co-Authored-By: Claude Opus 5 (1M context) --- test/fixtures/README.md | 20 ++++++++++++++++--- test/fixtures/generate.py | 40 ++++++++++++++++++++++++++++++++++---- test/fixtures/gzip.bin | Bin 365 -> 365 bytes 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/test/fixtures/README.md b/test/fixtures/README.md index 369ad0a..c69dbef 100644 --- a/test/fixtures/README.md +++ b/test/fixtures/README.md @@ -20,13 +20,27 @@ while staying readable — what has to hold is that both sides read each other. ## Regenerating -Requires no local install; `uv` fetches `numcodecs` on demand: +Requires no local install; `uv` fetches `numcodecs` on demand, pinned to the +version that wrote the checked-in buffers: ```sh cd test/fixtures -uv run --with numcodecs python generate.py +uv run --with numcodecs==0.16.5 python generate.py ``` Regenerate only to add a codec or a new option combination. Refreshing the existing files against a newer `numcodecs` weakens the test — it would hide a -format break by replacing the fixture that should have caught it. +format break by replacing the fixture that should have caught it. That is why +the version is pinned rather than resolved: unpinned, adding a single fixture +rewrites all of them against whatever release happens to be current. The +generator checks the pin and refuses to run under a different version, so +bumping it takes editing `EXPECTED_NUMCODECS` in +[`generate.py`](./generate.py) and this line together. + +Under that pin the output is byte-for-byte reproducible, so re-running the +generator on an unchanged tree leaves `git status` clean and any diff is a real +one. The single exception is handled in `normalize()`: a gzip member stores the +current time in its header, which otherwise made `gzip.bin` differ on every run +regardless of whether anything meaningful had changed. That field is advisory +and decoders ignore it, so it is zeroed the way `gzip -n` does; the compressed +payload is untouched Python output. diff --git a/test/fixtures/generate.py b/test/fixtures/generate.py index c68c276..fb1d09f 100644 --- a/test/fixtures/generate.py +++ b/test/fixtures/generate.py @@ -2,10 +2,12 @@ Run from this directory: - uv run --with numcodecs python generate.py + uv run --with numcodecs==0.16.5 python generate.py -See README.md before refreshing existing files — replacing a fixture against a -newer numcodecs hides exactly the format break it exists to catch. +The pin is load-bearing. Unpinned, adding one fixture resolves whatever +numcodecs is current and rewrites every existing buffer along with it — which +is precisely the silent refresh README.md warns against. Bump it deliberately, +never incidentally. """ import json @@ -14,6 +16,11 @@ import numcodecs import numpy as np +# The version the checked-in fixtures were written by. Regenerating under a +# different one is a decision, so it fails loudly rather than quietly replacing +# the buffers that exist to catch a format break. +EXPECTED_NUMCODECS = "0.16.5" + # Ascending u2 values compress well under every codec while still exercising the # shuffle filters, which reorder bytes within each 2-byte element. SOURCE = np.arange(256, dtype=" bytes: + """Strip the one field that makes a fixture differ from run to run. + + A gzip member carries the modification time in header bytes 4..8, so + numcodecs stamps the current clock into every `gzip.bin` it writes. That + made the fixture unreproducible: regenerating it changed the file even + under an identical numcodecs, which buries a real format change in noise + and means the buffer cannot be re-derived to check it. The field is + advisory — decoders ignore it — so zeroing it is what `gzip -n` does, and + leaves the compressed payload untouched. + """ + if not name.startswith("gzip"): + return encoded + assert encoded[:2] == b"\x1f\x8b", f"{name} is not a gzip member" + return encoded[:4] + b"\x00\x00\x00\x00" + encoded[8:] + + def main() -> None: + if numcodecs.__version__ != EXPECTED_NUMCODECS: + raise SystemExit( + f"numcodecs {numcodecs.__version__} is installed, but the fixtures were " + f"written by {EXPECTED_NUMCODECS}. Run with " + f"`uv run --with numcodecs=={EXPECTED_NUMCODECS} python generate.py`, or " + f"update EXPECTED_NUMCODECS here and in README.md if the bump is intended." + ) + here = pathlib.Path(__file__).parent raw = SOURCE.tobytes() (here / "source.u2.bin").write_bytes(raw) manifest = {} for name, codec in CODECS.items(): - encoded = bytes(codec.encode(raw)) + encoded = normalize(name, bytes(codec.encode(raw))) (here / f"{name}.bin").write_bytes(encoded) manifest[name] = {"config": codec.get_config(), "bytes": len(encoded)} print(f"{name:26} {len(encoded):5} bytes") diff --git a/test/fixtures/gzip.bin b/test/fixtures/gzip.bin index 06b5eec67710a5158eb0eecf7a9fdbbb7b177077..d5adc0599654ec95fb7c100b7738630983834661 100644 GIT binary patch delta 17 WcmaFM^p=T3zMF#q1U7P{G6Dc8xdXTW delta 17 YcmaFM^p=T3zMF%gG@&SKBS$JD05rb^X#fBK From 2d90299b4f307183fdf6b6c654c8c1ece5ad850b Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Thu, 6 Aug 2026 10:35:32 -0400 Subject: [PATCH 7/7] docs: correct the interop claim, license scope, and numcodecs versions - README claimed chunks decode "and vice versa", but only the Python to JavaScript direction is tested; asserting the reverse would mean running Python in CI. Narrowed to what the fixtures actually pin. - CODE_OF_CONDUCT dedicated "this work" to CC0 while the repository is MIT. Scoped the dedication to the Code of Conduct text, so it stays reusable without appearing to relicense the project. - AGENTS.md listed one `numcodecs` at 0.3, conflating two different packages: numcodecs.js (npm, the benchmark baseline) and Python numcodecs (PyPI 0.16.5, which wrote the fixtures). Split into separate rows. - Recorded the empty-frame and fallible-allocation conventions, and tagged the repository-structure fence as `text` (MD040). Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 39 ++++++++++++++++++++++++++++----------- CODE_OF_CONDUCT.md | 5 ++++- README.md | 11 +++++++---- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index dd50191..bbd3af4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,6 +83,22 @@ interop fixtures are the ones that fail when that happens. **no panics across `wasm_bindgen`**, because a panic traps the WASM instance and every later call fails with it, so one corrupt chunk breaks every subsequent read. +- **An empty `Vec` alone is not a sufficient error signal**, because a frame + that legitimately holds zero bytes decodes to exactly the same thing. Each + wrapper compares the result against the length the frame itself declares: + `blosc_declared_size`, the 4-byte LE header `src/lz4.ts` reads directly, and + `zstd_declared_content_size`. Only a mismatch is an error. Getting this wrong + is silent: with `out` supplied, a "failed" decode returns the caller's own + untouched buffer, which reads as data rather than as an error. Note the limit + of this check — it catches truncation and a short decode, but a Blosc1 frame + carries no checksum over its payload, so corruption that still parses can + decode to the declared length and be returned as valid. That is pinned in + [./test/compat.test.ts](./test/compat.test.ts) so it stays a known fact. +- **Allocations sized from a header must be fallible.** The plausibility guards + below bound what a header may claim, not what the allocator can supply; an + infallible `vec![0; n]` turns a shortfall into `handle_alloc_error`, which + aborts the module exactly like a panic. Reserve through `try_reserve_exact` + (see `try_zeroed`) so it degrades to an ordinary decode failure. - **Sizes read out of a compressed header are untrusted input.** Every decoder we wrap trusts them to index or to size an allocation up front — `blusc` indexes the source with them, `lz4_flex` zero-fills the declared size before @@ -112,14 +128,15 @@ interop fixtures are the ones that fail when that happens. ## Key dependencies -| Dependency | Version | Notes | -| ------------------------- | --------------------------- | -------------------------------------------------------------------------- | -| `blusc` | 0.0.6 | Pure-Rust Blosc (Blosc2 API, Blosc1-compatible output) — the `Blosc` codec | -| `lz4_flex` | ~0.12 | Pure-Rust LZ4 block compression — the `LZ4` codec | -| `zstd` | ~0.13 (no default features) | Zstd — the `Zstd` codec | -| `wasm-bindgen` / `js-sys` | 0.2 / 0.3 | WASM boundary | -| `fflate` | ^0.8 (peer) | Pure-JS gzip/zlib — the `GZip` and `Zlib` codecs | -| `vite-plus` (`vp`) | 0.2 | Build / test / check driver | -| `vitest` | 4 | Test runner | -| `tinybench` | 6 | Benchmark timing (benchmark package) | -| `numcodecs` | 0.3 (dev) | Head-to-head baseline in [./benchmark/](./benchmark/) | +| Dependency | Version | Notes | +| ------------------------- | --------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `blusc` | 0.0.6 | Pure-Rust Blosc (Blosc2 API, Blosc1-compatible output) — the `Blosc` codec | +| `lz4_flex` | ~0.12 | Pure-Rust LZ4 block compression — the `LZ4` codec | +| `zstd` | ~0.13 (no default features) | Zstd — the `Zstd` codec | +| `wasm-bindgen` / `js-sys` | 0.2 / 0.3 | WASM boundary | +| `fflate` | ^0.8 (peer) | Pure-JS gzip/zlib — the `GZip` and `Zlib` codecs | +| `vite-plus` (`vp`) | 0.2 | Build / test / check driver | +| `vitest` | 4 | Test runner | +| `tinybench` | 6 | Benchmark timing (benchmark package) | +| `numcodecs` (npm) | ^0.3.2 (dev) | **numcodecs.js** — the head-to-head baseline in [./benchmark/](./benchmark/) | +| `numcodecs` (PyPI) | 0.16.5 | **Python numcodecs** — wrote [./test/fixtures/](./test/fixtures/); never installed, `uv` fetches it pinned | diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index f396188..7a2597a 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -34,4 +34,7 @@ Maintainers can remove, block, or ban anyone who disrupts the project. ## License -This work is dedicated to the public domain under [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/). +This Code of Conduct — the text of this file — is dedicated to the public domain +under [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/), so +other projects may reuse it freely. It does not change the license of the rest of +the repository, which is [MIT](LICENSE.txt). diff --git a/README.md b/README.md index 148fac0..843d32d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ compiled to WASM (SIMD128) via blusc. Built for Zarr — chunks written by Python - numcodecs decode in the browser, and vice versa. + numcodecs decode in the browser, pinned by fixtures Python + actually wrote.

## 📦 The codecs @@ -45,8 +46,10 @@ Every class carries the numcodecs.js surface — static `codecId`, static Compatibility with the _Python_ side is checked against real chunks: the buffers in [`test/fixtures/`](test/fixtures/) were written by Python `numcodecs`, and [`test/interop.test.ts`](test/interop.test.ts) decodes every -one of them. The WASM module loads lazily on first encode/decode, so the -pure-JS codecs never pay for it. +one of them. That pins the Python → JavaScript direction; the reverse is the +same shared byte format but is not exercised here, since asserting it would +mean running Python in CI. The WASM module loads lazily on first +encode/decode, so the pure-JS codecs never pay for it. ## 🚀 Quick start @@ -103,7 +106,7 @@ pnpm test ### Repository structure -``` +```text rumcodecs/ ├── src/ # TypeScript codec classes (Blosc, GZip, Zlib, LZ4, Zstd) ├── crates/rumcodecs-wasm/ # Rust WASM bindings: blusc, lz4_flex, zstd