From fe2e149e9b83434486271dedc11d2e59f4b7801b Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 18 Nov 2026 10:10:14 +0800 Subject: [PATCH 1/3] test(base32): cover bulk scalar contracts --- bases/src/backend/base32.rs | 137 +++++++++++++++++++++++ bases/src/backend/base32/arch/mod.rs | 7 ++ benchmarks/README.md | 2 +- benchmarks/benches/throughput/buffers.rs | 3 + 4 files changed, 148 insertions(+), 1 deletion(-) diff --git a/bases/src/backend/base32.rs b/bases/src/backend/base32.rs index 6878811..ad2fdbf 100644 --- a/bases/src/backend/base32.rs +++ b/bases/src/backend/base32.rs @@ -24,6 +24,9 @@ use crate::codec; #[allow(unsafe_code)] mod arch; +// Keep short inputs and SIMD tails on the compact scalar path. Bulk helpers stay out of line +// so their chunk setup does not enlarge those callers; 32 quanta is a conservative threshold, +// not a crossover tuned for every target. const SCALAR_BULK_QUANTA: usize = 32; const STANDARD_UPPER: &[u8; 32] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; @@ -943,6 +946,140 @@ mod tests { check_roundtrip_and_arch_parity::(); } + fn check_scalar_bulk( + quantum: &[u8; 8], + tails: &[&[u8]; 5], + ) { + let mut raw = [0_u8; 170]; + for chunk in raw.as_chunks_mut::<5>().0 { + chunk.copy_from_slice(b"fooba"); + } + let decode = |wire: &[u8], output: &mut [u8]| { + let plan = decode_plan::(wire)?; + decode_scalar_bulk::( + wire, + plan, + codec::as_uninit_slice(output), + ) + }; + + // Call the bulk helpers directly so SIMD hosts also exercise these loops and errors. + for quanta in [31, 32, 33] { + for (raw_tail_len, tail) in tails.iter().enumerate() { + let raw = &raw[..quanta * 5 + raw_tail_len]; + let tail_len = if PADDED { + tail.len() + } else { + tail.iter() + .position(|&byte| byte == b'=') + .unwrap_or(tail.len()) + }; + let mut reference = [0_u8; 272]; + for chunk in reference[..quanta * 8].as_chunks_mut::<8>().0 { + chunk.copy_from_slice(quantum); + } + reference[quanta * 8..quanta * 8 + tail_len].copy_from_slice(&tail[..tail_len]); + let wire = &mut reference[..quanta * 8 + tail_len]; + if !UPPERCASE { + wire.make_ascii_lowercase(); + } + let plan = decode_plan::(wire).unwrap(); + + for extra in [0, 7] { + let mut encoded = [0xa5; 280]; + let written = encode_scalar_bulk::( + raw, + codec::as_uninit_slice(&mut encoded[1..1 + wire.len() + extra]), + ); + assert_eq!(&encoded[1..1 + written], wire); + assert_eq!(encoded[0], 0xa5); + assert!(encoded[1 + written..].iter().all(|&byte| byte == 0xa5)); + + let mut decoded = [0xa5; 178]; + let written = decode(wire, &mut decoded[1..1 + raw.len() + extra]).unwrap(); + assert_eq!(&decoded[1..1 + written], raw); + assert_eq!(decoded[0], 0xa5); + assert!(decoded[1 + written..].iter().all(|&byte| byte == 0xa5)); + } + + let mut output = [0_u8; 170]; + for index in [0, 7, 8, 247, 248, 255, 256, plan.data_len - 1] { + if index >= plan.data_len { + continue; + } + let original = wire[index]; + for byte in [b'!', 0xff] { + wire[index] = byte; + assert_eq!( + decode(wire, &mut output), + Err(DecodeError::InvalidByte { index, byte }) + ); + } + // Keep '=' interior so the length/padding plan still reaches the decoder. + if index + 1 < plan.data_len { + wire[index] = b'='; + assert_eq!( + decode(wire, &mut output), + Err(DecodeError::InvalidPadding { index }) + ); + } + wire[index] = original; + } + + let original = wire[8]; + wire[8] = if UPPERCASE { + original.to_ascii_lowercase() + } else { + original.to_ascii_uppercase() + }; + assert_eq!( + decode(wire, &mut output), + Err(DecodeError::InvalidByte { + index: 8, + byte: wire[8], + }) + ); + let written = decode_scalar_bulk::( + wire, + plan, + codec::as_uninit_slice(&mut output), + ) + .unwrap(); + assert_eq!(&output[..written], raw); + wire[8] = original; + + if plan.data_len != plan.complete_len { + let index = plan.data_len - 1; + // Symbol value 1 sets an unused low bit for each partial-quantum length. + wire[index] = match (EXTENDED_HEX, UPPERCASE) { + (true, _) => b'1', + (false, true) => b'B', + (false, false) => b'b', + }; + assert_eq!( + decode(wire, &mut output), + Err(DecodeError::NonCanonical { index }) + ); + } + } + } + } + + #[test] + fn scalar_bulk_preserves_rfc4648_bytes_and_error_offsets() { + // RFC 4648 encodings of "fooba" and the empty, "f", "fo", "foo", and "foob" tails. + let standard: [&[u8]; 5] = [b"", b"MY======", b"MZXQ====", b"MZXW6===", b"MZXW6YQ="]; + let hex: [&[u8]; 5] = [b"", b"CO======", b"CPNG====", b"CPNMU===", b"CPNMUOG="]; + check_scalar_bulk::(b"MZXW6YTB", &standard); + check_scalar_bulk::(b"MZXW6YTB", &standard); + check_scalar_bulk::(b"MZXW6YTB", &standard); + check_scalar_bulk::(b"MZXW6YTB", &standard); + check_scalar_bulk::(b"CPNMUOJ1", &hex); + check_scalar_bulk::(b"CPNMUOJ1", &hex); + check_scalar_bulk::(b"CPNMUOJ1", &hex); + check_scalar_bulk::(b"CPNMUOJ1", &hex); + } + fn verify_all_bytes_at_offsets< const EXTENDED_HEX: bool, const PADDED: bool, diff --git a/bases/src/backend/base32/arch/mod.rs b/bases/src/backend/base32/arch/mod.rs index 77c55a2..8a2162c 100644 --- a/bases/src/backend/base32/arch/mod.rs +++ b/bases/src/backend/base32/arch/mod.rs @@ -13,6 +13,13 @@ // limitations under the License. //! Base32 backend dispatch. +//! +//! `encode` and `decode` return consumed input and written output byte counts. `None` means no +//! SIMD backend was selected and permits the caller to use its bulk scalar fallback. `Some` +//! resumes the ordinary scalar path at the returned offsets, including `Some((0, 0))` for short +//! inputs or decoding that stopped at the first invalid SIMD block. Short inputs can return +//! `Some((0, 0))` before CPU feature detection, so it does not prove SIMD support is available. +//! SIMD decoding leaves the first invalid block for scalar validation to report its exact error. use core::mem::MaybeUninit; diff --git a/benchmarks/README.md b/benchmarks/README.md index 8b8a990..8597ddc 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -10,7 +10,7 @@ Run the same fixed workload for every implementation in a scenario. The results | Streaming Base64 | 16 KiB | Process chunks of at most 257 bytes into a fixed destination. | `base64` I/O adapters with general-purpose and SIMD engines | | Invalid Base64 input | 1 KiB before encoding | Reject `!` at the first or last data symbol of a padded message. | `data-encoding`, `base64`, `base64::Simd`, `base64-simd` | -All families have caller-buffer cases at 8 B, 32 B, 1 KiB and 16 KiB; Base64 also includes 64 B. This separates short-field overhead from bulk throughput. Owned workloads cover a 32-byte hex field, 1 KiB of padded Base32 or Base64, and an 8-byte unpadded URL-safe cursor. Owned decoders receive a string. Streaming includes finalization or reading to EOF, including the I/O adapter's buffering. The invalid-input cases check acceptance of the original message before corrupting one symbol. +All families have caller-buffer cases at 8 B, 32 B, 1 KiB and 16 KiB; Base64 also includes 64 B. Base32 adds 159 B, 160 B and 164 B to cover the scalar bulk threshold and a partial quantum above it. These cases use normal CPU dispatch, so exercising the bulk scalar path requires a scalar build. This separates short-field overhead from bulk throughput. Owned workloads cover a 32-byte hex field, 1 KiB of padded Base32 or Base64, and an 8-byte unpadded URL-safe cursor. Owned decoders receive a string. Streaming includes finalization or reading to EOF, including the I/O adapter's buffering. The invalid-input cases check acceptance of the original message before corrupting one symbol. ```shell cargo x bench --no-run diff --git a/benchmarks/benches/throughput/buffers.rs b/benchmarks/benches/throughput/buffers.rs index a7ab48e..4021d5e 100644 --- a/benchmarks/benches/throughput/buffers.rs +++ b/benchmarks/benches/throughput/buffers.rs @@ -92,6 +92,9 @@ pub fn register(runner: &mut Runner) { &[ (8, 20_000_000, 20_000_000), (32, 15_000_000, 10_000_000), + (159, 10_000_000, 5_000_000), + (160, 10_000_000, 5_000_000), + (164, 10_000_000, 5_000_000), (1024, 4_300_000, 2_450_000), (16_384, 250_000, 150_000), ], From e199ee13fc26913eab6aead9957eca4a8211e529 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 18 Nov 2026 10:23:53 +0800 Subject: [PATCH 2/3] test(base32): focus bulk coverage on error offsets --- bases/src/backend/base32.rs | 157 ++++----------------------- bases/src/backend/base32/arch/mod.rs | 9 +- 2 files changed, 26 insertions(+), 140 deletions(-) diff --git a/bases/src/backend/base32.rs b/bases/src/backend/base32.rs index ad2fdbf..0c15523 100644 --- a/bases/src/backend/base32.rs +++ b/bases/src/backend/base32.rs @@ -24,9 +24,7 @@ use crate::codec; #[allow(unsafe_code)] mod arch; -// Keep short inputs and SIMD tails on the compact scalar path. Bulk helpers stay out of line -// so their chunk setup does not enlarge those callers; 32 quanta is a conservative threshold, -// not a crossover tuned for every target. +// Keep bulk setup out of the compact short-input and SIMD-tail paths. const SCALAR_BULK_QUANTA: usize = 32; const STANDARD_UPPER: &[u8; 32] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; @@ -946,138 +944,29 @@ mod tests { check_roundtrip_and_arch_parity::(); } - fn check_scalar_bulk( - quantum: &[u8; 8], - tails: &[&[u8]; 5], - ) { - let mut raw = [0_u8; 170]; - for chunk in raw.as_chunks_mut::<5>().0 { - chunk.copy_from_slice(b"fooba"); - } - let decode = |wire: &[u8], output: &mut [u8]| { - let plan = decode_plan::(wire)?; - decode_scalar_bulk::( - wire, - plan, - codec::as_uninit_slice(output), - ) - }; - - // Call the bulk helpers directly so SIMD hosts also exercise these loops and errors. - for quanta in [31, 32, 33] { - for (raw_tail_len, tail) in tails.iter().enumerate() { - let raw = &raw[..quanta * 5 + raw_tail_len]; - let tail_len = if PADDED { - tail.len() - } else { - tail.iter() - .position(|&byte| byte == b'=') - .unwrap_or(tail.len()) - }; - let mut reference = [0_u8; 272]; - for chunk in reference[..quanta * 8].as_chunks_mut::<8>().0 { - chunk.copy_from_slice(quantum); - } - reference[quanta * 8..quanta * 8 + tail_len].copy_from_slice(&tail[..tail_len]); - let wire = &mut reference[..quanta * 8 + tail_len]; - if !UPPERCASE { - wire.make_ascii_lowercase(); - } - let plan = decode_plan::(wire).unwrap(); - - for extra in [0, 7] { - let mut encoded = [0xa5; 280]; - let written = encode_scalar_bulk::( - raw, - codec::as_uninit_slice(&mut encoded[1..1 + wire.len() + extra]), - ); - assert_eq!(&encoded[1..1 + written], wire); - assert_eq!(encoded[0], 0xa5); - assert!(encoded[1 + written..].iter().all(|&byte| byte == 0xa5)); - - let mut decoded = [0xa5; 178]; - let written = decode(wire, &mut decoded[1..1 + raw.len() + extra]).unwrap(); - assert_eq!(&decoded[1..1 + written], raw); - assert_eq!(decoded[0], 0xa5); - assert!(decoded[1 + written..].iter().all(|&byte| byte == 0xa5)); - } - - let mut output = [0_u8; 170]; - for index in [0, 7, 8, 247, 248, 255, 256, plan.data_len - 1] { - if index >= plan.data_len { - continue; - } - let original = wire[index]; - for byte in [b'!', 0xff] { - wire[index] = byte; - assert_eq!( - decode(wire, &mut output), - Err(DecodeError::InvalidByte { index, byte }) - ); - } - // Keep '=' interior so the length/padding plan still reaches the decoder. - if index + 1 < plan.data_len { - wire[index] = b'='; - assert_eq!( - decode(wire, &mut output), - Err(DecodeError::InvalidPadding { index }) - ); - } - wire[index] = original; - } - - let original = wire[8]; - wire[8] = if UPPERCASE { - original.to_ascii_lowercase() - } else { - original.to_ascii_uppercase() - }; - assert_eq!( - decode(wire, &mut output), - Err(DecodeError::InvalidByte { - index: 8, - byte: wire[8], - }) - ); - let written = decode_scalar_bulk::( - wire, - plan, - codec::as_uninit_slice(&mut output), - ) - .unwrap(); - assert_eq!(&output[..written], raw); - wire[8] = original; - - if plan.data_len != plan.complete_len { - let index = plan.data_len - 1; - // Symbol value 1 sets an unused low bit for each partial-quantum length. - wire[index] = match (EXTENDED_HEX, UPPERCASE) { - (true, _) => b'1', - (false, true) => b'B', - (false, false) => b'b', - }; - assert_eq!( - decode(wire, &mut output), - Err(DecodeError::NonCanonical { index }) - ); - } - } - } - } - #[test] - fn scalar_bulk_preserves_rfc4648_bytes_and_error_offsets() { - // RFC 4648 encodings of "fooba" and the empty, "f", "fo", "foo", and "foob" tails. - let standard: [&[u8]; 5] = [b"", b"MY======", b"MZXQ====", b"MZXW6===", b"MZXW6YQ="]; - let hex: [&[u8]; 5] = [b"", b"CO======", b"CPNG====", b"CPNMU===", b"CPNMUOG="]; - check_scalar_bulk::(b"MZXW6YTB", &standard); - check_scalar_bulk::(b"MZXW6YTB", &standard); - check_scalar_bulk::(b"MZXW6YTB", &standard); - check_scalar_bulk::(b"MZXW6YTB", &standard); - check_scalar_bulk::(b"CPNMUOJ1", &hex); - check_scalar_bulk::(b"CPNMUOJ1", &hex); - check_scalar_bulk::(b"CPNMUOJ1", &hex); - check_scalar_bulk::(b"CPNMUOJ1", &hex); + fn scalar_bulk_reports_global_error_offsets() { + // 32 complete quanta followed by the RFC 4648 encoding of "f". + let mut input = [b'A'; 264]; + input[256..].copy_from_slice(b"MY======"); + let plan = decode_plan::(&input).unwrap(); + let mut output = [MaybeUninit::uninit(); 161]; + + input[255] = b'!'; + assert_eq!( + decode_scalar_bulk::(&input, plan, &mut output), + Err(DecodeError::InvalidByte { + index: 255, + byte: b'!', + }) + ); + + input[255] = b'A'; + input[257] = b'Z'; + assert_eq!( + decode_scalar_bulk::(&input, plan, &mut output), + Err(DecodeError::NonCanonical { index: 257 }) + ); } fn verify_all_bytes_at_offsets< diff --git a/bases/src/backend/base32/arch/mod.rs b/bases/src/backend/base32/arch/mod.rs index 8a2162c..092d5e7 100644 --- a/bases/src/backend/base32/arch/mod.rs +++ b/bases/src/backend/base32/arch/mod.rs @@ -14,12 +14,9 @@ //! Base32 backend dispatch. //! -//! `encode` and `decode` return consumed input and written output byte counts. `None` means no -//! SIMD backend was selected and permits the caller to use its bulk scalar fallback. `Some` -//! resumes the ordinary scalar path at the returned offsets, including `Some((0, 0))` for short -//! inputs or decoding that stopped at the first invalid SIMD block. Short inputs can return -//! `Some((0, 0))` before CPU feature detection, so it does not prove SIMD support is available. -//! SIMD decoding leaves the first invalid block for scalar validation to report its exact error. +//! `encode` and `decode` return consumed/written byte counts. `None` permits bulk scalar fallback; +//! `Some` resumes ordinary scalar processing. `Some((0, 0))` can mean a short input or an invalid +//! first SIMD block, and does not establish whether SIMD is available. use core::mem::MaybeUninit; From 69fcdde29691dcbb7e953704af5d69f0090d7513 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 18 Nov 2026 10:32:41 +0800 Subject: [PATCH 3/3] docs(base32): remove redundant implementation commentary --- bases/src/backend/base32.rs | 1 - bases/src/backend/base32/arch/mod.rs | 4 ---- 2 files changed, 5 deletions(-) diff --git a/bases/src/backend/base32.rs b/bases/src/backend/base32.rs index 0c15523..a879329 100644 --- a/bases/src/backend/base32.rs +++ b/bases/src/backend/base32.rs @@ -24,7 +24,6 @@ use crate::codec; #[allow(unsafe_code)] mod arch; -// Keep bulk setup out of the compact short-input and SIMD-tail paths. const SCALAR_BULK_QUANTA: usize = 32; const STANDARD_UPPER: &[u8; 32] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; diff --git a/bases/src/backend/base32/arch/mod.rs b/bases/src/backend/base32/arch/mod.rs index 092d5e7..77c55a2 100644 --- a/bases/src/backend/base32/arch/mod.rs +++ b/bases/src/backend/base32/arch/mod.rs @@ -13,10 +13,6 @@ // limitations under the License. //! Base32 backend dispatch. -//! -//! `encode` and `decode` return consumed/written byte counts. `None` permits bulk scalar fallback; -//! `Some` resumes ordinary scalar processing. `Some((0, 0))` can mean a short input or an invalid -//! first SIMD block, and does not establish whether SIMD is available. use core::mem::MaybeUninit;