From 02ccd834e4cc9e5fb7cf8cb20b5f28f5024c01d7 Mon Sep 17 00:00:00 2001 From: cijiugechu Date: Fri, 20 Nov 2026 22:30:26 +0800 Subject: [PATCH] refactor(backend): narrow unsafe scopes in SIMD kernels --- bases/src/backend/base16/arch/aarch64.rs | 147 ++++---- bases/src/backend/base16/arch/x86.rs | 227 ++++++------- bases/src/backend/base32/arch/aarch64.rs | 156 +++++---- bases/src/backend/base32/arch/x86.rs | 377 +++++++++++---------- bases/src/backend/base64/arch/aarch64.rs | 336 ++++++++++--------- bases/src/backend/base64/arch/x86.rs | 407 ++++++++++++----------- 6 files changed, 866 insertions(+), 784 deletions(-) diff --git a/bases/src/backend/base16/arch/aarch64.rs b/bases/src/backend/base16/arch/aarch64.rs index bc9ca2b..cd65e44 100644 --- a/bases/src/backend/base16/arch/aarch64.rs +++ b/bases/src/backend/base16/arch/aarch64.rs @@ -126,56 +126,66 @@ unsafe fn encode_neon( len: usize, ) { let table = if UPPERCASE { UPPER_HEX } else { LOWER_HEX }; - // SAFETY: Caller guarantees `len` readable bytes at `in_ptr`, `len * 2` writable bytes at - // `out_ptr`, `len` is a multiple of 8, and `table` is a 16-byte static array. - unsafe { - let hex_tbl = vld1q_u8(table.as_ptr()); - let mask_0f = vdupq_n_u8(0x0f); - let mut rem = len; + // SAFETY: `table` is a 16-byte static array. + let hex_tbl = unsafe { vld1q_u8(table.as_ptr()) }; + let mask_0f = vdupq_n_u8(0x0f); + let mut rem = len; - while rem >= 32 { - let v0 = vld1q_u8(in_ptr); - let v1 = vld1q_u8(in_ptr.add(16)); + while rem >= 32 { + // SAFETY: Caller guarantees at least `rem >= 32` readable bytes at `in_ptr`. + let (v0, v1) = unsafe { (vld1q_u8(in_ptr), vld1q_u8(in_ptr.add(16))) }; - let hi0 = vqtbl1q_u8(hex_tbl, vshrq_n_u8::<4>(v0)); - let lo0 = vqtbl1q_u8(hex_tbl, vandq_u8(v0, mask_0f)); - let out0 = vzip1q_u8(hi0, lo0); - let out1 = vzip2q_u8(hi0, lo0); + let hi0 = vqtbl1q_u8(hex_tbl, vshrq_n_u8::<4>(v0)); + let lo0 = vqtbl1q_u8(hex_tbl, vandq_u8(v0, mask_0f)); + let out0 = vzip1q_u8(hi0, lo0); + let out1 = vzip2q_u8(hi0, lo0); - let hi1 = vqtbl1q_u8(hex_tbl, vshrq_n_u8::<4>(v1)); - let lo1 = vqtbl1q_u8(hex_tbl, vandq_u8(v1, mask_0f)); - let out2 = vzip1q_u8(hi1, lo1); - let out3 = vzip2q_u8(hi1, lo1); + let hi1 = vqtbl1q_u8(hex_tbl, vshrq_n_u8::<4>(v1)); + let lo1 = vqtbl1q_u8(hex_tbl, vandq_u8(v1, mask_0f)); + let out2 = vzip1q_u8(hi1, lo1); + let out3 = vzip2q_u8(hi1, lo1); + // SAFETY: Caller guarantees at least `rem * 2 >= 64` writable bytes at `out_ptr`, and + // `in_ptr`/`out_ptr` remain within their buffers after advancing by 32 and 64 bytes. + unsafe { vst1q_u8(out_ptr, out0); vst1q_u8(out_ptr.add(16), out1); vst1q_u8(out_ptr.add(32), out2); vst1q_u8(out_ptr.add(48), out3); - in_ptr = in_ptr.add(32); out_ptr = out_ptr.add(64); - rem -= 32; } + rem -= 32; + } - if rem >= 16 { - let v = vld1q_u8(in_ptr); - let hi = vqtbl1q_u8(hex_tbl, vshrq_n_u8::<4>(v)); - let lo = vqtbl1q_u8(hex_tbl, vandq_u8(v, mask_0f)); - vst1q_u8(out_ptr, vzip1q_u8(hi, lo)); - vst1q_u8(out_ptr.add(16), vzip2q_u8(hi, lo)); + if rem >= 16 { + // SAFETY: Caller guarantees at least `rem >= 16` readable bytes at `in_ptr`. + let v = unsafe { vld1q_u8(in_ptr) }; + let hi = vqtbl1q_u8(hex_tbl, vshrq_n_u8::<4>(v)); + let lo = vqtbl1q_u8(hex_tbl, vandq_u8(v, mask_0f)); + let out0 = vzip1q_u8(hi, lo); + let out1 = vzip2q_u8(hi, lo); + // SAFETY: Caller guarantees at least `rem * 2 >= 32` writable bytes at `out_ptr`, and + // `in_ptr`/`out_ptr` remain within their buffers after advancing by 16 and 32 bytes. + unsafe { + vst1q_u8(out_ptr, out0); + vst1q_u8(out_ptr.add(16), out1); in_ptr = in_ptr.add(16); out_ptr = out_ptr.add(32); - rem -= 16; } + rem -= 16; + } - if rem >= 8 { - let v8 = vld1_u8(in_ptr); - let v = vcombine_u8(v8, v8); - let hi = vqtbl1q_u8(hex_tbl, vshrq_n_u8::<4>(v)); - let lo = vqtbl1q_u8(hex_tbl, vandq_u8(v, mask_0f)); - vst1q_u8(out_ptr, vzip1q_u8(hi, lo)); - } + if rem >= 8 { + // SAFETY: Caller guarantees at least `rem >= 8` readable bytes at `in_ptr`. + let v8 = unsafe { vld1_u8(in_ptr) }; + let v = vcombine_u8(v8, v8); + let hi = vqtbl1q_u8(hex_tbl, vshrq_n_u8::<4>(v)); + let lo = vqtbl1q_u8(hex_tbl, vandq_u8(v, mask_0f)); + let out = vzip1q_u8(hi, lo); + // SAFETY: Caller guarantees at least `rem * 2 >= 16` writable bytes at `out_ptr`. + unsafe { vst1q_u8(out_ptr, out) }; } } @@ -194,53 +204,53 @@ unsafe fn decode_neon( ) -> usize { let hi_mask_ptr = select_hi_mask::().as_ptr(); let mut done_16 = 0; - // SAFETY: Caller guarantees `chunks_16 * 16` readable bytes at `in_ptr`, `chunks_16 * 8` - // writable bytes at `out_ptr`, non-overtaking pointer advancement for in-place decoding, and - // 16-byte static table pointers. - unsafe { - let hi_mask_tbl = vld1q_u8(hi_mask_ptr); - let lo_mask_tbl = vld1q_u8(LO_MASK.as_ptr()); - let hi_off_tbl = vld1q_u8(HI_OFFSET.as_ptr()); - let mask_0f = vdupq_n_u8(0x0f); - let mut saw_invalid = false; + // SAFETY: All three pointers reference 16-byte static lookup tables. + let (hi_mask_tbl, lo_mask_tbl, hi_off_tbl) = unsafe { + ( + vld1q_u8(hi_mask_ptr), + vld1q_u8(LO_MASK.as_ptr()), + vld1q_u8(HI_OFFSET.as_ptr()), + ) + }; + let mask_0f = vdupq_n_u8(0x0f); + let mut saw_invalid = false; - while done_16 + 2 <= chunks_16 { - let uint8x16x2_t(even, odd) = vld2q_u8(in_ptr); - let Some(packed) = - decode_pair_neon(even, odd, hi_mask_tbl, lo_mask_tbl, hi_off_tbl, mask_0f) - else { - saw_invalid = true; - break; - }; + while done_16 + 2 <= chunks_16 { + // SAFETY: `done_16 + 2 <= chunks_16` guarantees 32 readable bytes at `in_ptr`. + let uint8x16x2_t(even, odd) = unsafe { vld2q_u8(in_ptr) }; + let Some(packed) = + decode_pair_neon(even, odd, hi_mask_tbl, lo_mask_tbl, hi_off_tbl, mask_0f) + else { + saw_invalid = true; + break; + }; + // SAFETY: `done_16 + 2 <= chunks_16` guarantees 16 writable bytes at `out_ptr`, in-place + // writes lag behind `in_ptr`, and advancing `in_ptr`/`out_ptr` by 32 and 16 bytes stays + // within both buffers. + unsafe { vst1q_u8(out_ptr, packed); in_ptr = in_ptr.add(32); out_ptr = out_ptr.add(16); - done_16 += 2; } + done_16 += 2; + } - if !saw_invalid - && done_16 < chunks_16 - && let Some(packed) = decode_16_neon( - vld1q_u8(in_ptr), - hi_mask_tbl, - lo_mask_tbl, - hi_off_tbl, - mask_0f, - ) - { - vst1_u8(out_ptr, packed); + if !saw_invalid && done_16 < chunks_16 { + // SAFETY: `done_16 < chunks_16` guarantees 16 readable bytes at `in_ptr`. + let chunk = unsafe { vld1q_u8(in_ptr) }; + if let Some(packed) = decode_16_neon(chunk, hi_mask_tbl, lo_mask_tbl, hi_off_tbl, mask_0f) { + // SAFETY: `done_16 < chunks_16` guarantees 8 writable bytes at `out_ptr`, and this + // chunk's input was already loaded so in-place writes do not overwrite unread input. + unsafe { vst1_u8(out_ptr, packed) }; done_16 += 1; } } done_16 } -/// # Safety -/// -/// Requires NEON target feature. #[inline] #[target_feature(enable = "neon")] -unsafe fn decode_16_neon( +fn decode_16_neon( chunk: uint8x16_t, hi_mask_tbl: uint8x16_t, lo_mask_tbl: uint8x16_t, @@ -262,12 +272,9 @@ unsafe fn decode_16_neon( Some(vsli_n_u8::<4>(odd, even)) } -/// # Safety -/// -/// Requires NEON target feature. #[inline] #[target_feature(enable = "neon")] -unsafe fn decode_pair_neon( +fn decode_pair_neon( even: uint8x16_t, odd: uint8x16_t, hi_mask_tbl: uint8x16_t, diff --git a/bases/src/backend/base16/arch/x86.rs b/bases/src/backend/base16/arch/x86.rs index 2ecac6b..c381467 100644 --- a/bases/src/backend/base16/arch/x86.rs +++ b/bases/src/backend/base16/arch/x86.rs @@ -228,12 +228,9 @@ pub fn decode_in_place( (done_32 * 32, done_32 * 16) } -/// # Safety -/// -/// Requires SSSE3 support on the current CPU. #[inline] #[target_feature(enable = "ssse3")] -unsafe fn hex_table_128() -> __m128i { +fn hex_table_128() -> __m128i { if UPPERCASE { _mm_setr_epi8( b'0' as i8, b'1' as i8, b'2' as i8, b'3' as i8, b'4' as i8, b'5' as i8, b'6' as i8, @@ -251,30 +248,29 @@ unsafe fn hex_table_128() -> __m128i { /// # Safety /// -/// Requires SSSE3 support on the current CPU. +/// Caller must ensure `in_ptr` is valid for 16 reads and `out_ptr` for 32 writes, with SSSE3 +/// enabled. #[inline] #[target_feature(enable = "ssse3")] unsafe fn encode_chunk_16(in_ptr: *const u8, out_ptr: *mut u8, hex_tbl: __m128i, mask_0f: __m128i) { - // SAFETY: Caller guarantees `in_ptr` is valid for 16 reads and `out_ptr` for 32 writes. + // SAFETY: Caller guarantees `in_ptr` is valid for 16 readable bytes. + let v = unsafe { _mm_loadu_si128(in_ptr.cast()) }; + let hi = _mm_and_si128(_mm_srli_epi16(v, 4), mask_0f); + let lo = _mm_and_si128(v, mask_0f); + let hi_ascii = _mm_shuffle_epi8(hex_tbl, hi); + let lo_ascii = _mm_shuffle_epi8(hex_tbl, lo); + let out0 = _mm_unpacklo_epi8(hi_ascii, lo_ascii); + let out1 = _mm_unpackhi_epi8(hi_ascii, lo_ascii); + // SAFETY: Caller guarantees `out_ptr` is valid for 32 writes. unsafe { - let v = _mm_loadu_si128(in_ptr.cast()); - let hi = _mm_and_si128(_mm_srli_epi16(v, 4), mask_0f); - let lo = _mm_and_si128(v, mask_0f); - let hi_ascii = _mm_shuffle_epi8(hex_tbl, hi); - let lo_ascii = _mm_shuffle_epi8(hex_tbl, lo); - let out0 = _mm_unpacklo_epi8(hi_ascii, lo_ascii); - let out1 = _mm_unpackhi_epi8(hi_ascii, lo_ascii); _mm_storeu_si128(out_ptr.cast(), out0); _mm_storeu_si128(out_ptr.add(16).cast(), out1); } } -/// # Safety -/// -/// Requires SSSE3 support on the current CPU. #[cfg(not(target_feature = "avx2"))] #[target_feature(enable = "ssse3")] -unsafe fn encode_ssse3( +fn encode_ssse3( input: &[u8], output: &mut [MaybeUninit], ) -> (usize, usize) { @@ -285,11 +281,12 @@ unsafe fn encode_ssse3( let mut in_ptr = input.as_ptr(); let mut out_ptr = output.as_mut_ptr().cast::(); - // SAFETY: `chunks * 16 <= input.len()` and `chunks * 32 <= output.len()`, and SSSE3 is enabled. - unsafe { - let hex_tbl = hex_table_128::(); - let mask_0f = _mm_set1_epi8(0x0f); - for _ in 0..chunks { + let hex_tbl = hex_table_128::(); + let mask_0f = _mm_set1_epi8(0x0f); + for _ in 0..chunks { + // SAFETY: `chunks * 16 <= input.len()` and `chunks * 32 <= output.len()`, so each + // iteration reads 16 input bytes, writes 32 output bytes, and stays in bounds. + unsafe { encode_chunk_16(in_ptr, out_ptr, hex_tbl, mask_0f); in_ptr = in_ptr.add(16); out_ptr = out_ptr.add(32); @@ -298,12 +295,9 @@ unsafe fn encode_ssse3( (chunks * 16, chunks * 32) } -/// # Safety -/// -/// Requires AVX2 support on the current CPU. #[cfg(any(feature = "std", target_feature = "avx2"))] #[target_feature(enable = "avx2")] -unsafe fn encode_avx2( +fn encode_avx2( input: &[u8], output: &mut [MaybeUninit], ) -> (usize, usize) { @@ -311,54 +305,54 @@ unsafe fn encode_avx2( let mut input_index = chunks * 32; let mut output_index = chunks * 64; - // SAFETY: `chunks * 32 <= input.len()`, `chunks * 64 <= output.len()`, and if the trailing - // 16-byte condition holds, 16 more input bytes and 32 more output bytes are in bounds. - unsafe { - let hex_128 = hex_table_128::(); - if chunks > 0 { - let hex_tbl = _mm256_set_m128i(hex_128, hex_128); - let mask_0f = _mm256_set1_epi8(0x0f); - let mut in_ptr = input.as_ptr(); - let mut out_ptr = output.as_mut_ptr().cast::(); - for _ in 0..chunks { - let v = _mm256_loadu_si256(in_ptr.cast()); - let hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), mask_0f); - let lo = _mm256_and_si256(v, mask_0f); - let hi_ascii = _mm256_shuffle_epi8(hex_tbl, hi); - let lo_ascii = _mm256_shuffle_epi8(hex_tbl, lo); - let unp_lo = _mm256_unpacklo_epi8(hi_ascii, lo_ascii); - let unp_hi = _mm256_unpackhi_epi8(hi_ascii, lo_ascii); - let out0 = _mm256_permute2x128_si256::<0x20>(unp_lo, unp_hi); - let out1 = _mm256_permute2x128_si256::<0x31>(unp_lo, unp_hi); + let hex_128 = hex_table_128::(); + if chunks > 0 { + let hex_tbl = _mm256_set_m128i(hex_128, hex_128); + let mask_0f = _mm256_set1_epi8(0x0f); + let mut in_ptr = input.as_ptr(); + let mut out_ptr = output.as_mut_ptr().cast::(); + for _ in 0..chunks { + // SAFETY: `chunks * 32 <= input.len()`, so `in_ptr` is valid for 32 reads. + let v = unsafe { _mm256_loadu_si256(in_ptr.cast()) }; + let hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), mask_0f); + let lo = _mm256_and_si256(v, mask_0f); + let hi_ascii = _mm256_shuffle_epi8(hex_tbl, hi); + let lo_ascii = _mm256_shuffle_epi8(hex_tbl, lo); + let unp_lo = _mm256_unpacklo_epi8(hi_ascii, lo_ascii); + let unp_hi = _mm256_unpackhi_epi8(hi_ascii, lo_ascii); + let out0 = _mm256_permute2x128_si256::<0x20>(unp_lo, unp_hi); + let out1 = _mm256_permute2x128_si256::<0x31>(unp_lo, unp_hi); + // SAFETY: `chunks * 64 <= output.len()`, so `out_ptr` is valid for 64 writes and + // advancing both pointers stays in bounds. + unsafe { _mm256_storeu_si256(out_ptr.cast(), out0); _mm256_storeu_si256(out_ptr.add(32).cast(), out1); in_ptr = in_ptr.add(32); out_ptr = out_ptr.add(64); } } - if input_index + 16 <= input.len() && output_index + 32 <= output.len() { - let mask_0f = _mm_set1_epi8(0x0f); + } + if input_index + 16 <= input.len() && output_index + 32 <= output.len() { + let mask_0f = _mm_set1_epi8(0x0f); + // SAFETY: The guard above ensures `input_index` and `output_index` are in bounds with 16 + // readable input bytes and 32 writable output bytes, and AVX2 implies SSSE3. + unsafe { encode_chunk_16( input.as_ptr().add(input_index), output.as_mut_ptr().cast::().add(output_index), hex_128, mask_0f, ); - input_index += 16; - output_index += 32; } + input_index += 16; + output_index += 32; } (input_index, output_index) } -/// # Safety -/// -/// Requires SSSE3 support on the current CPU. #[inline] #[target_feature(enable = "ssse3")] -unsafe fn decode_nibbles_128( - v: __m128i, -) -> __m128i { +fn decode_nibbles_128(v: __m128i) -> __m128i { // Saturation maps digits to 0-9 and letters to 10-15; all other bytes stay above 15. let digits = _mm_sub_epi8( _mm_subs_epu8( @@ -392,24 +386,26 @@ unsafe fn decode_nibbles_128( unsafe fn decode_chunk_32( in_ptr: *const u8, ) -> Option<__m128i> { - // SAFETY: Caller guarantees `in_ptr` is valid for 32 readable bytes and SSSE3 is enabled. - unsafe { - let in0 = _mm_loadu_si128(in_ptr.cast()); - let in1 = _mm_loadu_si128(in_ptr.add(16).cast()); - - let nib0 = decode_nibbles_128::(in0); - let nib1 = decode_nibbles_128::(in1); - // Biasing by 127 - 15 sets the sign bit exactly when either nibble is invalid. - let err = _mm_adds_epu8(_mm_or_si128(nib0, nib1), _mm_set1_epi8(112)); - if _mm_movemask_epi8(err) != 0 { - return None; - } + // SAFETY: Caller guarantees `in_ptr` is valid for 32 readable bytes. + let (in0, in1) = unsafe { + ( + _mm_loadu_si128(in_ptr.cast()), + _mm_loadu_si128(in_ptr.add(16).cast()), + ) + }; - let mul_0110 = _mm_set1_epi16(0x0110); - let pair0 = _mm_maddubs_epi16(nib0, mul_0110); - let pair1 = _mm_maddubs_epi16(nib1, mul_0110); - Some(_mm_packus_epi16(pair0, pair1)) + let nib0 = decode_nibbles_128::(in0); + let nib1 = decode_nibbles_128::(in1); + // Biasing by 127 - 15 sets the sign bit exactly when either nibble is invalid. + let err = _mm_adds_epu8(_mm_or_si128(nib0, nib1), _mm_set1_epi8(112)); + if _mm_movemask_epi8(err) != 0 { + return None; } + + let mul_0110 = _mm_set1_epi16(0x0110); + let pair0 = _mm_maddubs_epi16(nib0, mul_0110); + let pair1 = _mm_maddubs_epi16(nib1, mul_0110); + Some(_mm_packus_epi16(pair0, pair1)) } /// # Safety @@ -427,31 +423,28 @@ unsafe fn decode_ssse3( chunks: usize, ) -> usize { let mut done = 0; - // SAFETY: Caller guarantees `chunks * 32` reads at `in_ptr`, `chunks * 16` writes at `out_ptr`, - // and non-overtaking pointer advancement for in-place decoding. - unsafe { - while done < chunks { - let Some(packed) = decode_chunk_32::(in_ptr) else { - break; - }; + while done < chunks { + // SAFETY: `done < chunks` guarantees 32 readable bytes at `in_ptr`. + let Some(packed) = (unsafe { decode_chunk_32::(in_ptr) }) else { + break; + }; + // SAFETY: `done < chunks` guarantees 16 writable bytes at `out_ptr`, in-place writes lag + // behind `in_ptr`, and advancing `in_ptr`/`out_ptr` by 32 and 16 bytes stays within both + // buffers. + unsafe { _mm_storeu_si128(out_ptr.cast(), packed); in_ptr = in_ptr.add(32); out_ptr = out_ptr.add(16); - done += 1; } + done += 1; } done } -/// # Safety -/// -/// Requires AVX2 support on the current CPU. #[cfg(any(feature = "std", target_feature = "avx2"))] #[inline] #[target_feature(enable = "avx2")] -unsafe fn decode_nibbles_256( - v: __m256i, -) -> __m256i { +fn decode_nibbles_256(v: __m256i) -> __m256i { let digits = _mm256_sub_epi8( _mm256_subs_epu8( _mm256_add_epi8(v, _mm256_set1_epi8(!b'9' as i8)), @@ -485,24 +478,26 @@ unsafe fn decode_nibbles_256( unsafe fn decode_chunk_64( in_ptr: *const u8, ) -> Option<__m256i> { - // SAFETY: Caller guarantees `in_ptr` is valid for 64 readable bytes and AVX2 is enabled. - unsafe { - let in0 = _mm256_loadu_si256(in_ptr.cast()); - let in1 = _mm256_loadu_si256(in_ptr.add(32).cast()); - - let nib0 = decode_nibbles_256::(in0); - let nib1 = decode_nibbles_256::(in1); - let err = _mm256_adds_epu8(_mm256_or_si256(nib0, nib1), _mm256_set1_epi8(112)); - if _mm256_movemask_epi8(err) != 0 { - return None; - } + // SAFETY: Caller guarantees `in_ptr` is valid for 64 readable bytes. + let (in0, in1) = unsafe { + ( + _mm256_loadu_si256(in_ptr.cast()), + _mm256_loadu_si256(in_ptr.add(32).cast()), + ) + }; - let mul_0110 = _mm256_set1_epi16(0x0110); - let pair0 = _mm256_maddubs_epi16(nib0, mul_0110); - let pair1 = _mm256_maddubs_epi16(nib1, mul_0110); - let packed = _mm256_packus_epi16(pair0, pair1); - Some(_mm256_permute4x64_epi64::<0xD8>(packed)) + let nib0 = decode_nibbles_256::(in0); + let nib1 = decode_nibbles_256::(in1); + let err = _mm256_adds_epu8(_mm256_or_si256(nib0, nib1), _mm256_set1_epi8(112)); + if _mm256_movemask_epi8(err) != 0 { + return None; } + + let mul_0110 = _mm256_set1_epi16(0x0110); + let pair0 = _mm256_maddubs_epi16(nib0, mul_0110); + let pair1 = _mm256_maddubs_epi16(nib1, mul_0110); + let packed = _mm256_packus_epi16(pair0, pair1); + Some(_mm256_permute4x64_epi64::<0xD8>(packed)) } /// # Safety @@ -522,23 +517,29 @@ unsafe fn decode_avx2( let chunks_64 = chunks_32 / 2; let mut done_32 = 0; - // SAFETY: Caller guarantees `chunks_32 * 32` reads at `in_ptr`, `chunks_32 * 16` writes at - // `out_ptr`, and non-overtaking pointer advancement for in-place decoding. - unsafe { - for _ in 0..chunks_64 { - let Some(packed) = decode_chunk_64::(in_ptr) else { - return done_32; - }; + for _ in 0..chunks_64 { + // SAFETY: `done_32 + 2 <= chunks_32` guarantees 64 readable bytes at `in_ptr`. + let Some(packed) = (unsafe { decode_chunk_64::(in_ptr) }) else { + return done_32; + }; + // SAFETY: `done_32 + 2 <= chunks_32` guarantees 32 writable bytes at `out_ptr`, in-place + // writes lag behind `in_ptr`, and advancing `in_ptr`/`out_ptr` by 64 and 32 bytes stays + // within both buffers. + unsafe { _mm256_storeu_si256(out_ptr.cast(), packed); in_ptr = in_ptr.add(64); out_ptr = out_ptr.add(32); - done_32 += 2; } + done_32 += 2; + } - if done_32 < chunks_32 - && let Some(packed) = decode_chunk_32::(in_ptr) - { - _mm_storeu_si128(out_ptr.cast(), packed); + if done_32 < chunks_32 { + // SAFETY: `done_32 < chunks_32` guarantees 32 readable bytes at `in_ptr`, and AVX2 implies + // SSSE3. + if let Some(packed) = unsafe { decode_chunk_32::(in_ptr) } { + // SAFETY: `done_32 < chunks_32` guarantees 16 writable bytes at `out_ptr`, and this + // chunk's input was already loaded so in-place writes do not overwrite unread input. + unsafe { _mm_storeu_si128(out_ptr.cast(), packed) }; done_32 += 1; } } diff --git a/bases/src/backend/base32/arch/aarch64.rs b/bases/src/backend/base32/arch/aarch64.rs index 09bd637..e45ffb0 100644 --- a/bases/src/backend/base32/arch/aarch64.rs +++ b/bases/src/backend/base32/arch/aarch64.rs @@ -172,38 +172,48 @@ unsafe fn encode_neon( input_len: usize, ) { let alpha_ptr = select_alphabet::().as_ptr(); - // SAFETY: Caller guarantees sufficient readable/writable bytes and `alpha_ptr` points to 32 - // bytes. - unsafe { - let alphabet_tbl = uint8x16x2_t(vld1q_u8(alpha_ptr), vld1q_u8(alpha_ptr.add(16))); - let w_shuf = vld1q_u8(W_SHUF.as_ptr()); - let even_shifts: int16x8_t = vld1q_s16(EVEN_SHIFTS.as_ptr()); - let odd_shifts: int16x8_t = vld1q_s16(ODD_SHIFTS.as_ptr()); - let wide_chunks = ((input_len - 6) / 10).min(chunks); - for _ in 0..wide_chunks { - let raw = vld1q_u8(in_ptr); - let ascii = encode_chunk_neon(raw, alphabet_tbl, w_shuf, even_shifts, odd_shifts); + // SAFETY: `alpha_ptr` points to 32 bytes and the remaining pointers reference 16-byte static + // constant arrays. + let (alphabet_tbl, w_shuf, even_shifts, odd_shifts) = unsafe { + ( + uint8x16x2_t(vld1q_u8(alpha_ptr), vld1q_u8(alpha_ptr.add(16))), + vld1q_u8(W_SHUF.as_ptr()), + vld1q_s16(EVEN_SHIFTS.as_ptr()), + vld1q_s16(ODD_SHIFTS.as_ptr()), + ) + }; + let wide_chunks = ((input_len - 6) / 10).min(chunks); + for _ in 0..wide_chunks { + // SAFETY: `wide_chunks` guarantees 16 readable bytes at `in_ptr`. + let raw = unsafe { vld1q_u8(in_ptr) }; + let ascii = encode_chunk_neon(raw, alphabet_tbl, w_shuf, even_shifts, odd_shifts); + // SAFETY: Caller guarantees `chunks * 16` writable bytes at `out_ptr`, and advancing by 10 + // and 16 bytes stays within both buffers. + unsafe { vst1q_u8(out_ptr, ascii); in_ptr = in_ptr.add(10); out_ptr = out_ptr.add(16); } - if wide_chunks < chunks { - // The last two quanta need only ten readable bytes, even at an allocation boundary. - let low = ptr::read_unaligned(in_ptr.cast::()); - let high = u64::from(ptr::read_unaligned(in_ptr.add(8).cast::())); - let raw = vreinterpretq_u8_u64(vsetq_lane_u64::<1>(high, vdupq_n_u64(low))); - let ascii = encode_chunk_neon(raw, alphabet_tbl, w_shuf, even_shifts, odd_shifts); - vst1q_u8(out_ptr, ascii); - } + } + if wide_chunks < chunks { + // The last two quanta need only ten readable bytes, even at an allocation boundary. + // SAFETY: Caller guarantees at least 10 remaining readable bytes at `in_ptr`. + let (low, high) = unsafe { + ( + ptr::read_unaligned(in_ptr.cast::()), + u64::from(ptr::read_unaligned(in_ptr.add(8).cast::())), + ) + }; + let raw = vreinterpretq_u8_u64(vsetq_lane_u64::<1>(high, vdupq_n_u64(low))); + let ascii = encode_chunk_neon(raw, alphabet_tbl, w_shuf, even_shifts, odd_shifts); + // SAFETY: Caller guarantees 16 remaining writable bytes at `out_ptr`. + unsafe { vst1q_u8(out_ptr, ascii) }; } } -/// # Safety -/// -/// Requires NEON target feature. #[inline] #[target_feature(enable = "neon")] -unsafe fn encode_chunk_neon( +fn encode_chunk_neon( raw: uint8x16_t, alphabet: uint8x16x2_t, shuffle: uint8x16_t, @@ -243,65 +253,76 @@ unsafe fn decode_neon= chunks * 10` writes at - // `out_ptr`, non-overtaking pointer advancement for in-place decoding, and 16-byte static - // lookup tables. - unsafe { - let hi_mask_tbl = vld1q_u8(hi_mask_ptr); - let lo_mask_tbl = vld1q_u8(lo_mask_ptr); - let roll_tbl = vld1q_u8(roll_ptr); - let pack_shuf = vld1q_u8(PACK_SHUF.as_ptr()); - let mask_0f = vdupq_n_u8(0x0f); - let mask_001f = vdupq_n_u16(0x001f); - let mask_03ff = vdupq_n_u32(0x03ff); + // SAFETY: All four pointers reference 16-byte static lookup tables. + let (hi_mask_tbl, lo_mask_tbl, roll_tbl, pack_shuf) = unsafe { + ( + vld1q_u8(hi_mask_ptr), + vld1q_u8(lo_mask_ptr), + vld1q_u8(roll_ptr), + vld1q_u8(PACK_SHUF.as_ptr()), + ) + }; + let mask_0f = vdupq_n_u8(0x0f); + let mask_001f = vdupq_n_u16(0x001f); + let mask_03ff = vdupq_n_u32(0x03ff); - while done + 2 <= wide_chunks { - let v0 = vld1q_u8(in_ptr); - let v1 = vld1q_u8(in_ptr.add(16)); - let (hi0, chk0) = classify_chunk_16_neon(v0, hi_mask_tbl, lo_mask_tbl, mask_0f); - let (hi1, chk1) = classify_chunk_16_neon(v1, hi_mask_tbl, lo_mask_tbl, mask_0f); - if vminvq_u8(vminq_u8(chk0, chk1)) == 0 { - return done; - } - let res0 = pack_chunk_16_neon(v0, hi0, roll_tbl, pack_shuf, mask_001f, mask_03ff); - let res1 = pack_chunk_16_neon(v1, hi1, roll_tbl, pack_shuf, mask_001f, mask_03ff); + while done + 2 <= wide_chunks { + // SAFETY: `done + 2 <= wide_chunks <= chunks` guarantees 32 readable bytes at `in_ptr`. + let (v0, v1) = unsafe { (vld1q_u8(in_ptr), vld1q_u8(in_ptr.add(16))) }; + let (hi0, chk0) = classify_chunk_16_neon(v0, hi_mask_tbl, lo_mask_tbl, mask_0f); + let (hi1, chk1) = classify_chunk_16_neon(v1, hi_mask_tbl, lo_mask_tbl, mask_0f); + if vminvq_u8(vminq_u8(chk0, chk1)) == 0 { + return done; + } + let res0 = pack_chunk_16_neon(v0, hi0, roll_tbl, pack_shuf, mask_001f, mask_03ff); + let res1 = pack_chunk_16_neon(v1, hi1, roll_tbl, pack_shuf, mask_001f, mask_03ff); + // SAFETY: `done + 2 <= wide_chunks <= chunks` guarantees `done * 10 + 26 <= out_len`, + // in-place writes lag behind `in_ptr`, and advancing `in_ptr`/`out_ptr` by 32 and 20 bytes + // stays within both buffers. + unsafe { vst1q_u8(out_ptr, res0); vst1q_u8(out_ptr.add(10), res1); in_ptr = in_ptr.add(32); out_ptr = out_ptr.add(20); - done += 2; } + done += 2; + } - while done < chunks { - let v = vld1q_u8(in_ptr); - let (hi, chk) = classify_chunk_16_neon(v, hi_mask_tbl, lo_mask_tbl, mask_0f); - if vminvq_u8(chk) == 0 { - return done; - } - let res = pack_chunk_16_neon(v, hi, roll_tbl, pack_shuf, mask_001f, mask_03ff); - if done < wide_chunks { - vst1q_u8(out_ptr, res); - } else { - vst1_u8(out_ptr, vget_low_u8(res)); - ptr::write_unaligned( - out_ptr.add(8).cast::(), - vgetq_lane_u16::<4>(vreinterpretq_u16_u8(res)), - ); + while done < chunks { + // SAFETY: `done < chunks` guarantees 16 readable bytes at `in_ptr`. + let v = unsafe { vld1q_u8(in_ptr) }; + let (hi, chk) = classify_chunk_16_neon(v, hi_mask_tbl, lo_mask_tbl, mask_0f); + if vminvq_u8(chk) == 0 { + return done; + } + let res = pack_chunk_16_neon(v, hi, roll_tbl, pack_shuf, mask_001f, mask_03ff); + if done < wide_chunks { + // SAFETY: `done < wide_chunks` guarantees `done * 10 + 16 <= out_len`, and in-place + // writes lag behind `in_ptr`. + unsafe { vst1q_u8(out_ptr, res) }; + } else { + let lo = vget_low_u8(res); + let hi16 = vgetq_lane_u16::<4>(vreinterpretq_u16_u8(res)); + // SAFETY: `done < chunks` guarantees 10 writable bytes at `out_ptr`, and this chunk's + // input was already loaded so in-place writes do not overwrite unread input. + unsafe { + vst1_u8(out_ptr, lo); + ptr::write_unaligned(out_ptr.add(8).cast::(), hi16); } + } + // SAFETY: Advancing by 16 and 10 bytes stays within the caller-guaranteed buffers. + unsafe { in_ptr = in_ptr.add(16); out_ptr = out_ptr.add(10); - done += 1; } + done += 1; } done } -/// # Safety -/// -/// Requires NEON target feature. #[inline] #[target_feature(enable = "neon")] -unsafe fn classify_chunk_16_neon( +fn classify_chunk_16_neon( v: uint8x16_t, hi_mask_tbl: uint8x16_t, lo_mask_tbl: uint8x16_t, @@ -313,12 +334,9 @@ unsafe fn classify_chunk_16_neon( (hi, chk) } -/// # Safety -/// -/// Requires NEON target feature. #[inline] #[target_feature(enable = "neon")] -unsafe fn pack_chunk_16_neon( +fn pack_chunk_16_neon( v: uint8x16_t, hi: uint8x16_t, roll_tbl: uint8x16_t, diff --git a/bases/src/backend/base32/arch/x86.rs b/bases/src/backend/base32/arch/x86.rs index 98ae1e8..a5efd5b 100644 --- a/bases/src/backend/base32/arch/x86.rs +++ b/bases/src/backend/base32/arch/x86.rs @@ -298,12 +298,9 @@ pub fn decode_in_place() -> ( +fn encode_128_consts() -> ( __m128i, __m128i, __m128i, @@ -375,27 +372,22 @@ unsafe fn encode_chunk_16( base: __m128i, diff: __m128i, ) { - // SAFETY: Caller guarantees `in_ptr` is valid for 16 readable bytes and `out_ptr` for 16 - // writes. - unsafe { - let raw = _mm_loadu_si128(in_ptr.cast()); - let w = _mm_shuffle_epi8(raw, w_shuf); - let even = _mm_mulhi_epu16(_mm_and_si128(w, even_mask), even_mul); - let odd = _mm_mullo_epi16(_mm_and_si128(w, odd_mask), odd_mul); - let indices = _mm_or_si128(even, odd); - let is_alt = _mm_cmpgt_epi8(indices, thresh); - let offset = _mm_add_epi8(base, _mm_and_si128(is_alt, diff)); - let ascii = _mm_add_epi8(indices, offset); - _mm_storeu_si128(out_ptr.cast(), ascii); - } + // SAFETY: Caller guarantees `in_ptr` is valid for 16 readable bytes. + let raw = unsafe { _mm_loadu_si128(in_ptr.cast()) }; + let w = _mm_shuffle_epi8(raw, w_shuf); + let even = _mm_mulhi_epu16(_mm_and_si128(w, even_mask), even_mul); + let odd = _mm_mullo_epi16(_mm_and_si128(w, odd_mask), odd_mul); + let indices = _mm_or_si128(even, odd); + let is_alt = _mm_cmpgt_epi8(indices, thresh); + let offset = _mm_add_epi8(base, _mm_and_si128(is_alt, diff)); + let ascii = _mm_add_epi8(indices, offset); + // SAFETY: Caller guarantees `out_ptr` is valid for 16 writes. + unsafe { _mm_storeu_si128(out_ptr.cast(), ascii) }; } -/// # Safety -/// -/// Requires SSSE3 support on the current CPU. #[cfg(not(target_feature = "avx2"))] #[target_feature(enable = "ssse3")] -unsafe fn encode_ssse3( +fn encode_ssse3( input: &[u8], output: &mut [MaybeUninit], ) -> (usize, usize) { @@ -409,12 +401,13 @@ unsafe fn encode_ssse3( let mut in_ptr = input.as_ptr(); let mut out_ptr = output.as_mut_ptr().cast::(); - // SAFETY: `chunks > 0` guarantees `input.len() >= (chunks - 1) * 10 + 16` and `output.len() >= - // chunks * 16`. - unsafe { - let (w_shuf, even_mask, even_mul, odd_mask, odd_mul, thresh, base, diff, _) = - encode_128_consts::(); - for _ in 0..chunks { + let (w_shuf, even_mask, even_mul, odd_mask, odd_mul, thresh, base, diff, _) = + encode_128_consts::(); + for _ in 0..chunks { + // SAFETY: `chunks > 0` guarantees `input.len() >= (chunks - 1) * 10 + 16` and + // `output.len() >= chunks * 16`, so each iteration reads 16 bytes, writes 16 bytes, and + // stays in bounds. + unsafe { encode_chunk_16( in_ptr, out_ptr, w_shuf, even_mask, even_mul, odd_mask, odd_mul, thresh, base, diff, ); @@ -425,12 +418,9 @@ unsafe fn encode_ssse3( (chunks * 10, chunks * 16) } -/// # Safety -/// -/// Requires AVX2 support on the current CPU. #[cfg(any(feature = "std", target_feature = "avx2"))] #[target_feature(enable = "avx2")] -unsafe fn encode_avx2( +fn encode_avx2( input: &[u8], output: &mut [MaybeUninit], ) -> (usize, usize) { @@ -442,44 +432,52 @@ unsafe fn encode_avx2( let mut input_index = chunks * 20; let mut output_index = chunks * 32; - // SAFETY: `chunks > 0` implies `input.len() >= (chunks - 1) * 20 + 26` and `output.len() >= - // chunks * 32`. The trailing 16-byte check ensures `input_index + 16 <= input.len()` and - // `output_index + 16 <= output.len()`. - unsafe { - let (w_shuf_128, em_128, eml_128, om_128, oml_128, th_128, bs_128, df_128, _) = - encode_128_consts::(); - - if chunks > 0 { - let w_shuf = _mm256_set_m128i(w_shuf_128, w_shuf_128); - let even_mask = _mm256_set_m128i(em_128, em_128); - let even_mul = _mm256_set_m128i(eml_128, eml_128); - let odd_mask = _mm256_set_m128i(om_128, om_128); - let odd_mul = _mm256_set_m128i(oml_128, oml_128); - let thresh = _mm256_set_m128i(th_128, th_128); - let base = _mm256_set_m128i(bs_128, bs_128); - let diff = _mm256_set_m128i(df_128, df_128); - - let mut in_ptr = input.as_ptr(); - let mut out_ptr = output.as_mut_ptr().cast::(); - for _ in 0..chunks { - let raw = _mm256_set_m128i( + let (w_shuf_128, em_128, eml_128, om_128, oml_128, th_128, bs_128, df_128, _) = + encode_128_consts::(); + + if chunks > 0 { + let w_shuf = _mm256_set_m128i(w_shuf_128, w_shuf_128); + let even_mask = _mm256_set_m128i(em_128, em_128); + let even_mul = _mm256_set_m128i(eml_128, eml_128); + let odd_mask = _mm256_set_m128i(om_128, om_128); + let odd_mul = _mm256_set_m128i(oml_128, oml_128); + let thresh = _mm256_set_m128i(th_128, th_128); + let base = _mm256_set_m128i(bs_128, bs_128); + let diff = _mm256_set_m128i(df_128, df_128); + + let mut in_ptr = input.as_ptr(); + let mut out_ptr = output.as_mut_ptr().cast::(); + for _ in 0..chunks { + // SAFETY: `chunks > 0` implies `input.len() >= (chunks - 1) * 20 + 26`, so both + // 16-byte loads at `in_ptr` and `in_ptr.add(10)` are in bounds. + let (hi_raw, lo_raw) = unsafe { + ( _mm_loadu_si128(in_ptr.add(10).cast()), _mm_loadu_si128(in_ptr.cast()), - ); - let w = _mm256_shuffle_epi8(raw, w_shuf); - let even = _mm256_mulhi_epu16(_mm256_and_si256(w, even_mask), even_mul); - let odd = _mm256_mullo_epi16(_mm256_and_si256(w, odd_mask), odd_mul); - let indices = _mm256_or_si256(even, odd); - let is_alt = _mm256_cmpgt_epi8(indices, thresh); - let offset = _mm256_add_epi8(base, _mm256_and_si256(is_alt, diff)); - let ascii = _mm256_add_epi8(indices, offset); + ) + }; + let raw = _mm256_set_m128i(hi_raw, lo_raw); + let w = _mm256_shuffle_epi8(raw, w_shuf); + let even = _mm256_mulhi_epu16(_mm256_and_si256(w, even_mask), even_mul); + let odd = _mm256_mullo_epi16(_mm256_and_si256(w, odd_mask), odd_mul); + let indices = _mm256_or_si256(even, odd); + let is_alt = _mm256_cmpgt_epi8(indices, thresh); + let offset = _mm256_add_epi8(base, _mm256_and_si256(is_alt, diff)); + let ascii = _mm256_add_epi8(indices, offset); + // SAFETY: `output.len() >= chunks * 32`, so `out_ptr` is valid for 32 writes and + // advancing both pointers stays in bounds. + unsafe { _mm256_storeu_si256(out_ptr.cast(), ascii); in_ptr = in_ptr.add(20); out_ptr = out_ptr.add(32); } } + } - if input_index + 16 <= input.len() && output_index + 16 <= output.len() { + if input_index + 16 <= input.len() && output_index + 16 <= output.len() { + // SAFETY: The guard above ensures `input_index` and `output_index` are in bounds with 16 + // readable input bytes and 16 writable output bytes, and AVX2 implies SSSE3. + unsafe { encode_chunk_16( input.as_ptr().add(input_index), output.as_mut_ptr().cast::().add(output_index), @@ -492,24 +490,18 @@ unsafe fn encode_avx2( bs_128, df_128, ); - input_index += 10; - output_index += 16; } + input_index += 10; + output_index += 16; } (input_index, output_index) } -/// # Safety -/// -/// Requires SSSE3 support on the current CPU. #[inline] #[target_feature(enable = "ssse3")] -unsafe fn decode_128_tables< - const EXTENDED_HEX: bool, - const UPPERCASE: bool, - const STRICT_CASE: bool, ->() -> (__m128i, __m128i, __m128i, __m128i) { +fn decode_128_tables() +-> (__m128i, __m128i, __m128i, __m128i) { let hi = select_hi_mask::(); let lo = if EXTENDED_HEX { LO_MASK_HEX @@ -550,22 +542,20 @@ unsafe fn decode_chunk_16( mul_0001_0400: __m128i, zero: __m128i, ) -> Option<__m128i> { - // SAFETY: Caller guarantees `in_ptr` is valid for 16 readable bytes and SSSE3 is enabled. - unsafe { - let v = _mm_loadu_si128(in_ptr.cast()); - let hi = _mm_and_si128(_mm_srli_epi16(v, 4), mask_0f); - let lo = _mm_and_si128(v, mask_0f); - let chk = _mm_and_si128(_mm_shuffle_epi8(hi_mask, hi), _mm_shuffle_epi8(lo_mask, lo)); - if _mm_movemask_epi8(_mm_cmpeq_epi8(chk, zero)) != 0 { - return None; - } - - let vals = _mm_add_epi8(v, _mm_shuffle_epi8(roll_tbl, hi)); - let merged10 = _mm_maddubs_epi16(vals, mul_0120); - let packed20 = _mm_madd_epi16(merged10, mul_0001_0400); - let q = _mm_or_si128(_mm_slli_epi64(packed20, 20), _mm_srli_epi64(packed20, 32)); - Some(_mm_shuffle_epi8(q, pack_shuf)) + // SAFETY: Caller guarantees `in_ptr` is valid for 16 readable bytes. + let v = unsafe { _mm_loadu_si128(in_ptr.cast()) }; + let hi = _mm_and_si128(_mm_srli_epi16(v, 4), mask_0f); + let lo = _mm_and_si128(v, mask_0f); + let chk = _mm_and_si128(_mm_shuffle_epi8(hi_mask, hi), _mm_shuffle_epi8(lo_mask, lo)); + if _mm_movemask_epi8(_mm_cmpeq_epi8(chk, zero)) != 0 { + return None; } + + let vals = _mm_add_epi8(v, _mm_shuffle_epi8(roll_tbl, hi)); + let merged10 = _mm_maddubs_epi16(vals, mul_0120); + let packed20 = _mm_madd_epi16(merged10, mul_0001_0400); + let q = _mm_or_si128(_mm_slli_epi64(packed20, 20), _mm_srli_epi64(packed20, 32)); + Some(_mm_shuffle_epi8(q, pack_shuf)) } /// # Safety @@ -574,14 +564,11 @@ unsafe fn decode_chunk_16( #[inline] #[target_feature(enable = "ssse3")] unsafe fn store_10_ssse3(out_ptr: *mut u8, res: __m128i) { - // SAFETY: Caller guarantees `out_ptr` is valid for 10 writes and `res` holds the 10 decoded - // bytes in its low bytes. + let hi16 = _mm_extract_epi16::<4>(res) as u16; + // SAFETY: Caller guarantees `out_ptr` is valid for 10 writes. unsafe { _mm_storel_epi64(out_ptr.cast(), res); - ptr::write_unaligned( - out_ptr.add(8).cast::(), - _mm_extract_epi16::<4>(res) as u16, - ); + ptr::write_unaligned(out_ptr.add(8).cast::(), hi16); } } @@ -602,18 +589,17 @@ unsafe fn decode_ssse3 usize { let wide_chunks = (out_len.saturating_sub(6) / 10).min(chunks); let mut done = 0; - // SAFETY: Caller guarantees `chunks * 16` reads at `in_ptr`, `out_len >= chunks * 10` writes at - // `out_ptr`, and non-overtaking pointer advancement for in-place decoding. - unsafe { - let (hi_mask, lo_mask, roll_tbl, pack_shuf) = - decode_128_tables::(); - let mask_0f = _mm_set1_epi8(0x0f); - let mul_0120 = _mm_set1_epi16(0x0120); - let mul_0001_0400 = _mm_set1_epi32(0x0001_0400); - let zero = _mm_setzero_si128(); - - while done < wide_chunks { - let Some(res) = decode_chunk_16( + let (hi_mask, lo_mask, roll_tbl, pack_shuf) = + decode_128_tables::(); + let mask_0f = _mm_set1_epi8(0x0f); + let mul_0120 = _mm_set1_epi16(0x0120); + let mul_0001_0400 = _mm_set1_epi32(0x0001_0400); + let zero = _mm_setzero_si128(); + + while done < wide_chunks { + // SAFETY: `done < wide_chunks <= chunks` guarantees 16 readable bytes at `in_ptr`. + let Some(res) = (unsafe { + decode_chunk_16( in_ptr, hi_mask, lo_mask, @@ -623,17 +609,25 @@ unsafe fn decode_ssse3 Option<__m256i> { - // SAFETY: Caller guarantees `in_ptr` is valid for 32 readable bytes and AVX2 is enabled. - unsafe { - let v = _mm256_loadu_si256(in_ptr.cast()); - let hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), mask_0f); - let lo = _mm256_and_si256(v, mask_0f); - let chk = _mm256_and_si256( - _mm256_shuffle_epi8(hi_mask, hi), - _mm256_shuffle_epi8(lo_mask, lo), - ); - if _mm256_movemask_epi8(_mm256_cmpeq_epi8(chk, zero)) != 0 { - return None; - } - - let vals = _mm256_add_epi8(v, _mm256_shuffle_epi8(roll_tbl, hi)); - let merged10 = _mm256_maddubs_epi16(vals, mul_0120); - let packed20 = _mm256_madd_epi16(merged10, mul_0001_0400); - let q = _mm256_or_si256( - _mm256_slli_epi64(packed20, 20), - _mm256_srli_epi64(packed20, 32), - ); - Some(_mm256_shuffle_epi8(q, pack_shuf)) + // SAFETY: Caller guarantees `in_ptr` is valid for 32 readable bytes. + let v = unsafe { _mm256_loadu_si256(in_ptr.cast()) }; + let hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), mask_0f); + let lo = _mm256_and_si256(v, mask_0f); + let chk = _mm256_and_si256( + _mm256_shuffle_epi8(hi_mask, hi), + _mm256_shuffle_epi8(lo_mask, lo), + ); + if _mm256_movemask_epi8(_mm256_cmpeq_epi8(chk, zero)) != 0 { + return None; } + + let vals = _mm256_add_epi8(v, _mm256_shuffle_epi8(roll_tbl, hi)); + let merged10 = _mm256_maddubs_epi16(vals, mul_0120); + let packed20 = _mm256_madd_epi16(merged10, mul_0001_0400); + let q = _mm256_or_si256( + _mm256_slli_epi64(packed20, 20), + _mm256_srli_epi64(packed20, 32), + ); + Some(_mm256_shuffle_epi8(q, pack_shuf)) } /// # Safety @@ -704,15 +701,14 @@ unsafe fn decode_chunk_32( #[inline] #[target_feature(enable = "avx2")] unsafe fn store_20_avx2(out_ptr: *mut u8, res: __m256i) { - // SAFETY: `res` contains 10 decoded bytes in `res_lo[0..10]` (with bytes `10..16` zeroed) and - // 10 decoded bytes in `res_hi[0..10]`, and caller guarantees `out_ptr` is valid for 20 writes. + let res_lo = _mm256_castsi256_si128(res); + let res_hi = _mm256_extracti128_si256::<1>(res); + let lo16 = _mm_or_si128(res_lo, _mm_slli_si128::<10>(res_hi)); + let hi4 = _mm_cvtsi128_si32(_mm_srli_si128::<6>(res_hi)); + // SAFETY: Caller guarantees `out_ptr` is valid for 20 writes. unsafe { - let res_lo = _mm256_castsi256_si128(res); - let res_hi = _mm256_extracti128_si256::<1>(res); - let lo16 = _mm_or_si128(res_lo, _mm_slli_si128::<10>(res_hi)); _mm_storeu_si128(out_ptr.cast(), lo16); - let hi4 = _mm_srli_si128::<6>(res_hi); - ptr::write_unaligned(out_ptr.add(16).cast::(), _mm_cvtsi128_si32(hi4)); + ptr::write_unaligned(out_ptr.add(16).cast::(), hi4); } } @@ -735,23 +731,22 @@ unsafe fn decode_avx2= chunks_16 * 10` - // writes at `out_ptr`, and non-overtaking pointer advancement for in-place decoding. - unsafe { - let (hi_128, lo_128, roll_128, shuf_128) = - decode_128_tables::(); - if chunks_32 > 0 { - let hi_mask = _mm256_set_m128i(hi_128, hi_128); - let lo_mask = _mm256_set_m128i(lo_128, lo_128); - let roll_tbl = _mm256_set_m128i(roll_128, roll_128); - let pack_shuf = _mm256_set_m128i(shuf_128, shuf_128); - let mask_0f = _mm256_set1_epi8(0x0f); - let mul_0120 = _mm256_set1_epi16(0x0120); - let mul_0001_0400 = _mm256_set1_epi32(0x0001_0400); - let zero = _mm256_setzero_si256(); - - for idx_32 in 0..chunks_32 { - let Some(res) = decode_chunk_32( + let (hi_128, lo_128, roll_128, shuf_128) = + decode_128_tables::(); + if chunks_32 > 0 { + let hi_mask = _mm256_set_m128i(hi_128, hi_128); + let lo_mask = _mm256_set_m128i(lo_128, lo_128); + let roll_tbl = _mm256_set_m128i(roll_128, roll_128); + let pack_shuf = _mm256_set_m128i(shuf_128, shuf_128); + let mask_0f = _mm256_set1_epi8(0x0f); + let mul_0120 = _mm256_set1_epi16(0x0120); + let mul_0001_0400 = _mm256_set1_epi32(0x0001_0400); + let zero = _mm256_setzero_si256(); + + for idx_32 in 0..chunks_32 { + // SAFETY: `idx_32 < chunks_32` guarantees 32 readable bytes at `in_ptr`. + let Some(res) = (unsafe { + decode_chunk_32( in_ptr, hi_mask, lo_mask, @@ -761,27 +756,43 @@ unsafe fn decode_avx2(res)); - } else { - store_20_avx2(out_ptr, res); + ) + }) else { + return done_16; + }; + if idx_32 < wide_32 { + let lo = _mm256_castsi256_si128(res); + let hi = _mm256_extracti128_si256::<1>(res); + // SAFETY: `idx_32 < wide_32` guarantees `idx_32 * 20 + 26 <= out_len`, and + // in-place writes lag behind `in_ptr`. + unsafe { + _mm_storeu_si128(out_ptr.cast(), lo); + _mm_storeu_si128(out_ptr.add(10).cast(), hi); } + } else { + // SAFETY: `idx_32 < chunks_32` guarantees 20 writable bytes at `out_ptr`, and this + // chunk's input was already loaded so in-place writes do not overwrite unread + // input. + unsafe { store_20_avx2(out_ptr, res) }; + } + // SAFETY: Advancing by 32 and 20 bytes stays within the caller-guaranteed buffers. + unsafe { in_ptr = in_ptr.add(32); out_ptr = out_ptr.add(20); - done_16 += 2; } + done_16 += 2; } + } - if done_16 < chunks_16 { - let mask_0f = _mm_set1_epi8(0x0f); - let mul_0120 = _mm_set1_epi16(0x0120); - let mul_0001_0400 = _mm_set1_epi32(0x0001_0400); - let zero = _mm_setzero_si128(); - if let Some(res) = decode_chunk_16( + if done_16 < chunks_16 { + let mask_0f = _mm_set1_epi8(0x0f); + let mul_0120 = _mm_set1_epi16(0x0120); + let mul_0001_0400 = _mm_set1_epi32(0x0001_0400); + let zero = _mm_setzero_si128(); + // SAFETY: `done_16 < chunks_16` guarantees 16 readable bytes at `in_ptr`, and AVX2 implies + // SSSE3. + if let Some(res) = unsafe { + decode_chunk_16( in_ptr, hi_128, lo_128, @@ -791,14 +802,18 @@ unsafe fn decode_avx2( #[inline] #[target_feature(enable = "neon")] unsafe fn load_12_neon(in_ptr: *const u8, rem_total_in: usize) -> uint8x16_t { - // SAFETY: Caller guarantees at least `rem_total_in >= 12` readable bytes at `in_ptr`. - unsafe { - if rem_total_in >= 16 { - vld1q_u8(in_ptr) - } else { - let lo = vld1_u8(in_ptr); - let hi32 = ptr::read_unaligned(in_ptr.add(8).cast::()); - let hi = vreinterpret_u8_u32(vset_lane_u32::<0>(hi32, vdup_n_u32(0))); - vcombine_u8(lo, hi) - } + if rem_total_in >= 16 { + // SAFETY: Caller guarantees at least `rem_total_in >= 16` readable bytes at `in_ptr`. + unsafe { vld1q_u8(in_ptr) } + } else { + // SAFETY: Caller guarantees at least `rem_total_in >= 12` readable bytes at `in_ptr`. + let (lo, hi32) = unsafe { + ( + vld1_u8(in_ptr), + ptr::read_unaligned(in_ptr.add(8).cast::()), + ) + }; + let hi = vreinterpret_u8_u32(vset_lane_u32::<0>(hi32, vdup_n_u32(0))); + vcombine_u8(lo, hi) } } @@ -173,20 +176,19 @@ unsafe fn load_12_neon(in_ptr: *const u8, rem_total_in: usize) -> uint8x16_t { #[inline] #[target_feature(enable = "neon")] unsafe fn store_12_neon(out_ptr: *mut u8, packed: uint8x16_t) { + let lo = vget_low_u8(packed); + let hi = vgetq_lane_u32::<2>(vreinterpretq_u32_u8(packed)); // SAFETY: Caller guarantees `out_ptr` is valid for 12 writes. unsafe { - vst1_u8(out_ptr, vget_low_u8(packed)); - ptr::write_unaligned( - out_ptr.add(8).cast::(), - vgetq_lane_u32::<2>(vreinterpretq_u32_u8(packed)), - ); + vst1_u8(out_ptr, lo); + ptr::write_unaligned(out_ptr.add(8).cast::(), hi); } } /// # Safety /// -/// Caller must ensure `in_ptr` is valid for `total_in >= consumed_in` reads (multiple of 12) and -/// `out_ptr` is valid for `(consumed_in / 12) * 16` non-overlapping writes, with NEON enabled. +/// Caller must ensure `in_ptr` is valid for `rem_total_in >= rem_in` reads (`rem_in` a multiple of +/// 12) and `out_ptr` is valid for `(rem_in / 12) * 16` non-overlapping writes, with NEON enabled. #[inline] #[target_feature(enable = "neon")] unsafe fn encode_neon( @@ -201,53 +203,69 @@ unsafe fn encode_neon( STANDARD } .as_ptr(); - // SAFETY: Caller guarantees sufficient readable/writable bytes and static tables are valid. - unsafe { - let alphabet_tbl = uint8x16x4_t( + // SAFETY: `alpha_ptr` points to a 64-byte static alphabet table. + let alphabet_tbl = unsafe { + uint8x16x4_t( vld1q_u8(alpha_ptr), vld1q_u8(alpha_ptr.add(16)), vld1q_u8(alpha_ptr.add(32)), vld1q_u8(alpha_ptr.add(48)), - ); - let mask_3f = vdupq_n_u8(0x3f); - - while rem_in >= 48 { - let uint8x16x3_t(b0, b1, b2) = vld3q_u8(in_ptr); - let idx0 = vshrq_n_u8::<2>(b0); - let idx1 = vandq_u8(vorrq_u8(vshlq_n_u8::<4>(b0), vshrq_n_u8::<4>(b1)), mask_3f); - let idx2 = vandq_u8(vorrq_u8(vshlq_n_u8::<2>(b1), vshrq_n_u8::<6>(b2)), mask_3f); - let idx3 = vandq_u8(b2, mask_3f); - - let out0 = vqtbl4q_u8(alphabet_tbl, idx0); - let out1 = vqtbl4q_u8(alphabet_tbl, idx1); - let out2 = vqtbl4q_u8(alphabet_tbl, idx2); - let out3 = vqtbl4q_u8(alphabet_tbl, idx3); + ) + }; + let mask_3f = vdupq_n_u8(0x3f); + + while rem_in >= 48 { + // SAFETY: `rem_in >= 48` guarantees 48 readable bytes at `in_ptr`. + let uint8x16x3_t(b0, b1, b2) = unsafe { vld3q_u8(in_ptr) }; + let idx0 = vshrq_n_u8::<2>(b0); + let idx1 = vandq_u8(vorrq_u8(vshlq_n_u8::<4>(b0), vshrq_n_u8::<4>(b1)), mask_3f); + let idx2 = vandq_u8(vorrq_u8(vshlq_n_u8::<2>(b1), vshrq_n_u8::<6>(b2)), mask_3f); + let idx3 = vandq_u8(b2, mask_3f); + + let out0 = vqtbl4q_u8(alphabet_tbl, idx0); + let out1 = vqtbl4q_u8(alphabet_tbl, idx1); + let out2 = vqtbl4q_u8(alphabet_tbl, idx2); + let out3 = vqtbl4q_u8(alphabet_tbl, idx3); + // SAFETY: `rem_in >= 48` guarantees 64 writable bytes at `out_ptr`, and advancing by 48 + // and 64 bytes stays within both buffers. + unsafe { vst4q_u8(out_ptr, uint8x16x4_t(out0, out1, out2, out3)); - in_ptr = in_ptr.add(48); out_ptr = out_ptr.add(64); - rem_in -= 48; - rem_total_in -= 48; } - if rem_in >= 12 { - let shuf_a = vld1q_u8(ENC_SHUF_A.as_ptr()); - let shl_a = vld1q_s8(ENC_SHL_A.as_ptr()); - let shuf_b = vld1q_u8(ENC_SHUF_B.as_ptr()); - let shr_b = vld1q_s8(ENC_SHR_B.as_ptr()); - - while rem_in >= 12 { - let v = load_12_neon(in_ptr, rem_total_in); - let part_a = vshlq_u8(vqtbl1q_u8(v, shuf_a), shl_a); - let part_b = vshlq_u8(vqtbl1q_u8(v, shuf_b), shr_b); - let idx = vandq_u8(vorrq_u8(part_a, part_b), mask_3f); - vst1q_u8(out_ptr, vqtbl4q_u8(alphabet_tbl, idx)); + rem_in -= 48; + rem_total_in -= 48; + } + if rem_in >= 12 { + // SAFETY: All four pointers reference 16-byte static constant tables. + let (shuf_a, shl_a, shuf_b, shr_b) = unsafe { + ( + vld1q_u8(ENC_SHUF_A.as_ptr()), + vld1q_s8(ENC_SHL_A.as_ptr()), + vld1q_u8(ENC_SHUF_B.as_ptr()), + vld1q_s8(ENC_SHR_B.as_ptr()), + ) + }; + + while rem_in >= 12 { + // SAFETY: `rem_total_in >= rem_in >= 12` guarantees enough readable bytes at `in_ptr`. + let v = unsafe { load_12_neon(in_ptr, rem_total_in) }; + let part_a = vshlq_u8(vqtbl1q_u8(v, shuf_a), shl_a); + let part_b = vshlq_u8(vqtbl1q_u8(v, shuf_b), shr_b); + let idx = vandq_u8(vorrq_u8(part_a, part_b), mask_3f); + let ascii = vqtbl4q_u8(alphabet_tbl, idx); + // SAFETY: `rem_in >= 12` guarantees 16 writable bytes at `out_ptr`, and advancing by + // 12 and 16 bytes stays within both buffers. + unsafe { + vst1q_u8(out_ptr, ascii); in_ptr = in_ptr.add(12); out_ptr = out_ptr.add(16); - rem_in -= 12; - rem_total_in -= 12; } + + rem_in -= 12; + rem_total_in -= 12; } } } @@ -282,81 +300,99 @@ unsafe fn decode_neon( let wide_16 = (out_len.saturating_sub(4) / 12).min(limit_16); let mut done_16 = 0; - // SAFETY: Caller guarantees `limit_16 * 16` reads at `in_ptr`, `out_len` writes at `out_ptr`, - // non-overtaking pointer advancement for in-place decoding, and 16-byte static lookup tables. - unsafe { - let hi_mask_tbl = vld1q_u8(HI_MASK.as_ptr()); - let lo_mask_tbl = vld1q_u8(lo_mask_ptr); - let roll_tbl = vld1q_u8(roll_ptr); - let mask_0f = vdupq_n_u8(0x0f); - let special_ch = vdupq_n_u8(if URL_SAFE_ALPHABET { b'_' } else { b'/' }); - let special_adj = vdupq_n_u8(if URL_SAFE_ALPHABET { 33 } else { 3 }); - - let limit_64_as_16 = (limit_16 / 4) * 4; - while done_16 < limit_64_as_16 { - let uint8x16x4_t(c0, c1, c2, c3) = vld4q_u8(in_ptr); - let Some(packed) = decode_chunk_64_neon::( - c0, - c1, - c2, - c3, + // SAFETY: All three pointers reference 16-byte static lookup tables. + let (hi_mask_tbl, lo_mask_tbl, roll_tbl) = unsafe { + ( + vld1q_u8(HI_MASK.as_ptr()), + vld1q_u8(lo_mask_ptr), + vld1q_u8(roll_ptr), + ) + }; + let mask_0f = vdupq_n_u8(0x0f); + let special_ch = vdupq_n_u8(if URL_SAFE_ALPHABET { b'_' } else { b'/' }); + let special_adj = vdupq_n_u8(if URL_SAFE_ALPHABET { 33 } else { 3 }); + + let limit_64_as_16 = (limit_16 / 4) * 4; + while done_16 < limit_64_as_16 { + // SAFETY: `done_16 + 4 <= limit_16` guarantees 64 readable bytes at `in_ptr`. + let uint8x16x4_t(c0, c1, c2, c3) = unsafe { vld4q_u8(in_ptr) }; + let Some(packed) = decode_chunk_64_neon::( + c0, + c1, + c2, + c3, + hi_mask_tbl, + lo_mask_tbl, + roll_tbl, + mask_0f, + special_ch, + special_adj, + ) else { + return (done_16, true); + }; + // SAFETY: `done_16 + 4 <= limit_16` guarantees 48 writable bytes at `out_ptr`, in-place + // writes lag behind `in_ptr`, and advancing `in_ptr`/`out_ptr` by 64 and 48 bytes stays + // within both buffers. + unsafe { + vst3q_u8(out_ptr, packed); + in_ptr = in_ptr.add(64); + out_ptr = out_ptr.add(48); + } + done_16 += 4; + } + + if done_16 < limit_16 { + // SAFETY: All three pointers reference 16-byte static constant tables. + let (dec_shl, dec_shr, dec_pack) = unsafe { + ( + vld1q_s8(DEC_SHL.as_ptr()), + vld1q_s8(DEC_SHR.as_ptr()), + vld1q_u8(DEC_PACK.as_ptr()), + ) + }; + + while done_16 < limit_16 { + // SAFETY: `done_16 < limit_16` guarantees 16 readable bytes at `in_ptr`. + let c = unsafe { vld1q_u8(in_ptr) }; + let Some(packed) = decode_chunk_16_neon::( + c, hi_mask_tbl, lo_mask_tbl, roll_tbl, mask_0f, special_ch, special_adj, + dec_shl, + dec_shr, + dec_pack, ) else { return (done_16, true); }; - vst3q_u8(out_ptr, packed); - in_ptr = in_ptr.add(64); - out_ptr = out_ptr.add(48); - done_16 += 4; - } - - if done_16 < limit_16 { - let dec_shl = vld1q_s8(DEC_SHL.as_ptr()); - let dec_shr = vld1q_s8(DEC_SHR.as_ptr()); - let dec_pack = vld1q_u8(DEC_PACK.as_ptr()); - - while done_16 < limit_16 { - let c = vld1q_u8(in_ptr); - let Some(packed) = decode_chunk_16_neon::( - c, - hi_mask_tbl, - lo_mask_tbl, - roll_tbl, - mask_0f, - special_ch, - special_adj, - dec_shl, - dec_shr, - dec_pack, - ) else { - return (done_16, true); - }; - if done_16 < wide_16 { - vst1q_u8(out_ptr, packed); - } else { - store_12_neon(out_ptr, packed); - } + if done_16 < wide_16 { + // SAFETY: `done_16 < wide_16` guarantees `done_16 * 12 + 16 <= out_len`, and + // in-place writes lag behind `in_ptr`. + unsafe { vst1q_u8(out_ptr, packed) }; + } else { + // SAFETY: `done_16 < limit_16` guarantees 12 writable bytes at `out_ptr`, and this + // chunk's input was already loaded so in-place writes do not overwrite unread + // input. + unsafe { store_12_neon(out_ptr, packed) }; + } + // SAFETY: Advancing by 16 and 12 bytes stays within the caller-guaranteed buffers. + unsafe { in_ptr = in_ptr.add(16); out_ptr = out_ptr.add(12); - done_16 += 1; } + done_16 += 1; } } (done_16, false) } -/// # Safety -/// -/// Requires NEON target feature. #[inline] #[target_feature(enable = "neon")] #[allow(clippy::too_many_arguments)] -unsafe fn decode_chunk_16_neon( +fn decode_chunk_16_neon( c: uint8x16_t, hi_mask_tbl: uint8x16_t, lo_mask_tbl: uint8x16_t, @@ -368,27 +404,21 @@ unsafe fn decode_chunk_16_neon( dec_shr: int8x16_t, dec_pack: uint8x16_t, ) -> Option { - // SAFETY: Caller guarantees NEON is enabled. - unsafe { - let hi = vshrq_n_u8::<4>(c); - let lo = vandq_u8(c, mask_0f); - let chk = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi), vqtbl1q_u8(lo_mask_tbl, lo)); - if vminvq_u8(chk) == 0 { - return None; - } - let v = translate_lane_neon::(c, hi, roll_tbl, special_ch, special_adj); - let v_next = vextq_u8::<1>(v, v); - let merged = vorrq_u8(vshlq_u8(v, dec_shl), vshlq_u8(v_next, dec_shr)); - Some(vqtbl1q_u8(merged, dec_pack)) + let hi = vshrq_n_u8::<4>(c); + let lo = vandq_u8(c, mask_0f); + let chk = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi), vqtbl1q_u8(lo_mask_tbl, lo)); + if vminvq_u8(chk) == 0 { + return None; } + let v = translate_lane_neon::(c, hi, roll_tbl, special_ch, special_adj); + let v_next = vextq_u8::<1>(v, v); + let merged = vorrq_u8(vshlq_u8(v, dec_shl), vshlq_u8(v_next, dec_shr)); + Some(vqtbl1q_u8(merged, dec_pack)) } -/// # Safety -/// -/// Requires NEON target feature. #[inline] #[target_feature(enable = "neon")] -unsafe fn translate_lane_neon( +fn translate_lane_neon( c: uint8x16_t, hi: uint8x16_t, roll_tbl: uint8x16_t, @@ -404,13 +434,10 @@ unsafe fn translate_lane_neon( } } -/// # Safety -/// -/// Requires NEON target feature. #[inline] #[target_feature(enable = "neon")] #[allow(clippy::too_many_arguments)] -unsafe fn decode_chunk_64_neon( +fn decode_chunk_64_neon( c0: uint8x16_t, c1: uint8x16_t, c2: uint8x16_t, @@ -422,41 +449,34 @@ unsafe fn decode_chunk_64_neon( special_ch: uint8x16_t, special_adj: uint8x16_t, ) -> Option { - // SAFETY: Caller guarantees NEON is enabled. - unsafe { - let hi0 = vshrq_n_u8::<4>(c0); - let lo0 = vandq_u8(c0, mask_0f); - let chk0 = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi0), vqtbl1q_u8(lo_mask_tbl, lo0)); - - let hi1 = vshrq_n_u8::<4>(c1); - let lo1 = vandq_u8(c1, mask_0f); - let chk1 = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi1), vqtbl1q_u8(lo_mask_tbl, lo1)); + let hi0 = vshrq_n_u8::<4>(c0); + let lo0 = vandq_u8(c0, mask_0f); + let chk0 = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi0), vqtbl1q_u8(lo_mask_tbl, lo0)); - let hi2 = vshrq_n_u8::<4>(c2); - let lo2 = vandq_u8(c2, mask_0f); - let chk2 = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi2), vqtbl1q_u8(lo_mask_tbl, lo2)); + let hi1 = vshrq_n_u8::<4>(c1); + let lo1 = vandq_u8(c1, mask_0f); + let chk1 = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi1), vqtbl1q_u8(lo_mask_tbl, lo1)); - let hi3 = vshrq_n_u8::<4>(c3); - let lo3 = vandq_u8(c3, mask_0f); - let chk3 = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi3), vqtbl1q_u8(lo_mask_tbl, lo3)); + let hi2 = vshrq_n_u8::<4>(c2); + let lo2 = vandq_u8(c2, mask_0f); + let chk2 = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi2), vqtbl1q_u8(lo_mask_tbl, lo2)); - let min_chk = vminq_u8(vminq_u8(chk0, chk1), vminq_u8(chk2, chk3)); - if vminvq_u8(min_chk) == 0 { - return None; - } + let hi3 = vshrq_n_u8::<4>(c3); + let lo3 = vandq_u8(c3, mask_0f); + let chk3 = vandq_u8(vqtbl1q_u8(hi_mask_tbl, hi3), vqtbl1q_u8(lo_mask_tbl, lo3)); - let v0 = - translate_lane_neon::(c0, hi0, roll_tbl, special_ch, special_adj); - let v1 = - translate_lane_neon::(c1, hi1, roll_tbl, special_ch, special_adj); - let v2 = - translate_lane_neon::(c2, hi2, roll_tbl, special_ch, special_adj); - let v3 = - translate_lane_neon::(c3, hi3, roll_tbl, special_ch, special_adj); - - let out0 = vorrq_u8(vshlq_n_u8::<2>(v0), vshrq_n_u8::<4>(v1)); - let out1 = vorrq_u8(vshlq_n_u8::<4>(v1), vshrq_n_u8::<2>(v2)); - let out2 = vorrq_u8(vshlq_n_u8::<6>(v2), v3); - Some(uint8x16x3_t(out0, out1, out2)) + let min_chk = vminq_u8(vminq_u8(chk0, chk1), vminq_u8(chk2, chk3)); + if vminvq_u8(min_chk) == 0 { + return None; } + + let v0 = translate_lane_neon::(c0, hi0, roll_tbl, special_ch, special_adj); + let v1 = translate_lane_neon::(c1, hi1, roll_tbl, special_ch, special_adj); + let v2 = translate_lane_neon::(c2, hi2, roll_tbl, special_ch, special_adj); + let v3 = translate_lane_neon::(c3, hi3, roll_tbl, special_ch, special_adj); + + let out0 = vorrq_u8(vshlq_n_u8::<2>(v0), vshrq_n_u8::<4>(v1)); + let out1 = vorrq_u8(vshlq_n_u8::<4>(v1), vshrq_n_u8::<2>(v2)); + let out2 = vorrq_u8(vshlq_n_u8::<6>(v2), v3); + Some(uint8x16x3_t(out0, out1, out2)) } diff --git a/bases/src/backend/base64/arch/x86.rs b/bases/src/backend/base64/arch/x86.rs index 30895b1..4bd02f2 100644 --- a/bases/src/backend/base64/arch/x86.rs +++ b/bases/src/backend/base64/arch/x86.rs @@ -257,12 +257,9 @@ pub fn decode_in_place( (done_16 * 16, done_16 * 12, had_invalid) } -/// # Safety -/// -/// Requires SSSE3 support on the current CPU. #[inline] #[target_feature(enable = "ssse3")] -unsafe fn encode_128_consts() -> ( +fn encode_128_consts() -> ( __m128i, __m128i, __m128i, @@ -310,28 +307,23 @@ unsafe fn encode_chunk_12( c25: __m128i, offset_lut: __m128i, ) { - // SAFETY: Caller guarantees `in_ptr` is valid for 16 readable bytes and `out_ptr` for 16 - // writes. - unsafe { - let raw = _mm_loadu_si128(in_ptr.cast()); - let w = _mm_shuffle_epi8(raw, shuf); - let t0 = _mm_mulhi_epu16(_mm_and_si128(w, mask0), mul0); - let t1 = _mm_mullo_epi16(_mm_and_si128(w, mask1), mul1); - let indices = _mm_or_si128(t0, t1); - let result = _mm_subs_epu8(indices, c51); - // Map A-Z to slot 0, a-z to slot 1, and the remaining symbols to slots 2-13. - let key = _mm_sub_epi8(result, _mm_cmpgt_epi8(indices, c25)); - let ascii = _mm_add_epi8(indices, _mm_shuffle_epi8(offset_lut, key)); - _mm_storeu_si128(out_ptr.cast(), ascii); - } + // SAFETY: Caller guarantees `in_ptr` is valid for 16 readable bytes. + let raw = unsafe { _mm_loadu_si128(in_ptr.cast()) }; + let w = _mm_shuffle_epi8(raw, shuf); + let t0 = _mm_mulhi_epu16(_mm_and_si128(w, mask0), mul0); + let t1 = _mm_mullo_epi16(_mm_and_si128(w, mask1), mul1); + let indices = _mm_or_si128(t0, t1); + let result = _mm_subs_epu8(indices, c51); + // Map A-Z to slot 0, a-z to slot 1, and the remaining symbols to slots 2-13. + let key = _mm_sub_epi8(result, _mm_cmpgt_epi8(indices, c25)); + let ascii = _mm_add_epi8(indices, _mm_shuffle_epi8(offset_lut, key)); + // SAFETY: Caller guarantees `out_ptr` is valid for 16 writes. + unsafe { _mm_storeu_si128(out_ptr.cast(), ascii) }; } -/// # Safety -/// -/// Requires SSSE3 support on the current CPU. #[cfg(not(target_feature = "avx2"))] #[target_feature(enable = "ssse3")] -unsafe fn encode_ssse3( +fn encode_ssse3( input: &[u8], output: &mut [MaybeUninit], ) -> (usize, usize) { @@ -345,12 +337,13 @@ unsafe fn encode_ssse3( let mut in_ptr = input.as_ptr(); let mut out_ptr = output.as_mut_ptr().cast::(); - // SAFETY: `chunks > 0` implies `input.len() >= (chunks - 1) * 12 + 16` and `output.len() >= - // chunks * 16`. - unsafe { - let (shuf, mask0, mul0, mask1, mul1, c51, c25, offset_lut) = - encode_128_consts::(); - for _ in 0..chunks { + let (shuf, mask0, mul0, mask1, mul1, c51, c25, offset_lut) = + encode_128_consts::(); + for _ in 0..chunks { + // SAFETY: `chunks > 0` implies `input.len() >= (chunks - 1) * 12 + 16` and + // `output.len() >= chunks * 16`, so each iteration reads 16 bytes, writes 16 bytes, and + // stays in bounds. + unsafe { encode_chunk_12( in_ptr, out_ptr, shuf, mask0, mul0, mask1, mul1, c51, c25, offset_lut, ); @@ -361,12 +354,9 @@ unsafe fn encode_ssse3( (chunks * 12, chunks * 16) } -/// # Safety -/// -/// Requires AVX2 support on the current CPU. #[cfg(any(feature = "std", target_feature = "avx2"))] #[target_feature(enable = "avx2")] -unsafe fn encode_avx2( +fn encode_avx2( input: &[u8], output: &mut [MaybeUninit], ) -> (usize, usize) { @@ -378,44 +368,52 @@ unsafe fn encode_avx2( let mut input_index = chunks * 24; let mut output_index = chunks * 32; - // SAFETY: `chunks > 0` guarantees `input.len() >= (chunks - 1) * 24 + 28` and `output.len() >= - // chunks * 32`. The trailing 16-byte check ensures `input_index + 16 <= input.len()` and - // `output_index + 16 <= output.len()`. - unsafe { - let (sh_128, m0_128, ml0_128, m1_128, ml1_128, c51_128, c25_128, lut_128) = - encode_128_consts::(); - - if chunks > 0 { - let shuf = _mm256_set_m128i(sh_128, sh_128); - let mask0 = _mm256_set_m128i(m0_128, m0_128); - let mul0 = _mm256_set_m128i(ml0_128, ml0_128); - let mask1 = _mm256_set_m128i(m1_128, m1_128); - let mul1 = _mm256_set_m128i(ml1_128, ml1_128); - let c51 = _mm256_set_m128i(c51_128, c51_128); - let c25 = _mm256_set_m128i(c25_128, c25_128); - let offset_lut = _mm256_set_m128i(lut_128, lut_128); - - let mut in_ptr = input.as_ptr(); - let mut out_ptr = output.as_mut_ptr().cast::(); - for _ in 0..chunks { - let raw = _mm256_set_m128i( + let (sh_128, m0_128, ml0_128, m1_128, ml1_128, c51_128, c25_128, lut_128) = + encode_128_consts::(); + + if chunks > 0 { + let shuf = _mm256_set_m128i(sh_128, sh_128); + let mask0 = _mm256_set_m128i(m0_128, m0_128); + let mul0 = _mm256_set_m128i(ml0_128, ml0_128); + let mask1 = _mm256_set_m128i(m1_128, m1_128); + let mul1 = _mm256_set_m128i(ml1_128, ml1_128); + let c51 = _mm256_set_m128i(c51_128, c51_128); + let c25 = _mm256_set_m128i(c25_128, c25_128); + let offset_lut = _mm256_set_m128i(lut_128, lut_128); + + let mut in_ptr = input.as_ptr(); + let mut out_ptr = output.as_mut_ptr().cast::(); + for _ in 0..chunks { + // SAFETY: `chunks > 0` guarantees `input.len() >= (chunks - 1) * 24 + 28`, so both + // 16-byte loads at `in_ptr` and `in_ptr.add(12)` are in bounds. + let (hi_raw, lo_raw) = unsafe { + ( _mm_loadu_si128(in_ptr.add(12).cast()), _mm_loadu_si128(in_ptr.cast()), - ); - let w = _mm256_shuffle_epi8(raw, shuf); - let t0 = _mm256_mulhi_epu16(_mm256_and_si256(w, mask0), mul0); - let t1 = _mm256_mullo_epi16(_mm256_and_si256(w, mask1), mul1); - let indices = _mm256_or_si256(t0, t1); - let result = _mm256_subs_epu8(indices, c51); - let key = _mm256_sub_epi8(result, _mm256_cmpgt_epi8(indices, c25)); - let ascii = _mm256_add_epi8(indices, _mm256_shuffle_epi8(offset_lut, key)); + ) + }; + let raw = _mm256_set_m128i(hi_raw, lo_raw); + let w = _mm256_shuffle_epi8(raw, shuf); + let t0 = _mm256_mulhi_epu16(_mm256_and_si256(w, mask0), mul0); + let t1 = _mm256_mullo_epi16(_mm256_and_si256(w, mask1), mul1); + let indices = _mm256_or_si256(t0, t1); + let result = _mm256_subs_epu8(indices, c51); + let key = _mm256_sub_epi8(result, _mm256_cmpgt_epi8(indices, c25)); + let ascii = _mm256_add_epi8(indices, _mm256_shuffle_epi8(offset_lut, key)); + // SAFETY: `output.len() >= chunks * 32`, so `out_ptr` is valid for 32 writes and + // advancing both pointers stays in bounds. + unsafe { _mm256_storeu_si256(out_ptr.cast(), ascii); in_ptr = in_ptr.add(24); out_ptr = out_ptr.add(32); } } + } - if input_index + 16 <= input.len() && output_index + 16 <= output.len() { + if input_index + 16 <= input.len() && output_index + 16 <= output.len() { + // SAFETY: The guard above ensures `input_index` and `output_index` are in bounds with 16 + // readable input bytes and 16 writable output bytes, and AVX2 implies SSSE3. + unsafe { encode_chunk_12( input.as_ptr().add(input_index), output.as_mut_ptr().cast::().add(output_index), @@ -428,20 +426,17 @@ unsafe fn encode_avx2( c25_128, lut_128, ); - input_index += 12; - output_index += 16; } + input_index += 12; + output_index += 16; } (input_index, output_index) } -/// # Safety -/// -/// Requires SSSE3 support on the current CPU. #[inline] #[target_feature(enable = "ssse3")] -unsafe fn decode_128_tables() +fn decode_128_tables() -> (__m128i, __m128i, __m128i, __m128i, __m128i, __m128i) { let hi_mask = _mm_setr_epi8(0, 0, 1, 2, 4, 8, 4, 16, 0, 0, 0, 0, 0, 0, 0, 0); let lo = if URL_SAFE_ALPHABET { @@ -494,27 +489,25 @@ unsafe fn decode_chunk_16( mul_0001_1000: __m128i, zero: __m128i, ) -> Option<__m128i> { - // SAFETY: Caller guarantees `in_ptr` is valid for 16 readable bytes and SSSE3 is enabled. - unsafe { - let v = _mm_loadu_si128(in_ptr.cast()); - let hi = _mm_and_si128(_mm_srli_epi16(v, 4), mask_0f); - let lo = _mm_and_si128(v, mask_0f); - let chk = _mm_and_si128(_mm_shuffle_epi8(hi_mask, hi), _mm_shuffle_epi8(lo_mask, lo)); - if _mm_movemask_epi8(_mm_cmpeq_epi8(chk, zero)) != 0 { - return None; - } - - let base = _mm_add_epi8(v, _mm_shuffle_epi8(roll_tbl, hi)); - let adj = _mm_and_si128(_mm_cmpeq_epi8(v, special_ch), special_adj); - let vals = if URL_SAFE_ALPHABET { - _mm_add_epi8(base, adj) - } else { - _mm_sub_epi8(base, adj) - }; - let merged = _mm_maddubs_epi16(vals, mul_0140); - let packed = _mm_madd_epi16(merged, mul_0001_1000); - Some(_mm_shuffle_epi8(packed, pack_shuf)) + // SAFETY: Caller guarantees `in_ptr` is valid for 16 readable bytes. + let v = unsafe { _mm_loadu_si128(in_ptr.cast()) }; + let hi = _mm_and_si128(_mm_srli_epi16(v, 4), mask_0f); + let lo = _mm_and_si128(v, mask_0f); + let chk = _mm_and_si128(_mm_shuffle_epi8(hi_mask, hi), _mm_shuffle_epi8(lo_mask, lo)); + if _mm_movemask_epi8(_mm_cmpeq_epi8(chk, zero)) != 0 { + return None; } + + let base = _mm_add_epi8(v, _mm_shuffle_epi8(roll_tbl, hi)); + let adj = _mm_and_si128(_mm_cmpeq_epi8(v, special_ch), special_adj); + let vals = if URL_SAFE_ALPHABET { + _mm_add_epi8(base, adj) + } else { + _mm_sub_epi8(base, adj) + }; + let merged = _mm_maddubs_epi16(vals, mul_0140); + let packed = _mm_madd_epi16(merged, mul_0001_1000); + Some(_mm_shuffle_epi8(packed, pack_shuf)) } /// # Safety @@ -523,13 +516,11 @@ unsafe fn decode_chunk_16( #[inline] #[target_feature(enable = "ssse3")] unsafe fn store_12_ssse3(out_ptr: *mut u8, res: __m128i) { - // SAFETY: Caller guarantees `out_ptr` is valid for 12 non-overlapping writes. + let hi32 = _mm_cvtsi128_si32(_mm_srli_si128::<8>(res)); + // SAFETY: Caller guarantees `out_ptr` is valid for 12 writes. unsafe { _mm_storel_epi64(out_ptr.cast(), res); - ptr::write_unaligned( - out_ptr.add(8).cast::(), - _mm_cvtsi128_si32(_mm_srli_si128::<8>(res)), - ); + ptr::write_unaligned(out_ptr.add(8).cast::(), hi32); } } @@ -550,18 +541,17 @@ unsafe fn decode_ssse3( ) -> (usize, bool) { let wide_chunks = (out_len.saturating_sub(4) / 12).min(chunks); let mut done = 0; - // SAFETY: Caller guarantees `chunks * 16` reads at `in_ptr`, `out_len` writes at `out_ptr`, and - // non-overtaking pointer advancement for in-place decoding. - unsafe { - let (hi_mask, lo_mask, roll_tbl, pack_shuf, special_ch, special_adj) = - decode_128_tables::(); - let mask_0f = _mm_set1_epi8(0x0f); - let mul_0140 = _mm_set1_epi32(0x0140_0140); - let mul_0001_1000 = _mm_set1_epi32(0x0001_1000); - let zero = _mm_setzero_si128(); - - while done < wide_chunks { - let Some(res) = decode_chunk_16::( + let (hi_mask, lo_mask, roll_tbl, pack_shuf, special_ch, special_adj) = + decode_128_tables::(); + let mask_0f = _mm_set1_epi8(0x0f); + let mul_0140 = _mm_set1_epi32(0x0140_0140); + let mul_0001_1000 = _mm_set1_epi32(0x0001_1000); + let zero = _mm_setzero_si128(); + + while done < wide_chunks { + // SAFETY: `done < wide_chunks <= chunks` guarantees 16 readable bytes at `in_ptr`. + let Some(res) = (unsafe { + decode_chunk_16::( in_ptr, hi_mask, lo_mask, @@ -573,17 +563,25 @@ unsafe fn decode_ssse3( mul_0140, mul_0001_1000, zero, - ) else { - return (done, true); - }; + ) + }) else { + return (done, true); + }; + // SAFETY: `done < wide_chunks <= chunks` guarantees `done * 12 + 16 <= out_len`, in-place + // writes lag behind `in_ptr`, and advancing `in_ptr`/`out_ptr` by 16 and 12 bytes stays + // within both buffers. + unsafe { _mm_storeu_si128(out_ptr.cast(), res); in_ptr = in_ptr.add(16); out_ptr = out_ptr.add(12); - done += 1; } + done += 1; + } - if done < chunks { - let Some(res) = decode_chunk_16::( + if done < chunks { + // SAFETY: `done < chunks` guarantees 16 readable bytes at `in_ptr`. + let Some(res) = (unsafe { + decode_chunk_16::( in_ptr, hi_mask, lo_mask, @@ -595,12 +593,14 @@ unsafe fn decode_ssse3( mul_0140, mul_0001_1000, zero, - ) else { - return (done, true); - }; - store_12_ssse3(out_ptr, res); - done += 1; - } + ) + }) else { + return (done, true); + }; + // SAFETY: `done < chunks` guarantees 12 writable bytes at `out_ptr`, and this chunk's input + // was already loaded so in-place writes do not overwrite unread input. + unsafe { store_12_ssse3(out_ptr, res) }; + done += 1; } (done, false) } @@ -626,30 +626,28 @@ unsafe fn decode_chunk_32( mul_0001_1000: __m256i, zero: __m256i, ) -> Option<__m256i> { - // SAFETY: Caller guarantees `in_ptr` is valid for 32 readable bytes and AVX2 is enabled. - unsafe { - let v = _mm256_loadu_si256(in_ptr.cast()); - let hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), mask_0f); - let lo = _mm256_and_si256(v, mask_0f); - let chk = _mm256_and_si256( - _mm256_shuffle_epi8(hi_mask, hi), - _mm256_shuffle_epi8(lo_mask, lo), - ); - if _mm256_movemask_epi8(_mm256_cmpeq_epi8(chk, zero)) != 0 { - return None; - } - - let base = _mm256_add_epi8(v, _mm256_shuffle_epi8(roll_tbl, hi)); - let adj = _mm256_and_si256(_mm256_cmpeq_epi8(v, special_ch), special_adj); - let vals = if URL_SAFE_ALPHABET { - _mm256_add_epi8(base, adj) - } else { - _mm256_sub_epi8(base, adj) - }; - let merged = _mm256_maddubs_epi16(vals, mul_0140); - let packed = _mm256_madd_epi16(merged, mul_0001_1000); - Some(_mm256_shuffle_epi8(packed, pack_shuf)) + // SAFETY: Caller guarantees `in_ptr` is valid for 32 readable bytes. + let v = unsafe { _mm256_loadu_si256(in_ptr.cast()) }; + let hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), mask_0f); + let lo = _mm256_and_si256(v, mask_0f); + let chk = _mm256_and_si256( + _mm256_shuffle_epi8(hi_mask, hi), + _mm256_shuffle_epi8(lo_mask, lo), + ); + if _mm256_movemask_epi8(_mm256_cmpeq_epi8(chk, zero)) != 0 { + return None; } + + let base = _mm256_add_epi8(v, _mm256_shuffle_epi8(roll_tbl, hi)); + let adj = _mm256_and_si256(_mm256_cmpeq_epi8(v, special_ch), special_adj); + let vals = if URL_SAFE_ALPHABET { + _mm256_add_epi8(base, adj) + } else { + _mm256_sub_epi8(base, adj) + }; + let merged = _mm256_maddubs_epi16(vals, mul_0140); + let packed = _mm256_madd_epi16(merged, mul_0001_1000); + Some(_mm256_shuffle_epi8(packed, pack_shuf)) } /// # Safety @@ -659,11 +657,12 @@ unsafe fn decode_chunk_32( #[inline] #[target_feature(enable = "avx2")] unsafe fn store_24_avx2(out_ptr: *mut u8, res: __m256i) { - // SAFETY: Caller guarantees `out_ptr` is valid for 24 writes, and `res` holds bytes `0..16` - // in its low lane and bytes `16..24` in the low 8 bytes of its high lane. + let lo = _mm256_castsi256_si128(res); + let hi = _mm256_extracti128_si256::<1>(res); + // SAFETY: Caller guarantees `out_ptr` is valid for 24 writes. unsafe { - _mm_storeu_si128(out_ptr.cast(), _mm256_castsi256_si128(res)); - _mm_storel_epi64(out_ptr.add(16).cast(), _mm256_extracti128_si256::<1>(res)); + _mm_storeu_si128(out_ptr.cast(), lo); + _mm_storel_epi64(out_ptr.add(16).cast(), hi); } } @@ -687,25 +686,24 @@ unsafe fn decode_avx2( let wide_32 = (out_len.saturating_sub(4) / 24).min(chunks_32); let mut done_16 = 0; - // SAFETY: Caller guarantees `chunks_16 * 16` reads at `in_ptr`, `out_len` writes at `out_ptr`, - // and non-overtaking pointer advancement for in-place decoding. - unsafe { - let (hi_128, lo_128, roll_128, shuf_128, spec_ch_128, spec_adj_128) = - decode_128_tables::(); - if chunks_32 > 0 { - let hi_mask = _mm256_set_m128i(hi_128, hi_128); - let lo_mask = _mm256_set_m128i(lo_128, lo_128); - let roll_tbl = _mm256_set_m128i(roll_128, roll_128); - let pack_shuf = _mm256_set_m128i(shuf_128, shuf_128); - let special_ch = _mm256_set_m128i(spec_ch_128, spec_ch_128); - let special_adj = _mm256_set_m128i(spec_adj_128, spec_adj_128); - let mask_0f = _mm256_set1_epi8(0x0f); - let mul_0140 = _mm256_set1_epi32(0x0140_0140); - let mul_0001_1000 = _mm256_set1_epi32(0x0001_1000); - let zero = _mm256_setzero_si256(); - - for _ in 0..wide_32 { - let Some(shuf) = decode_chunk_32::( + let (hi_128, lo_128, roll_128, shuf_128, spec_ch_128, spec_adj_128) = + decode_128_tables::(); + if chunks_32 > 0 { + let hi_mask = _mm256_set_m128i(hi_128, hi_128); + let lo_mask = _mm256_set_m128i(lo_128, lo_128); + let roll_tbl = _mm256_set_m128i(roll_128, roll_128); + let pack_shuf = _mm256_set_m128i(shuf_128, shuf_128); + let special_ch = _mm256_set_m128i(spec_ch_128, spec_ch_128); + let special_adj = _mm256_set_m128i(spec_adj_128, spec_adj_128); + let mask_0f = _mm256_set1_epi8(0x0f); + let mul_0140 = _mm256_set1_epi32(0x0140_0140); + let mul_0001_1000 = _mm256_set1_epi32(0x0001_1000); + let zero = _mm256_setzero_si256(); + + for _ in 0..wide_32 { + // SAFETY: `done_16 + 2 <= chunks_16` guarantees 32 readable bytes at `in_ptr`. + let Some(shuf) = (unsafe { + decode_chunk_32::( in_ptr, hi_mask, lo_mask, @@ -717,18 +715,28 @@ unsafe fn decode_avx2( mul_0140, mul_0001_1000, zero, - ) else { - return (done_16, true); - }; - _mm_storeu_si128(out_ptr.cast(), _mm256_castsi256_si128(shuf)); - _mm_storeu_si128(out_ptr.add(12).cast(), _mm256_extracti128_si256::<1>(shuf)); + ) + }) else { + return (done_16, true); + }; + let lo = _mm256_castsi256_si128(shuf); + let hi = _mm256_extracti128_si256::<1>(shuf); + // SAFETY: `done_16 / 2 < wide_32 <= chunks_32` guarantees `done_16 + 2 <= chunks_16` + // and `done_16 * 12 + 28 <= out_len`, in-place writes lag behind `in_ptr`, and + // advancing `in_ptr`/`out_ptr` by 32 and 24 bytes stays within both buffers. + unsafe { + _mm_storeu_si128(out_ptr.cast(), lo); + _mm_storeu_si128(out_ptr.add(12).cast(), hi); in_ptr = in_ptr.add(32); out_ptr = out_ptr.add(24); - done_16 += 2; } + done_16 += 2; + } - if wide_32 < chunks_32 { - let Some(shuf) = decode_chunk_32::( + if wide_32 < chunks_32 { + // SAFETY: `done_16 + 2 <= chunks_16` guarantees 32 readable bytes at `in_ptr`. + let Some(shuf) = (unsafe { + decode_chunk_32::( in_ptr, hi_mask, lo_mask, @@ -740,25 +748,34 @@ unsafe fn decode_avx2( mul_0140, mul_0001_1000, zero, - ) else { - return (done_16, true); - }; - let lane_perm = _mm256_setr_epi32(0, 1, 2, 4, 5, 6, 3, 7); - let res = _mm256_permutevar8x32_epi32(shuf, lane_perm); + ) + }) else { + return (done_16, true); + }; + let lane_perm = _mm256_setr_epi32(0, 1, 2, 4, 5, 6, 3, 7); + let res = _mm256_permutevar8x32_epi32(shuf, lane_perm); + // SAFETY: `done_16 + 2 <= chunks_16` guarantees 24 writable bytes at `out_ptr`, + // in-place writes lag behind `in_ptr`, and advancing `in_ptr`/`out_ptr` by 32 and 24 + // bytes stays within both buffers. + unsafe { store_24_avx2(out_ptr, res); in_ptr = in_ptr.add(32); out_ptr = out_ptr.add(24); - done_16 += 2; } + done_16 += 2; } + } - let mut had_invalid = false; - if done_16 < chunks_16 { - let mask_0f = _mm_set1_epi8(0x0f); - let mul_0140 = _mm_set1_epi32(0x0140_0140); - let mul_0001_1000 = _mm_set1_epi32(0x0001_1000); - let zero = _mm_setzero_si128(); - if let Some(res) = decode_chunk_16::( + let mut had_invalid = false; + if done_16 < chunks_16 { + let mask_0f = _mm_set1_epi8(0x0f); + let mul_0140 = _mm_set1_epi32(0x0140_0140); + let mul_0001_1000 = _mm_set1_epi32(0x0001_1000); + let zero = _mm_setzero_si128(); + // SAFETY: `done_16 < chunks_16` guarantees 16 readable bytes at `in_ptr`, and AVX2 implies + // SSSE3. + if let Some(res) = unsafe { + decode_chunk_16::( in_ptr, hi_128, lo_128, @@ -770,17 +787,21 @@ unsafe fn decode_avx2( mul_0140, mul_0001_1000, zero, - ) { - if out_len.saturating_sub(done_16 * 12) >= 16 { - _mm_storeu_si128(out_ptr.cast(), res); - } else { - store_12_ssse3(out_ptr, res); - } - done_16 += 1; + ) + } { + if out_len.saturating_sub(done_16 * 12) >= 16 { + // SAFETY: The guard above ensures 16 writable bytes at `out_ptr`, and in-place + // writes lag behind `in_ptr`. + unsafe { _mm_storeu_si128(out_ptr.cast(), res) }; } else { - had_invalid = true; + // SAFETY: `done_16 < chunks_16` guarantees 12 writable bytes at `out_ptr`, this + // chunk's input was already loaded, and AVX2 implies SSSE3. + unsafe { store_12_ssse3(out_ptr, res) }; } + done_16 += 1; + } else { + had_invalid = true; } - (done_16, had_invalid) } + (done_16, had_invalid) }