From 9f68905c75687f88841933de5ae9531d9c62d4e0 Mon Sep 17 00:00:00 2001 From: Michael Yarichuk Date: Thu, 30 Apr 2026 22:51:25 +0300 Subject: [PATCH 1/5] fix: use EqualityComparer for POCO hashing to ignore padding --- src/SharpArena/Collections/Hashing.cs | 3 +-- .../Collections/ArenaDictionaryTests.cs | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/SharpArena/Collections/Hashing.cs b/src/SharpArena/Collections/Hashing.cs index af5070a..15cf4b7 100644 --- a/src/SharpArena/Collections/Hashing.cs +++ b/src/SharpArena/Collections/Hashing.cs @@ -18,8 +18,7 @@ public static uint Hash(T value) where T : unmanaged return (uint)Unsafe.As(ref value).GetHashCode(); } - var span = new ReadOnlySpan(&value, sizeof(T)); - return (uint)XxHash3.HashToUInt64(span); + return (uint)EqualityComparer.Default.GetHashCode(value); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs index e688905..15e195e 100644 --- a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs +++ b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs @@ -1,3 +1,4 @@ +using System.Runtime.InteropServices; using FluentAssertions; using SharpArena.Allocators; using SharpArena.Collections; @@ -5,12 +6,31 @@ namespace SharpArena.Tests.Collections; +[StructLayout(LayoutKind.Sequential)] +public struct PaddedKey : IEquatable +{ + public byte A; + public int B; // padding exists between A and B + public bool Equals(PaddedKey other) => A == other.A && B == other.B; + public override int GetHashCode() => HashCode.Combine(A, B); +} + public class ArenaDictionaryTests : IDisposable { private readonly ArenaAllocator _arena = new(); public void Dispose() => _arena.Dispose(); + [Fact] + public void PaddedStructHashing_IgnoresPadding() + { + var dict = new ArenaDictionary(_arena); + + var key1 = new PaddedKey { A = 1, B = 2 }; + dict.Add(key1, 42); + dict.ContainsKey(key1).Should().BeTrue(); + } + [Fact] public void Add_NewEntry_IncrementsCountAndEnablesLookup() { From 75f77e480084fc0899c6bda2c47678dbe96ddf01 Mon Sep 17 00:00:00 2001 From: Michael Yarichuk Date: Thu, 30 Apr 2026 22:54:47 +0300 Subject: [PATCH 2/5] feat: add ReadOnlySpan overloads for ArenaUtf8String keys --- src/SharpArena/Collections/ArenaDictionary.cs | 61 +++++++++++++++++++ .../Collections/ArenaDictionaryTests.cs | 23 +++++++ 2 files changed, 84 insertions(+) diff --git a/src/SharpArena/Collections/ArenaDictionary.cs b/src/SharpArena/Collections/ArenaDictionary.cs index 80e01fd..0449c08 100644 --- a/src/SharpArena/Collections/ArenaDictionary.cs +++ b/src/SharpArena/Collections/ArenaDictionary.cs @@ -267,6 +267,33 @@ public bool ContainsKey(ReadOnlySpan key) } } + /// + /// Specialized ContainsKey for ArenaUtf8String using ReadOnlySpan{byte} to avoid allocations. + /// + /// The key to locate in the . + /// if the contains an element with the key; otherwise, . + public bool ContainsKey(ReadOnlySpan key) + { + if (typeof(TKey) == typeof(ArenaUtf8String)) + { + CheckAlive(); + uint capacity = (uint)_header->Capacity; + int* buckets = _header->Buckets; + ArenaUtf8String* keys = (ArenaUtf8String*)_header->Keys; + uint mask = capacity - 1; + uint hash = Hashing.HashUtf8(key); + uint index = hash & mask; + while (true) + { + int entryIdxPlusOne = buckets[index]; + if (entryIdxPlusOne == 0) return false; + if (keys[entryIdxPlusOne - 1].Equals(key)) return true; + index = (index + 1) & mask; + } + } + return false; + } + /// /// Gets the value associated with the specified key. /// @@ -326,6 +353,40 @@ public bool TryGetValue(ReadOnlySpan key, out TValue value) return false; } + /// + /// Specialized TryGetValue for ArenaUtf8String using ReadOnlySpan{byte} to avoid allocations. + /// + /// The key whose value to get. + /// When this method returns, the value associated with the specified key, if the key is found; otherwise, the default value for the type of the parameter. This parameter is passed uninitialized. + /// if the contains an element with the specified key; otherwise, . + public bool TryGetValue(ReadOnlySpan key, out TValue value) + { + if (typeof(TKey) == typeof(ArenaUtf8String)) + { + CheckAlive(); + uint capacity = (uint)_header->Capacity; + int* buckets = _header->Buckets; + ArenaUtf8String* keys = (ArenaUtf8String*)_header->Keys; + TValue* values = (TValue*)_header->Values; + uint mask = capacity - 1; + uint hash = Hashing.HashUtf8(key); + uint index = hash & mask; + while (true) + { + int entryIdxPlusOne = buckets[index]; + if (entryIdxPlusOne == 0) break; + if (keys[entryIdxPlusOne - 1].Equals(key)) + { + value = values[entryIdxPlusOne - 1]; + return true; + } + index = (index + 1) & mask; + } + } + value = default; + return false; + } + /// /// Removes all keys and values from the . /// diff --git a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs index 15e195e..27cc57f 100644 --- a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs +++ b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs @@ -130,6 +130,29 @@ public void ArenaUtf16String_AsKey_WorksCorrectly() dict.ContainsKey(ArenaUtf16String.Clone("key3", _arena)).Should().BeFalse(); } + [Fact] + public void ContainsKey_Utf8ByteSpan_Works() + { + var dict = new ArenaDictionary(_arena); + var key = ArenaUtf8String.Clone("test", _arena); + dict.Add(key, 123); + + ReadOnlySpan query = "test"u8; + dict.ContainsKey(query).Should().BeTrue(); + } + + [Fact] + public void TryGetValue_Utf8ByteSpan_Works() + { + var dict = new ArenaDictionary(_arena); + var key = ArenaUtf8String.Clone("test", _arena); + dict.Add(key, 123); + + ReadOnlySpan query = "test"u8; + dict.TryGetValue(query, out var val).Should().BeTrue(); + val.Should().Be(123); + } + [Fact] public void Clear_ResetsCountAndLookups() { From 7cb5d048d48351604bcca100c2e843ee85f893ab Mon Sep 17 00:00:00 2001 From: Michael Yarichuk Date: Thu, 30 Apr 2026 22:59:25 +0300 Subject: [PATCH 3/5] feat: support char span queries on Utf8 dictionaries via transcoding --- src/SharpArena/Collections/ArenaDictionary.cs | 143 +++++++++++++----- .../Collections/ArenaDictionaryTests.cs | 30 ++++ 2 files changed, 138 insertions(+), 35 deletions(-) diff --git a/src/SharpArena/Collections/ArenaDictionary.cs b/src/SharpArena/Collections/ArenaDictionary.cs index 0449c08..9fd9783 100644 --- a/src/SharpArena/Collections/ArenaDictionary.cs +++ b/src/SharpArena/Collections/ArenaDictionary.cs @@ -1,4 +1,6 @@ using System.Collections; +using System.Buffers; +using System.Text; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -242,29 +244,65 @@ public bool ContainsKey(TKey key) } /// - /// Specialized ContainsKey for ArenaUtf16String using ReadOnlySpan{char} to avoid allocations. + /// Specialized ContainsKey for string types using ReadOnlySpan{char} to avoid allocations. + /// Supports cross-encoding for ArenaUtf8String. /// /// The key to locate in the . /// if the contains an element with the key; otherwise, . public bool ContainsKey(ReadOnlySpan key) { - if (typeof(TKey) != typeof(ArenaUtf16String)) return false; - CheckAlive(); - - uint capacity = (uint)_header->Capacity; - int* buckets = _header->Buckets; - ArenaUtf16String* keys = (ArenaUtf16String*)_header->Keys; - uint mask = capacity - 1; - uint hash = Hashing.HashString(key); - uint index = hash & mask; + if (typeof(TKey) == typeof(ArenaUtf16String)) + { + CheckAlive(); + + uint capacity = (uint)_header->Capacity; + int* buckets = _header->Buckets; + ArenaUtf16String* keys = (ArenaUtf16String*)_header->Keys; + uint mask = capacity - 1; + uint hash = Hashing.HashString(key); + uint index = hash & mask; - while (true) + while (true) + { + int entryIdxPlusOne = buckets[index]; + if (entryIdxPlusOne == 0) return false; + if (keys[entryIdxPlusOne - 1].Equals(key)) return true; + index = (index + 1) & mask; + } + } + + if (typeof(TKey) == typeof(ArenaUtf8String)) { - int entryIdxPlusOne = buckets[index]; - if (entryIdxPlusOne == 0) return false; - if (keys[entryIdxPlusOne - 1].Equals(key)) return true; - index = (index + 1) & mask; + CheckAlive(); + int maxBytes = Encoding.UTF8.GetMaxByteCount(key.Length); + byte[]? rented = null; + Span buffer = maxBytes <= 512 ? stackalloc byte[512] : (rented = ArrayPool.Shared.Rent(maxBytes)); + try + { + int written = Encoding.UTF8.GetBytes(key, buffer); + ReadOnlySpan byteKey = buffer.Slice(0, written); + + uint capacity = (uint)_header->Capacity; + int* buckets = _header->Buckets; + ArenaUtf8String* keys = (ArenaUtf8String*)_header->Keys; + uint mask = capacity - 1; + uint hash = Hashing.HashUtf8(byteKey); + uint index = hash & mask; + while (true) + { + int entryIdxPlusOne = buckets[index]; + if (entryIdxPlusOne == 0) return false; + if (keys[entryIdxPlusOne - 1].Equals(byteKey)) return true; + index = (index + 1) & mask; + } + } + finally + { + if (rented != null) ArrayPool.Shared.Return(rented); + } } + + return false; } /// @@ -315,38 +353,73 @@ public bool TryGetValue(TKey key, out TValue value) } /// - /// Specialized TryGetValue for ArenaUtf16String using ReadOnlySpan{char} to avoid allocations. + /// Specialized TryGetValue for string types using ReadOnlySpan{char} to avoid allocations. + /// Supports cross-encoding for ArenaUtf8String. /// /// The key whose value to get. /// When this method returns, the value associated with the specified key, if the key is found; otherwise, the default value for the type of the parameter. This parameter is passed uninitialized. /// if the contains an element with the specified key; otherwise, . public bool TryGetValue(ReadOnlySpan key, out TValue value) { - if (typeof(TKey) != typeof(ArenaUtf16String)) + if (typeof(TKey) == typeof(ArenaUtf16String)) { - value = default; - return false; - } - CheckAlive(); + CheckAlive(); - uint capacity = (uint)_header->Capacity; - int* buckets = _header->Buckets; - ArenaUtf16String* keys = (ArenaUtf16String*)_header->Keys; - TValue* values = (TValue*)_header->Values; - uint mask = capacity - 1; - uint hash = Hashing.HashString(key); - uint index = hash & mask; + uint capacity = (uint)_header->Capacity; + int* buckets = _header->Buckets; + ArenaUtf16String* keys = (ArenaUtf16String*)_header->Keys; + TValue* values = (TValue*)_header->Values; + uint mask = capacity - 1; + uint hash = Hashing.HashString(key); + uint index = hash & mask; - while (true) + while (true) + { + int entryIdxPlusOne = buckets[index]; + if (entryIdxPlusOne == 0) break; + if (keys[entryIdxPlusOne - 1].Equals(key)) + { + value = values[entryIdxPlusOne - 1]; + return true; + } + index = (index + 1) & mask; + } + } + + if (typeof(TKey) == typeof(ArenaUtf8String)) { - int entryIdxPlusOne = buckets[index]; - if (entryIdxPlusOne == 0) break; - if (keys[entryIdxPlusOne - 1].Equals(key)) + CheckAlive(); + int maxBytes = Encoding.UTF8.GetMaxByteCount(key.Length); + byte[]? rented = null; + Span buffer = maxBytes <= 512 ? stackalloc byte[512] : (rented = ArrayPool.Shared.Rent(maxBytes)); + try { - value = values[entryIdxPlusOne - 1]; - return true; + int written = Encoding.UTF8.GetBytes(key, buffer); + ReadOnlySpan byteKey = buffer.Slice(0, written); + + uint capacity = (uint)_header->Capacity; + int* buckets = _header->Buckets; + ArenaUtf8String* keys = (ArenaUtf8String*)_header->Keys; + TValue* values = (TValue*)_header->Values; + uint mask = capacity - 1; + uint hash = Hashing.HashUtf8(byteKey); + uint index = hash & mask; + while (true) + { + int entryIdxPlusOne = buckets[index]; + if (entryIdxPlusOne == 0) break; + if (keys[entryIdxPlusOne - 1].Equals(byteKey)) + { + value = values[entryIdxPlusOne - 1]; + return true; + } + index = (index + 1) & mask; + } + } + finally + { + if (rented != null) ArrayPool.Shared.Return(rented); } - index = (index + 1) & mask; } value = default; diff --git a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs index 27cc57f..f1c3431 100644 --- a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs +++ b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs @@ -153,6 +153,36 @@ public void TryGetValue_Utf8ByteSpan_Works() val.Should().Be(123); } + [Fact] + public void ContainsKey_CharSpan_OnUtf8Dict_Works() + { + var dict = new ArenaDictionary(_arena); + dict.Add(ArenaUtf8String.Clone("hello", _arena), 99); + + dict.ContainsKey("hello".AsSpan()).Should().BeTrue(); + } + + [Fact] + public void TryGetValue_CharSpan_OnUtf8Dict_Works() + { + var dict = new ArenaDictionary(_arena); + dict.Add(ArenaUtf8String.Clone("hello", _arena), 99); + + dict.TryGetValue("hello".AsSpan(), out var val).Should().BeTrue(); + val.Should().Be(99); + } + + [Fact] + public void TryGetValue_LargeCharSpan_OnUtf8Dict_Works() + { + var dict = new ArenaDictionary(_arena); + string largeKey = new string('a', 600); + dict.Add(ArenaUtf8String.Clone(largeKey, _arena), 1234); + + dict.TryGetValue(largeKey.AsSpan(), out var val).Should().BeTrue(); + val.Should().Be(1234); + } + [Fact] public void Clear_ResetsCountAndLookups() { From 8a6d190dbccca0dd309b9474c1abf3bb431a3872 Mon Sep 17 00:00:00 2001 From: Michael Yarichuk Date: Thu, 30 Apr 2026 23:03:37 +0300 Subject: [PATCH 4/5] feat: support byte span queries on Utf16 dictionaries via transcoding --- src/SharpArena/Collections/ArenaDictionary.cs | 71 +++++++++++++++++++ .../Collections/ArenaDictionaryTests.cs | 21 ++++++ 2 files changed, 92 insertions(+) diff --git a/src/SharpArena/Collections/ArenaDictionary.cs b/src/SharpArena/Collections/ArenaDictionary.cs index 9fd9783..4d914ca 100644 --- a/src/SharpArena/Collections/ArenaDictionary.cs +++ b/src/SharpArena/Collections/ArenaDictionary.cs @@ -329,6 +329,39 @@ public bool ContainsKey(ReadOnlySpan key) index = (index + 1) & mask; } } + + if (typeof(TKey) == typeof(ArenaUtf16String)) + { + CheckAlive(); + int maxChars = Encoding.UTF8.GetMaxCharCount(key.Length); + char[]? rented = null; + Span buffer = maxChars <= 512 ? stackalloc char[512] : (rented = ArrayPool.Shared.Rent(maxChars)); + try + { + int written = Encoding.UTF8.GetChars(key, buffer); + ReadOnlySpan charKey = buffer.Slice(0, written); + + uint capacity = (uint)_header->Capacity; + int* buckets = _header->Buckets; + ArenaUtf16String* keys = (ArenaUtf16String*)_header->Keys; + uint mask = capacity - 1; + uint hash = Hashing.HashString(charKey); + uint index = hash & mask; + + while (true) + { + int entryIdxPlusOne = buckets[index]; + if (entryIdxPlusOne == 0) return false; + if (keys[entryIdxPlusOne - 1].Equals(charKey)) return true; + index = (index + 1) & mask; + } + } + finally + { + if (rented != null) ArrayPool.Shared.Return(rented); + } + } + return false; } @@ -456,6 +489,44 @@ public bool TryGetValue(ReadOnlySpan key, out TValue value) index = (index + 1) & mask; } } + + if (typeof(TKey) == typeof(ArenaUtf16String)) + { + CheckAlive(); + int maxChars = Encoding.UTF8.GetMaxCharCount(key.Length); + char[]? rented = null; + Span buffer = maxChars <= 512 ? stackalloc char[512] : (rented = ArrayPool.Shared.Rent(maxChars)); + try + { + int written = Encoding.UTF8.GetChars(key, buffer); + ReadOnlySpan charKey = buffer.Slice(0, written); + + uint capacity = (uint)_header->Capacity; + int* buckets = _header->Buckets; + ArenaUtf16String* keys = (ArenaUtf16String*)_header->Keys; + TValue* values = (TValue*)_header->Values; + uint mask = capacity - 1; + uint hash = Hashing.HashString(charKey); + uint index = hash & mask; + + while (true) + { + int entryIdxPlusOne = buckets[index]; + if (entryIdxPlusOne == 0) break; + if (keys[entryIdxPlusOne - 1].Equals(charKey)) + { + value = values[entryIdxPlusOne - 1]; + return true; + } + index = (index + 1) & mask; + } + } + finally + { + if (rented != null) ArrayPool.Shared.Return(rented); + } + } + value = default; return false; } diff --git a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs index f1c3431..dace6e8 100644 --- a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs +++ b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs @@ -183,6 +183,27 @@ public void TryGetValue_LargeCharSpan_OnUtf8Dict_Works() val.Should().Be(1234); } + [Fact] + public void ContainsKey_ByteSpan_OnUtf16Dict_Works() + { + var dict = new ArenaDictionary(_arena); + dict.Add(ArenaUtf16String.Clone("world", _arena), 77); + + ReadOnlySpan query = "world"u8; + dict.ContainsKey(query).Should().BeTrue(); + } + + [Fact] + public void TryGetValue_ByteSpan_OnUtf16Dict_Works() + { + var dict = new ArenaDictionary(_arena); + dict.Add(ArenaUtf16String.Clone("world", _arena), 77); + + ReadOnlySpan query = "world"u8; + dict.TryGetValue(query, out var val).Should().BeTrue(); + val.Should().Be(77); + } + [Fact] public void Clear_ResetsCountAndLookups() { From e3cc13fda7f90b86b19d054f81ac6eed5c620d34 Mon Sep 17 00:00:00 2001 From: Michael Yarichuk Date: Thu, 30 Apr 2026 23:07:07 +0300 Subject: [PATCH 5/5] test: add stress tests for large string transcoding in ArenaDictionary --- .../Collections/ArenaDictionaryTests.cs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs index dace6e8..19ea570 100644 --- a/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs +++ b/tests/SharpArena.Tests/Collections/ArenaDictionaryTests.cs @@ -1,4 +1,5 @@ using System.Runtime.InteropServices; +using System.Text; using FluentAssertions; using SharpArena.Allocators; using SharpArena.Collections; @@ -204,6 +205,57 @@ public void TryGetValue_ByteSpan_OnUtf16Dict_Works() val.Should().Be(77); } + [Fact] + public void TryGetValue_LargeByteSpan_OnUtf16Dict_Works() + { + var dict = new ArenaDictionary(_arena); + string largeKey = new string('b', 600); + dict.Add(ArenaUtf16String.Clone(largeKey, _arena), 5678); + + ReadOnlySpan query = Encoding.UTF8.GetBytes(largeKey); + dict.TryGetValue(query, out var val).Should().BeTrue(); + val.Should().Be(5678); + } + + [Fact] + public void ContainsKey_LargeByteSpan_OnUtf16Dict_Works() + { + var dict = new ArenaDictionary(_arena); + string largeKey = new string('c', 600); + dict.Add(ArenaUtf16String.Clone(largeKey, _arena), 999); + + ReadOnlySpan query = Encoding.UTF8.GetBytes(largeKey); + dict.ContainsKey(query).Should().BeTrue(); + } + + [Fact] + public void StressTest_CrossEncoding_LargeStrings() + { + var dictUtf8 = new ArenaDictionary(_arena); + var dictUtf16 = new ArenaDictionary(_arena); + + for (int i = 0; i < 100; i++) + { + string key = new string((char)('a' + (i % 26)), 513 + i); + dictUtf8.Add(ArenaUtf8String.Clone(key, _arena), i); + dictUtf16.Add(ArenaUtf16String.Clone(key, _arena), i); + } + + for (int i = 0; i < 100; i++) + { + string key = new string((char)('a' + (i % 26)), 513 + i); + + // Char -> Utf8 + dictUtf8.TryGetValue(key.AsSpan(), out var val1).Should().BeTrue(); + val1.Should().Be(i); + + // Byte -> Utf16 + ReadOnlySpan keyBytes = Encoding.UTF8.GetBytes(key); + dictUtf16.TryGetValue(keyBytes, out var val2).Should().BeTrue(); + val2.Should().Be(i); + } + } + [Fact] public void Clear_ResetsCountAndLookups() {