From 99154655086fd56a851390060514e09a33e6c7c2 Mon Sep 17 00:00:00 2001 From: Imran Akram Date: Mon, 7 Sep 2026 14:03:45 +0200 Subject: [PATCH 1/4] Add batch read/write API to LocalDictionary Every indexer, ContainsKey and TryGetValue call is its own SQL round trip, so a caller looping over keys pays a round trip per key. At 100 000 keys that is ~92 s of reads and ~78 s of writes against a warm database, which is what the fee run in MMS-8327 was spending most of its time on. PersistentBlobCache already had the batch primitives - Get(IEnumerable) and Insert(IDictionary), chunking at 950 keys per statement - but LocalDictionary exposed no way to reach them. This adds: IBulkDictionary GetRange(keys) / SetRange(items), a separate interface so a consumer holding only the interface can test for it LocalDictionary implements it, routing to the batch primitives DictionaryExtensions the same two methods on IDictionary, taking the batch path when available and looping otherwise In blocks of 1 000 the reads drop to 1.27 s. The writes drop to 75.94 s, of which about 74 s is JSON serialization rather than SQLite - that half is fixed by Xrm.Json.Serialization 1.2026.9.0, so the nuspec floor moves to it. Together they turn the pass from ~170 s into a few seconds. Contract notes: GetRange omits keys it did not find rather than returning a placeholder, matching TryGetValue per key, and duplicate keys collapse. The backend cannot distinguish a miss from a stored empty blob, so both read as a miss - the same as ContainsKey and TryGetValue already do. SetRange replaces existing keys like the indexer setter. 13 tests cover the fast path, the loop fallback, duplicate keys, missing keys, the 2 000-key chunk boundary, agreement with per-key TryGetValue, CRM attribute round trips, replacement and null arguments. Suite is 62 tests, all passing. --- .gitignore | 3 + CHANGELOG.md | 54 +++ README.md | 63 +++- .../BulkDictionaryTests.cs | 321 ++++++++++++++++++ .../Xrm.Persistent.Collections.Tests.csproj | 1 + Xrm.Persistent.Collections.nuspec | 6 +- .../DictionaryExtensions.cs | 96 ++++++ .../Interfaces/IBulkDictionary.cs | 50 +++ Xrm.Persistent.Collections/LocalDictionary.cs | 40 ++- .../Xrm.Persistent.Collections.csproj | 4 +- 10 files changed, 631 insertions(+), 7 deletions(-) create mode 100644 Xrm.Persistent.Collections.Tests/BulkDictionaryTests.cs create mode 100644 Xrm.Persistent.Collections/DictionaryExtensions.cs create mode 100644 Xrm.Persistent.Collections/Interfaces/IBulkDictionary.cs diff --git a/.gitignore b/.gitignore index ef0212b..2e4dd3e 100644 --- a/.gitignore +++ b/.gitignore @@ -297,3 +297,6 @@ __pycache__/ /CALVER_GUIDE.md /CALVER_SUMMARY.md /VERSIONING_STRATEGY.md + +# Claude Code workspace settings +.claude/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f1e87fc..5fbbec0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,60 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.2026.9.8] - 2026-09-07 + +### Performance Release + +Adds a batch read/write API and picks up a ~240x serialization fix from `Xrm.Json.Serialization`. +No breaking changes: `LocalDictionary` keeps its full `IDictionary` surface and +existing database files are unaffected. + +### Added +- **`IBulkDictionary`** (`Xrm.Persistent.Collections.Interfaces`) — `GetRange(IEnumerable keys)` + and `SetRange(IDictionary items)`. Deliberately a separate interface from `IDictionary`, + so a consumer holding only the interface can test for it and fall back. +- **`LocalDictionary` implements `IBulkDictionary`**, routing both members to the batch + operations `PersistentBlobCache` already had (`Get(IEnumerable)` and + `Insert(IDictionary)`), which chunk at 950 keys per statement. +- **`DictionaryExtensions.GetRange` / `SetRange`** — extension methods on `IDictionary` + that take the batch path when the target implements `IBulkDictionary` and fall back to a + per-key loop otherwise. This lets a caller that holds an `IDictionary` — not knowing + whether it is in memory or persistent — get the batch behaviour without a type check. +- 13 tests in `BulkDictionaryTests` covering the interface fast path, the loop fallback, duplicate + key collapsing, missing keys, chunk-boundary crossing at 2 000 keys, agreement with per-key + `TryGetValue`, CRM attribute round trips, replacement semantics and null argument rejection. + +### Contract notes +- `GetRange` **omits** keys it did not find, matching `TryGetValue` per key rather than returning a + placeholder. Duplicate keys in the input collapse to one entry. +- The backend returns an empty buffer for a key it did not find, so absence and "stored an empty + blob" are indistinguishable at that layer. `GetRange` treats both as a miss — the same as + `ContainsKey` and `TryGetValue` already do. +- `SetRange` replaces existing keys like the indexer setter, and does not throw on a key that + already exists. + +### Changed +- **`Xrm.Json.Serialization` floor raised from 1.2026.3.1 to 1.2026.9.0.** 1.2026.9.0 fixes a + per-call `ContractResolver` allocation that was discarding Newtonsoft's contract cache and + re-resolving every type by reflection on every entity. Because NuGet resolves lowest-applicable, + the floor has to move or downstream projects keep restoring the slow version. + +### Performance +Measured on .NET Framework 4.8 x64, 100 000 keys holding `IList` of five attributes each, +against a warm database: + +| | Per-key loop | `GetRange` / `SetRange` in blocks of 1 000 | +|---|---|---| +| Reads | ~92 s | **1.27 s** | +| Writes | ~78 s | 75.94 s → **~3 s** with `Xrm.Json.Serialization` 1.2026.9.0 | + +Batching alone is ~2.2x end to end (170 s → 77 s) and all but eliminates the read cost. The write +side was dominated not by SQLite but by JSON serialization — 74 of those 76 seconds — which is what +the serializer upgrade addresses. The two together are what turn a ~3 minute pass into a few seconds. + +For reference, the SQLite floor for the same 100 000 rows against the real `CacheItem` schema is +~12 s in blocks of 1 000, and ~2 s in a single transaction. + ## [2.2026.9.7] - 2026-09-07 ### 🔒 Security Release diff --git a/README.md b/README.md index 210225b..d967d0e 100644 --- a/README.md +++ b/README.md @@ -400,9 +400,15 @@ using (var dict = new LocalDictionary("data.db")) ## 📚 Dependencies & Compatibility -### Xrm.Json.Serialization v1.2026.3.1 +### Xrm.Json.Serialization v1.2026.9.0 This library uses the latest version of Xrm.Json.Serialization with major enhancements: +#### Serialization Performance +1.2026.9.0 fixes a per-call `ContractResolver` allocation that was discarding Newtonsoft's +contract cache and re-resolving every type by reflection on every entity. Serializing 100 000 +single-entity lists went from 82.31 s to 0.34 s, with byte-identical output. Nothing in the +JSON format or the public API changed. + #### New Data Type Support - **AliasedValue**: FetchXML queries with linked entities are now fully supported - **OptionSetValueCollection**: Multi-select picklists work seamlessly @@ -480,6 +486,43 @@ dict.Clear(); dict.Dispose(); ``` +### Bulk Operations + +`LocalDictionary` implements `IBulkDictionary`, which reads and writes many keys per +round trip instead of one: + +```csharp +using Xrm.Persistent.Collections.Interfaces; + +// Read many keys in one go. Keys that are not present are omitted from the result, +// the same way TryGetValue reports a miss. Duplicate keys collapse to one entry. +IDictionary found = dict.GetRange(new[] { "a", "b", "c" }); + +// Write many keys in one go. Existing keys are replaced, like the indexer setter. +dict.SetRange(new Dictionary +{ + ["a"] = first, + ["b"] = second +}); +``` + +If you hold the value as an `IDictionary` — so you cannot tell whether it is an +in-memory dictionary or a persistent one — use the extension methods instead. They take the +batch path when the target supports it and fall back to a per-key loop when it does not: + +```csharp +using Xrm.Persistent.Collections; + +IDictionary maybePersistent = GetCache(); + +var found = maybePersistent.GetRange(keys); // batched if persistent, looped if not +maybePersistent.SetRange(items); // same +``` + +Batch in blocks rather than passing 100 000 keys at once. The backend chunks at 950 keys per +SQL statement, but the whole result set is materialised in memory, so a block of about 1 000 +keeps both bounded. + ### Cache Introspection Methods ```csharp // Get all non-expired items (raw byte arrays) @@ -549,8 +592,24 @@ var errorLog = new LocalDictionary("errors.db"); - **Concurrent reads**: Excellent (WAL mode) - **Concurrent writes**: Serialized (SQLite limitation) +### Per-key access does not scale + +Every indexer or `ContainsKey` call is its own SQL round trip, so a per-key loop is linear in +round trips rather than in rows. Measured on .NET Framework 4.8 x64, 100 000 keys holding an +`IList` of five attributes each, warm database: + +| | Per-key loop | `GetRange` / `SetRange` in blocks of 1 000 | +|---|---|---| +| Reads | ~92 s | **1.27 s** | +| Writes | ~78 s | **~3 s** | + +For scale, raw SQLite for the same 100 000 rows is ~12 s in blocks of 1 000 and ~2 s in one +transaction — so on the write side the cost was never the database. It was JSON serialization, +which is why the `Xrm.Json.Serialization` 1.2026.9.0 floor matters as much as the batching does. + ### Performance Tips -- Batch writes when possible +- Use `GetRange()` / `SetRange()` instead of a per-key loop — this is the single biggest win +- Batch in blocks of about 1 000 keys rather than one call for everything - Avoid enumerating `Values` for large datasets - Use `ContainsKey()` instead of `TryGetValue()` when you only need existence check - Keep entity sizes reasonable (<1 MB per entity) diff --git a/Xrm.Persistent.Collections.Tests/BulkDictionaryTests.cs b/Xrm.Persistent.Collections.Tests/BulkDictionaryTests.cs new file mode 100644 index 0000000..04ec5c0 --- /dev/null +++ b/Xrm.Persistent.Collections.Tests/BulkDictionaryTests.cs @@ -0,0 +1,321 @@ +namespace Xrm.Persistent.Collections +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using Microsoft.Xrm.Sdk; + using Xunit; + + /// + /// Covers on and the + /// dispatch around it. The batch path issues completely + /// different SQL from the one-key-at-a-time path - a chunked WHERE Key IN (...) and a + /// single transaction rather than a statement per key - so it needs its own coverage rather than + /// leaning on the per-key tests next door. + /// + public class BulkDictionaryTests : IDisposable + { + #region Private Fields + + private readonly string dbPath; + private readonly LocalDictionary dictionary; + + #endregion Private Fields + + #region Public Constructors + + public BulkDictionaryTests() + { + var suffix = Guid.NewGuid(); + dbPath = Path.Combine(Directory.GetCurrentDirectory(), $"{nameof(BulkDictionaryTests)}-{suffix}.db"); + + dictionary = new LocalDictionary(dbPath); + } + + #endregion Public Constructors + + #region Public Methods + + public void Dispose() + { + dictionary?.Dispose(); + + if (File.Exists(dbPath)) + { + File.Delete(dbPath); + } + } + + [Fact] + public void Extension_Takes_The_Bulk_Path_Through_The_Interface() + { + // Arrange - this is the shape a consumer actually holds: the concrete type is erased + // behind IDictionary, and the extension has to find IBulkDictionary on it anyway. + IDictionary erased = dictionary; + var entity = new Entity("test", Guid.NewGuid()); + + // Act + erased.SetRange(new Dictionary { { "a", entity } }); + var actual = erased.GetRange(new[] { "a", "absent" }); + + // Assert - reading it back through the per-key indexer proves the write went to the + // database rather than to some in-memory shim the extension invented. + Assert.Single(actual); + Assert.Equal(entity.Id, actual["a"].Id); + Assert.Equal(entity.Id, dictionary["a"].Id); + } + + [Fact] + public void GetRange_Collapses_Duplicate_Keys() + { + // Arrange + dictionary["a"] = new Entity("test", Guid.NewGuid()); + + // Act + var actual = dictionary.GetRange(new[] { "a", "a", "a" }); + + // Assert + Assert.Single(actual); + } + + [Fact] + public void GetRange_Crosses_The_Chunk_Boundary() + { + // Arrange - the backend chunks at 950 keys per statement, so anything above that + // exercises more than one round trip and the merge of their results. The chunking is + // there to stay clear of SQLite's own bound-parameter ceiling, so a single-statement + // implementation would fail this test rather than silently degrade. + var expected = Enumerable.Range(0, 2000) + .ToDictionary(i => i.ToString(), i => new Entity("test", Guid.NewGuid())); + + dictionary.SetRange(expected); + + // Act + var actual = dictionary.GetRange(expected.Keys); + + // Assert + Assert.Equal(expected.Count, actual.Count); + + foreach (var pair in expected) + { + Assert.Equal(pair.Value.Id, actual[pair.Key].Id); + } + } + + [Fact] + public void GetRange_Matches_TryGetValue_Key_For_Key() + { + // Arrange - the batch path is only useful if it is indistinguishable from the loop it + // replaces, so assert that directly rather than trusting the two to agree. + var expected = Enumerable.Range(0, 50) + .ToDictionary(i => i.ToString(), i => new Entity("test", Guid.NewGuid())); + + dictionary.SetRange(expected); + + var keys = expected.Keys.Concat(new[] { "absent" }).ToArray(); + + // Act + var batch = dictionary.GetRange(keys); + + var loop = new Dictionary(); + foreach (var key in keys) + { + if (dictionary.TryGetValue(key, out var value)) + { + loop.Add(key, value); + } + } + + // Assert + Assert.Equal(loop.Keys.OrderBy(o => o), batch.Keys.OrderBy(o => o)); + + foreach (var pair in loop) + { + Assert.Equal(pair.Value.Id, batch[pair.Key].Id); + } + } + + [Fact] + public void GetRange_Omits_Missing_Keys() + { + // Arrange + var present = new Entity("test", Guid.NewGuid()); + dictionary["present"] = present; + + // Act + var actual = dictionary.GetRange(new[] { "present", "absent" }); + + // Assert - a miss is omitted rather than returned as a default value, so the caller can + // tell "never stored" from "stored a null". + Assert.Single(actual); + Assert.True(actual.ContainsKey("present")); + Assert.False(actual.ContainsKey("absent")); + Assert.Equal(present.Id, actual["present"].Id); + } + + [Fact] + public void GetRange_Returns_Empty_For_An_Empty_Key_Set() + { + // Arrange + dictionary["a"] = new Entity("test", Guid.NewGuid()); + + // Act + var actual = dictionary.GetRange(new string[0]); + + // Assert + Assert.Empty(actual); + } + + [Fact] + public void GetRange_Round_Trips_Crm_Attribute_Types() + { + // Arrange - the batch read deserializes through the same converters as the indexer, but + // it builds its result from a different query, so pin the CRM types down here too. + var id = Guid.NewGuid(); + var entity = new Entity("test", id); + entity.Attributes.Add("text", "value"); + entity.Attributes.Add("reference", new EntityReference("test", id)); + entity.Attributes.Add("option", new OptionSetValue(3)); + entity.Attributes.Add("money", new Money(12.5m)); + + dictionary.SetRange(new Dictionary { { "entity", entity } }); + + // Act + var actual = dictionary.GetRange(new[] { "entity" })["entity"]; + + // Assert + Assert.Equal(id, actual.Id); + Assert.Equal("value", actual["text"]); + Assert.Equal(id, (actual["reference"] as EntityReference).Id); + Assert.Equal(3, (actual["option"] as OptionSetValue).Value); + Assert.Equal(12.5m, (actual["money"] as Money).Value); + } + + [Fact] + public void Null_Arguments_Are_Rejected() + { + // Arrange - the extension is the reachable entry point, so cast to the interface type + // rather than letting the compiler bind to the instance methods. + IDictionary missing = null; + IDictionary erased = dictionary; + + // Act / Assert + Assert.Throws(() => missing.GetRange(new[] { "a" })); + Assert.Throws(() => missing.SetRange(new Dictionary())); + Assert.Throws(() => erased.GetRange(null)); + Assert.Throws(() => erased.SetRange(null)); + } + + [Fact] + public void Plain_Dictionary_Falls_Back_To_A_Loop() + { + // Arrange - a ConcurrentDictionary or a plain Dictionary does not implement + // IBulkDictionary, and the extension has to serve it anyway. This is the case a job + // engine hits on every run that stays below the persistent-storage threshold. + IDictionary plain = new Dictionary(); + var first = new Entity("test", Guid.NewGuid()); + var second = new Entity("test", Guid.NewGuid()); + + // Act + plain.SetRange(new Dictionary { { "a", first }, { "b", second } }); + var actual = plain.GetRange(new[] { "a", "b", "a", "absent" }); + + // Assert + Assert.Equal(2, plain.Count); + Assert.Equal(2, actual.Count); + Assert.Equal(first.Id, actual["a"].Id); + Assert.Equal(second.Id, actual["b"].Id); + } + + [Fact] + public void Plain_Dictionary_SetRange_Replaces_Existing_Keys() + { + // Arrange + var replacement = new Entity("test", Guid.NewGuid()); + IDictionary plain = new Dictionary + { + { "a", new Entity("test", Guid.NewGuid()) } + }; + + // Act - Add would throw here; the contract is replace, matching the indexer setter. + plain.SetRange(new Dictionary { { "a", replacement } }); + + // Assert + Assert.Single(plain); + Assert.Equal(replacement.Id, plain["a"].Id); + } + + [Fact] + public void SetRange_Accepts_An_Empty_Set() + { + // Arrange + dictionary["a"] = new Entity("test", Guid.NewGuid()); + + // Act + dictionary.SetRange(new Dictionary()); + + // Assert - an empty write is a no-op, not an error and not a truncation. + Assert.Single(dictionary.Keys); + } + + [Fact] + public void SetRange_Is_Visible_To_The_Per_Key_Readers() + { + // Arrange - a batch write has to land in the same rows the indexer and ContainsKey read, + // otherwise mixing the two APIs in one job would silently lose data. + var entity = new Entity("test", Guid.NewGuid()); + + // Act + dictionary.SetRange(new Dictionary { { "a", entity } }); + + // Assert + Assert.True(dictionary.ContainsKey("a")); + Assert.Equal(entity.Id, dictionary["a"].Id); + Assert.True(dictionary.TryGetValue("a", out var found)); + Assert.Equal(entity.Id, found.Id); + } + + [Fact] + public void SetRange_Replaces_Existing_Keys() + { + // Arrange + var original = new Entity("test", Guid.NewGuid()); + var replacement = new Entity("test", Guid.NewGuid()); + dictionary["a"] = original; + + // Act + dictionary.SetRange(new Dictionary { { "a", replacement } }); + + // Assert + Assert.Single(dictionary.Keys); + Assert.Equal(replacement.Id, dictionary["a"].Id); + } + + [Fact] + public void SetRange_Survives_A_New_Dictionary_Instance() + { + // Arrange + var expected = Enumerable.Range(0, 10) + .ToDictionary(i => i.ToString(), i => new Entity("test", Guid.NewGuid())); + + dictionary.SetRange(expected); + + // Act - reopen the same file, so this asserts the transaction actually committed. + using (var reopened = new LocalDictionary(dbPath)) + { + var actual = reopened.GetRange(expected.Keys); + + // Assert + Assert.Equal(expected.Count, actual.Count); + + foreach (var pair in expected) + { + Assert.Equal(pair.Value.Id, actual[pair.Key].Id); + } + } + } + + #endregion Public Methods + } +} diff --git a/Xrm.Persistent.Collections.Tests/Xrm.Persistent.Collections.Tests.csproj b/Xrm.Persistent.Collections.Tests/Xrm.Persistent.Collections.Tests.csproj index 5021990..a35ddac 100644 --- a/Xrm.Persistent.Collections.Tests/Xrm.Persistent.Collections.Tests.csproj +++ b/Xrm.Persistent.Collections.Tests/Xrm.Persistent.Collections.Tests.csproj @@ -200,6 +200,7 @@ + diff --git a/Xrm.Persistent.Collections.nuspec b/Xrm.Persistent.Collections.nuspec index 2b7176a..c604f04 100644 --- a/Xrm.Persistent.Collections.nuspec +++ b/Xrm.Persistent.Collections.nuspec @@ -1,8 +1,8 @@ - + Xrm.Persistent.Collections - 2.2026.9.7 + 2.2026.9.8 Xrm Persistent Collections Imran Akram Imran Akram @@ -82,7 +82,7 @@ - + - +