From 3eaf544e1677f0ede95163acf67a6a9e1f1a9986 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 10 Jun 2026 10:42:18 +0000 Subject: [PATCH 1/2] fix: handle ObjectDisposedException in AcquireLock when semaphore is evicted Catch ObjectDisposedException in both AcquireLockAsync and AcquireLock. When MemoryCache evicts and disposes a SemaphoreSlim between GetSemaphore and WaitAsync, remove the stale entry and retry with a fresh semaphore instead of propagating the exception as a fatal error. PROD-5987 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- HzMemoryCache/HzCacheMemoryLocker.cs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/HzMemoryCache/HzCacheMemoryLocker.cs b/HzMemoryCache/HzCacheMemoryLocker.cs index 606b9ca..6eeca6d 100644 --- a/HzMemoryCache/HzCacheMemoryLocker.cs +++ b/HzMemoryCache/HzCacheMemoryLocker.cs @@ -93,7 +93,18 @@ public async ValueTask AcquireLockAsync(string cacheName, string cacheIn var acquired = false; using (var waitForSemaphore = HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.WaitForSemaphore, HzActivities.Area.HzCacheMemoryLocker, key: key, async: true)) { - acquired = await semaphore.WaitAsync(timeout, token).ConfigureAwait(false); + try + { + acquired = await semaphore.WaitAsync(timeout, token).ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + // The MemoryCache evicted and disposed this semaphore between GetSemaphore and WaitAsync. + // Remove the stale entry and retry with a fresh semaphore. + lockCache.Remove(key); + semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger); + acquired = await semaphore.WaitAsync(timeout, token).ConfigureAwait(false); + } } if (acquired) @@ -136,7 +147,16 @@ public async ValueTask AcquireLockAsync(string cacheName, string cacheIn HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.WaitForSemaphore, HzActivities.Area.HzCacheMemoryLocker, key: key)) { - acquired = semaphore.Wait(timeout, token); + try + { + acquired = semaphore.Wait(timeout, token); + } + catch (ObjectDisposedException) + { + lockCache.Remove(key); + semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger); + acquired = semaphore.Wait(timeout, token); + } } if (acquired) From 055cb92b53c1bededf5f58a5ec581128fba4f1e4 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 10 Jun 2026 10:55:27 +0000 Subject: [PATCH 2/2] refactor: use retry loop with configurable max attempts Replace single retry with a for-loop bounded by maxLockRetries (default 3). The catch filter (when attempt < maxLockRetries) lets the final attempt's ObjectDisposedException propagate if retries are exhausted. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- HzMemoryCache/HzCacheMemoryLocker.cs | 43 ++++++++++++++++------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/HzMemoryCache/HzCacheMemoryLocker.cs b/HzMemoryCache/HzCacheMemoryLocker.cs index 6eeca6d..794c45a 100644 --- a/HzMemoryCache/HzCacheMemoryLocker.cs +++ b/HzMemoryCache/HzCacheMemoryLocker.cs @@ -10,6 +10,7 @@ namespace HzCache public class HzCacheMemoryLockerOptions { public int lockPoolSize { get; set; } = 7872; + public int maxLockRetries { get; set; } = 3; } public class HzCacheMemoryLocker @@ -93,17 +94,20 @@ public async ValueTask AcquireLockAsync(string cacheName, string cacheIn var acquired = false; using (var waitForSemaphore = HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.WaitForSemaphore, HzActivities.Area.HzCacheMemoryLocker, key: key, async: true)) { - try + for (var attempt = 0; attempt <= options.maxLockRetries; attempt++) { - acquired = await semaphore.WaitAsync(timeout, token).ConfigureAwait(false); - } - catch (ObjectDisposedException) - { - // The MemoryCache evicted and disposed this semaphore between GetSemaphore and WaitAsync. - // Remove the stale entry and retry with a fresh semaphore. - lockCache.Remove(key); - semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger); - acquired = await semaphore.WaitAsync(timeout, token).ConfigureAwait(false); + try + { + acquired = await semaphore.WaitAsync(timeout, token).ConfigureAwait(false); + break; + } + catch (ObjectDisposedException) when (attempt < options.maxLockRetries) + { + // The MemoryCache evicted and disposed this semaphore between GetSemaphore and WaitAsync. + // Remove the stale entry and retry with a fresh semaphore. + lockCache.Remove(key); + semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger); + } } } @@ -147,15 +151,18 @@ public async ValueTask AcquireLockAsync(string cacheName, string cacheIn HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.WaitForSemaphore, HzActivities.Area.HzCacheMemoryLocker, key: key)) { - try + for (var attempt = 0; attempt <= options.maxLockRetries; attempt++) { - acquired = semaphore.Wait(timeout, token); - } - catch (ObjectDisposedException) - { - lockCache.Remove(key); - semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger); - acquired = semaphore.Wait(timeout, token); + try + { + acquired = semaphore.Wait(timeout, token); + break; + } + catch (ObjectDisposedException) when (attempt < options.maxLockRetries) + { + lockCache.Remove(key); + semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger); + } } }