PROD-5987: fix race condition in HzCacheMemoryLocker causing disposed semaphore errors

HzMemoryCache/HzCacheMemoryLocker.cs

@@ -10,6 +10,7 @@ namespace HzCache
     public class HzCacheMemoryLockerOptions
     {
         public int lockPoolSize { get; set; } = 7872;
+        public int maxLockRetries { get; set; } = 3;
     }
 
     public class HzCacheMemoryLocker
@@ -93,7 +94,21 @@ public async ValueTask<object> AcquireLockAsync(string cacheName, string cacheIn
             var acquired = false;
             using (var waitForSemaphore = HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.WaitForSemaphore, HzActivities.Area.HzCacheMemoryLocker, key: key, async: true))
             {
-                acquired = await semaphore.WaitAsync(timeout, token).ConfigureAwait(false);
+                for (var attempt = 0; attempt <= options.maxLockRetries; attempt++)
+                {
+                    try
+                    {
+                        acquired = await semaphore.WaitAsync(timeout, token).ConfigureAwait(false);
+                        break;
+                    }
+                    catch (ObjectDisposedException) when (attempt < options.maxLockRetries)
+                    {
+                        // The MemoryCache evicted and disposed this semaphore between GetSemaphore and WaitAsync.
+                        // Remove the stale entry and retry with a fresh semaphore.
+                        lockCache.Remove(key);
+                        semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger);
+                    }
+                }
             }
 
             if (acquired)
@@ -136,7 +151,19 @@ public async ValueTask<object> AcquireLockAsync(string cacheName, string cacheIn
                    HzActivities.Source.StartActivityWithCommonTags(HzActivities.Names.WaitForSemaphore,
                        HzActivities.Area.HzCacheMemoryLocker, key: key))
             {
-                acquired = semaphore.Wait(timeout, token);
+                for (var attempt = 0; attempt <= options.maxLockRetries; attempt++)
+                {
+                    try
+                    {
+                        acquired = semaphore.Wait(timeout, token);
+                        break;
+                    }
+                    catch (ObjectDisposedException) when (attempt < options.maxLockRetries)
+                    {
+                        lockCache.Remove(key);
+                        semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger);
+                    }
+                }
             }
 
             if (acquired)