From 46fa1fbe83fb34d7ccbb56599a9cac0bfb657f3a Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Fri, 21 Aug 2026 16:27:58 -0700 Subject: [PATCH] Make HTTP health tests run serially to address flakiness. --- .../HealthPagesHttpListenerServiceTests.cs | 97 +++++++++++++------ 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/test/RedShirt.Example.JobWorker.UnitTests/Tests/Health/HealthPagesHttpListenerServiceTests.cs b/test/RedShirt.Example.JobWorker.UnitTests/Tests/Health/HealthPagesHttpListenerServiceTests.cs index 363256b4..8fcabb0c 100644 --- a/test/RedShirt.Example.JobWorker.UnitTests/Tests/Health/HealthPagesHttpListenerServiceTests.cs +++ b/test/RedShirt.Example.JobWorker.UnitTests/Tests/Health/HealthPagesHttpListenerServiceTests.cs @@ -10,13 +10,18 @@ namespace RedShirt.Example.JobWorker.UnitTests.Tests.Health; -public class HealthPagesHttpListenerServiceTests +/// +/// Serialize HTTP listener tests so parallel runs do not race on ephemeral ports. +/// +[CollectionDefinition(Name)] +public class HealthPagesHttpListenerCollection { - private static readonly HttpClient Client = new() - { - Timeout = TimeSpan.FromSeconds(1) - }; + public const string Name = "HealthPagesHttpListener"; +} +[Collection(HealthPagesHttpListenerCollection.Name)] +public class HealthPagesHttpListenerServiceTests +{ private static HealthPagesHttpListenerService CreateService( bool enabled, int port, @@ -74,25 +79,59 @@ private static int GetFreePort() return ((IPEndPoint) listener.LocalEndpoint).Port; } - private static async Task WaitForEndpointAsync(string url, TimeSpan? timeout = null) + private static async Task WaitForEndpointAsync(string url, HttpStatusCode? expectedStatus = null, + TimeSpan? timeout = null) { var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(5)); - using var client = new HttpClient(); + using var client = new HttpClient + { + Timeout = TimeSpan.FromSeconds(1) + }; + Exception? lastError = null; while (DateTime.UtcNow < deadline) { try { using var response = await client.GetAsync(url, TestContext.Current.CancellationToken); - return; + if (expectedStatus is null || response.StatusCode == expectedStatus) + { + return; + } } - catch (HttpRequestException) + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) { - await Task.Delay(50, TestContext.Current.CancellationToken); + lastError = ex; } + + await Task.Delay(25, TestContext.Current.CancellationToken); } - throw new TimeoutException($"Endpoint {url} did not become available."); + throw new TimeoutException( + $"Endpoint {url} did not become available" + + (expectedStatus is null ? "." : $" with status {expectedStatus}."), lastError); + } + + private static async Task<(HealthPagesHttpListenerService Service, int Port)> StartEnabledServiceAsync( + ICoreHealthStateReaderService? healthService = null) + { + // Ephemeral-port reservation is inherently racy; retry if another process claims the port first. + for (var attempt = 0; attempt < 5; attempt++) + { + var port = GetFreePort(); + var service = CreateService(true, port, healthService); + try + { + await service.StartAsync(CancellationToken.None); + return (service, port); + } + catch (HttpListenerException) when (attempt < 4) + { + await service.StopAsync(CancellationToken.None); + } + } + + throw new InvalidOperationException("Could not bind health HttpListener to a free port."); } [Theory] @@ -100,17 +139,14 @@ private static async Task WaitForEndpointAsync(string url, TimeSpan? timeout = n [InlineData(HealthPathConstants.HealthPath)] public async Task GetEndpoint_WhenEnabled_ReturnsOk(string path) { - var port = GetFreePort(); - var service = CreateService(true, port); - - await service.StartAsync(CancellationToken.None); + var (service, port) = await StartEnabledServiceAsync(); try { - await WaitForEndpointAsync($"http://127.0.0.1:{port}{path}"); + var url = $"http://127.0.0.1:{port}{path}"; + await WaitForEndpointAsync(url, HttpStatusCode.OK); using var client = new HttpClient(); - var response = await client.GetAsync($"http://127.0.0.1:{port}{path}", - TestContext.Current.CancellationToken); + var response = await client.GetAsync(url, TestContext.Current.CancellationToken); Assert.Equal(HttpStatusCode.OK, response.StatusCode); Assert.Equal("text/plain", response.Content.Headers.ContentType?.MediaType); @@ -125,13 +161,10 @@ public async Task GetEndpoint_WhenEnabled_ReturnsOk(string path) [Fact] public async Task GetEndpoint_WhenUnknownPath_ReturnsNotFound() { - var port = GetFreePort(); - var service = CreateService(true, port); - - await service.StartAsync(CancellationToken.None); + var (service, port) = await StartEnabledServiceAsync(); try { - await WaitForEndpointAsync($"http://127.0.0.1:{port}{HealthPathConstants.LivePath}"); + await WaitForEndpointAsync($"http://127.0.0.1:{port}{HealthPathConstants.LivePath}", HttpStatusCode.OK); using var client = new HttpClient(); var response = await client.GetAsync($"http://127.0.0.1:{port}/unknown", @@ -148,19 +181,17 @@ public async Task GetEndpoint_WhenUnknownPath_ReturnsNotFound() [Fact] public async Task HealthPath_WhenUnhealthy_ReturnsServiceUnavailable() { - var port = GetFreePort(); var health = new Mock(MockBehavior.Strict); health.Setup(h => h.IsHealthy()).Returns(false); - var service = CreateService(true, port, health.Object); - - await service.StartAsync(CancellationToken.None); + var (service, port) = await StartEnabledServiceAsync(health.Object); try { - await WaitForEndpointAsync($"http://127.0.0.1:{port}{HealthPathConstants.LivePath}"); + // Wait for the endpoint under test (and the 503 itself), not a different path. + var url = $"http://127.0.0.1:{port}{HealthPathConstants.HealthPath}"; + await WaitForEndpointAsync(url, HttpStatusCode.ServiceUnavailable); using var client = new HttpClient(); - var response = await client.GetAsync($"http://127.0.0.1:{port}{HealthPathConstants.HealthPath}", - TestContext.Current.CancellationToken); + var response = await client.GetAsync(url, TestContext.Current.CancellationToken); Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode); Assert.Equal("unhealthy", await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken)); @@ -181,8 +212,12 @@ public async Task StartAsync_WhenDisabled_DoesNotBindPort() await service.StartAsync(CancellationToken.None); try { + using var client = new HttpClient + { + Timeout = TimeSpan.FromSeconds(1) + }; await Assert.ThrowsAsync(() => - Client.GetAsync($"http://127.0.0.1:{port}{HealthPathConstants.LivePath}", + client.GetAsync($"http://127.0.0.1:{port}{HealthPathConstants.LivePath}", TestContext.Current.CancellationToken)); } finally