fix: retry transient schema request failures

src/json-schema-content-provider.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { TextDocumentContentProvider, Uri, workspace, window } from 'vscode';
+import { requestWithRetry } from './schema-request-retry';
 import { xhr, configure as configureHttpRequests, getErrorStatusDescription, XHRResponse } from 'request-light';
 import { SchemaExtensionAPI } from './schema-extension-api';
 import { ResponseError } from 'vscode-languageclient';
@@ -60,7 +61,7 @@ export async function getJsonSchemaContent(uri: string, schemaCache: IJSONSchema
   if (cachedETag) {
     headers['If-None-Match'] = cachedETag;
   }
-  return xhr({ url: uri, followRedirects: 5, headers })
+  return requestWithRetry(() => xhr({ url: uri, followRedirects: 5, headers }))
     .then(async (response) => {
       // cache only if server supports 'etag' header
       const etag = response.headers['etag'];
@@ -80,7 +81,7 @@ export async function getJsonSchemaContent(uri: string, schemaCache: IJSONSchema
         if (content === undefined) {
           console.error(`Cannot read cached content for: ${uri}, trying to load again`);
           delete headers['If-None-Match'];
-          return xhr({ url: uri, followRedirects: 5, headers })
+          return requestWithRetry(() => xhr({ url: uri, followRedirects: 5, headers }))
             .then((response) => {
               return response.responseText;
             })

src/schema-request-retry.ts

@@ -0,0 +1,111 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Red Hat, Inc. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/** HTTP statuses that mean "try again shortly". Any other status is a settled answer and is never retried. */
+const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504]);
+
+/** Connection level failures that are worth another attempt. */
+const RETRYABLE_ERROR_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'ECONNABORTED', 'EPIPE', 'EAI_AGAIN']);
+
+/** Additional attempts made after the first request fails. */
+const MAX_RETRIES = 2;
+
+/** Base delay for exponential backoff, in milliseconds. */
+const BASE_RETRY_DELAY_MS = 200;
+
+/**
+ * Upper bound for a single delay, in milliseconds. It also caps `Retry-After`, which servers sometimes give in
+ * minutes. Schema loading blocks validation, so waiting must not become noticeable while editing.
+ */
+const MAX_RETRY_DELAY_MS = 1000;
+
+/** The parts of a failed request that decide whether to retry. */
+export interface RetryableError {
+  status?: number;
+  code?: string;
+  headers?: Record<string, string>;
+}
+
+/** Overrides for the retry behaviour, meant for tests. */
+export interface RetryOptions {
+  /** Additional attempts made after the first request fails. */
+  maxRetries?: number;
+  /** Replaces the real wait, so tests do not sleep through the backoff. */
+  delay?: (ms: number) => Promise<void>;
+}
+
+const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Decide whether a failed request is worth repeating.
+ * @param error the rejection from the request
+ * @returns true when the failure looks temporary
+ */
+export function isRetryableError(error: RetryableError | undefined): boolean {
+  if (error?.code && RETRYABLE_ERROR_CODES.has(error.code)) {
+    return true;
+  }
+  // Connection failures can show up as status 0 or 404 with no real response, so only an explicitly retryable status
+  // counts.
+  return typeof error?.status === 'number' && RETRYABLE_STATUS_CODES.has(error.status);
+}
+
+/**
+ * Parse a `Retry-After` header, which is either a number of seconds or an HTTP date.
+ * @param headers the response headers, if there were any
+ * @param now the current time in milliseconds, used to measure an HTTP date
+ * @returns the delay in milliseconds, or undefined when the header is missing or unreadable
+ */
+export function parseRetryAfter(headers: Record<string, string> | undefined, now: number): number | undefined {
+  const value = headers?.['retry-after'] ?? headers?.['Retry-After'];
+  if (!value) {
+    return undefined;
+  }
+  const seconds = Number(value);
+  if (!Number.isNaN(seconds)) {
+    return seconds >= 0 ? seconds * 1000 : undefined;
+  }
+  const date = Date.parse(value);
+  return Number.isNaN(date) ? undefined : Math.max(0, date - now);
+}
+
+/**
+ * Work out how long to wait before the next attempt. A server supplied `Retry-After` wins. Otherwise back off
+ * exponentially with jitter, so several schemas failing together do not retry in lockstep. The result is capped.
+ * @param error the rejection from the request
+ * @param attempt the zero based index of the attempt that just failed
+ * @returns how long to wait, in milliseconds
+ */
+export function getRetryDelay(error: RetryableError | undefined, attempt: number): number {
+  const retryAfter = parseRetryAfter(error?.headers, Date.now());
+  if (retryAfter !== undefined) {
+    return Math.min(retryAfter, MAX_RETRY_DELAY_MS);
+  }
+  const backoff = BASE_RETRY_DELAY_MS * Math.pow(2, attempt);
+  const jitter = Math.random() * BASE_RETRY_DELAY_MS;
+  return Math.min(backoff + jitter, MAX_RETRY_DELAY_MS);
+}
+
+/**
+ * Run a request, retrying a small number of times when it fails for a temporary reason. A failure that is not
+ * retryable, or one that uses up the retries, is rethrown unchanged.
+ * @param request sends one attempt
+ * @param options overrides for the retry behaviour, used by tests
+ * @returns whatever the successful attempt returns
+ */
+export async function requestWithRetry<T>(request: () => Promise<T>, options: RetryOptions = {}): Promise<T> {
+  const maxRetries = options.maxRetries ?? MAX_RETRIES;
+  const delay = options.delay ?? sleep;
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return await request();
+    } catch (error) {
+      if (attempt >= maxRetries || !isRetryableError(error as RetryableError)) {
+        throw error;
+      }
+      await delay(getRetryDelay(error as RetryableError, attempt));
+    }
+  }
+}

test/schema-request-retry.test.ts

@@ -0,0 +1,135 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Red Hat, Inc. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import * as assert from 'assert';
+import { getRetryDelay, isRetryableError, parseRetryAfter, requestWithRetry } from '../src/schema-request-retry';
+
+describe('Schema request retry', () => {
+  const noDelay = (): Promise<void> => Promise.resolve();
+
+  function failingThenSucceeding(
+    failures: unknown[],
+    result = 'schema'
+  ): { request: () => Promise<string>; calls: () => number } {
+    let calls = 0;
+    return {
+      request: async () => {
+        const failure = failures[calls++];
+        if (failure !== undefined) {
+          throw failure;
+        }
+        return result;
+      },
+      calls: () => calls,
+    };
+  }
+
+  describe('isRetryableError', () => {
+    it('retries temporary HTTP statuses', () => {
+      for (const status of [429, 502, 503, 504]) {
+        assert.strictEqual(isRetryableError({ status }), true, `status ${status}`);
+      }
+    });
+
+    it('does not retry settled HTTP statuses', () => {
+      for (const status of [304, 400, 401, 403, 404, 500]) {
+        assert.strictEqual(isRetryableError({ status }), false, `status ${status}`);
+      }
+    });
+
+    it('retries connection level errors', () => {
+      for (const code of ['ECONNRESET', 'ETIMEDOUT', 'ECONNABORTED', 'EPIPE', 'EAI_AGAIN']) {
+        assert.strictEqual(isRetryableError({ code }), true, code);
+      }
+      assert.strictEqual(isRetryableError({ code: 'ENOTFOUND' }), false);
+    });
+
+    it('does not retry when there is nothing to inspect', () => {
+      assert.strictEqual(isRetryableError(undefined), false);
+      assert.strictEqual(isRetryableError({}), false);
+    });
+  });
+
+  describe('parseRetryAfter', () => {
+    it('reads delta-seconds', () => {
+      assert.strictEqual(parseRetryAfter({ 'retry-after': '2' }, 0), 2000);
+      assert.strictEqual(parseRetryAfter({ 'Retry-After': '0' }, 0), 0);
+    });
+
+    it('reads an HTTP date relative to now', () => {
+      const now = Date.parse('2026-01-01T00:00:00Z');
+      assert.strictEqual(parseRetryAfter({ 'retry-after': 'Thu, 01 Jan 2026 00:00:03 GMT' }, now), 3000);
+      assert.strictEqual(parseRetryAfter({ 'retry-after': 'Wed, 31 Dec 2025 23:59:00 GMT' }, now), 0);
+    });
+
+    it('ignores a missing, negative or unreadable header', () => {
+      assert.strictEqual(parseRetryAfter(undefined, 0), undefined);
+      assert.strictEqual(parseRetryAfter({}, 0), undefined);
+      assert.strictEqual(parseRetryAfter({ 'retry-after': '-5' }, 0), undefined);
+      assert.strictEqual(parseRetryAfter({ 'retry-after': 'soon' }, 0), undefined);
+    });
+  });
+
+  describe('getRetryDelay', () => {
+    it('caps a long Retry-After at one second', () => {
+      assert.strictEqual(getRetryDelay({ headers: { 'retry-after': '120' } }, 0), 1000);
+    });
+
+    it('honours a short Retry-After', () => {
+      assert.strictEqual(getRetryDelay({ headers: { 'retry-after': '0' } }, 0), 0);
+    });
+
+    it('backs off exponentially without exceeding the cap', () => {
+      const first = getRetryDelay({}, 0);
+      assert.ok(first >= 200 && first <= 400, `first delay ${first}`);
+      assert.ok(getRetryDelay({}, 10) <= 1000);
+    });
+  });
+
+  describe('requestWithRetry', () => {
+    it('returns straight away when the first attempt works', async () => {
+      const { request, calls } = failingThenSucceeding([]);
+      assert.strictEqual(await requestWithRetry(request, { delay: noDelay }), 'schema');
+      assert.strictEqual(calls(), 1);
+    });
+
+    it('recovers after a temporary failure', async () => {
+      const { request, calls } = failingThenSucceeding([{ status: 503 }]);
+      assert.strictEqual(await requestWithRetry(request, { delay: noDelay }), 'schema');
+      assert.strictEqual(calls(), 2);
+    });
+
+    it('gives up after the retry budget and rethrows the last error unchanged', async () => {
+      const error = { status: 503, responseText: 'unavailable' };
+      const { request, calls } = failingThenSucceeding([error, error, error, error]);
+      await assert.rejects(requestWithRetry(request, { delay: noDelay }), (thrown) => thrown === error);
+      assert.strictEqual(calls(), 3);
+    });
+
+    it('does not retry a settled failure', async () => {
+      const error = { status: 404 };
+      const { request, calls } = failingThenSucceeding([error]);
+      await assert.rejects(requestWithRetry(request, { delay: noDelay }), (thrown) => thrown === error);
+      assert.strictEqual(calls(), 1);
+    });
+
+    it('waits between attempts using the supplied delay', async () => {
+      const waits: number[] = [];
+      const { request } = failingThenSucceeding([{ status: 429, headers: { 'retry-after': '1' } }]);
+      await requestWithRetry(request, {
+        delay: (ms) => {
+          waits.push(ms);
+          return Promise.resolve();
+        },
+      });
+      assert.deepStrictEqual(waits, [1000]);
+    });
+
+    it('honours a custom retry count', async () => {
+      const { request, calls } = failingThenSucceeding([{ status: 503 }, { status: 503 }]);
+      await assert.rejects(requestWithRetry(request, { delay: noDelay, maxRetries: 0 }));
+      assert.strictEqual(calls(), 1);
+    });
+  });
+});