fix(installer): keep retrying readiness while the published port resets

scripts/install-ubuntu-vbox.sh

@@ -188,7 +188,7 @@ printf 'Starte den UnoSim-Backenddienst …\n'
 sudo env PWD="$PROJECT_DIR" docker compose --project-directory "$PROJECT_DIR" up -d --no-build unosim-backend
 
 printf 'Warte auf die Backend-Readiness …\n'
-BACKEND_READINESS="$(curl --retry 10 --retry-connrefused --retry-delay 2 --fail --silent --show-error \
+BACKEND_READINESS="$(curl --retry 10 --retry-connrefused --retry-all-errors --retry-delay 2 --fail --silent --show-error \
   http://127.0.0.1:3000/api/readiness)" || {
   sudo env PWD="$PROJECT_DIR" docker compose --project-directory "$PROJECT_DIR" logs --tail 80 unosim-backend >&2 || true
   fail 'Der Backenddienst wurde gestartet, meldet aber keine Readiness. Prüfe die Compose-Logs.'
@@ -239,7 +239,7 @@ else
 fi
 
 printf 'Prüfe HTTPS-Gateway und Proxy …\n'
-GATEWAY_READINESS="$(curl --retry 10 --retry-connrefused --retry-delay 2 --fail --silent --show-error \
+GATEWAY_READINESS="$(curl --retry 10 --retry-connrefused --retry-all-errors --retry-delay 2 --fail --silent --show-error \
   --insecure --resolve 'unosim.vbox:443:127.0.0.1' \
   https://unosim.vbox/api/readiness)" || fail 'Nginx ist aktiv, aber die Readiness-Prüfung über HTTPS ist fehlgeschlagen.'
 printf 'Gateway-Readiness: %s\n' "$GATEWAY_READINESS"

tests/server/ubuntu-vbox-installer-readiness.test.ts

@@ -0,0 +1,82 @@
+import { execFile } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { createServer, type AddressInfo, type Server } from "node:net";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+
+const repoRoot = join(__dirname, "../..");
+const installer = readFileSync(join(repoRoot, "scripts/install-ubuntu-vbox.sh"), "utf8");
+const servers: Server[] = [];
+
+/** curl options of each readiness check, as the installer runs them (URL and TLS options excluded). */
+function readinessCurlOptions(): Record<string, string[]> {
+  const checks = [...installer.matchAll(/^(\w+_READINESS)="\$\(curl ([^\\\n]+)\\$/gm)];
+  return Object.fromEntries(checks.map(([, name, options]) => [name, options.trim().split(/\s+/)]));
+}
+
+/**
+ * Behaves like Docker's published port while the backend has not started to
+ * listen yet: the first connections are accepted and reset, later ones are
+ * answered by the backend.
+ */
+async function startResettingBackend(resets: number): Promise<{ url: string; connections: () => number }> {
+  let connections = 0;
+  const server = createServer((socket) => {
+    connections++;
+    if (connections <= resets) {
+      socket.resetAndDestroy();
+      return;
+    }
+    socket.once("data", () => {
+      const body = '{"status":"ready"}';
+      socket.end(`HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: ${body.length}\r\nConnection: close\r\n\r\n${body}`);
+    });
+  });
+  servers.push(server);
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const { port } = server.address() as AddressInfo;
+  return { url: `http://127.0.0.1:${port}/api/readiness`, connections: () => connections };
+}
+
+function curl(options: string[], url: string): Promise<{ exitCode: number; stdout: string }> {
+  return new Promise((resolve) => {
+    execFile("curl", [...options, url], { encoding: "utf8" }, (error, stdout) => {
+      resolve({ exitCode: error ? Number(error.code) : 0, stdout });
+    });
+  });
+}
+
+afterEach(async () => {
+  await Promise.all(servers.splice(0).map((server) => new Promise<void>((resolve) => server.close(() => resolve()))));
+});
+
+describe("Ubuntu/VBox installer readiness checks", () => {
+  it("finds both readiness checks of the installer", () => {
+    expect(Object.keys(readinessCurlOptions()).sort()).toEqual(["BACKEND_READINESS", "GATEWAY_READINESS"]);
+  });
+
+  it.each(["BACKEND_READINESS", "GATEWAY_READINESS"])(
+    "%s keeps retrying while the published port resets connections",
+    async (check) => {
+      const backend = await startResettingBackend(1);
+
+      const result = await curl(readinessCurlOptions()[check], backend.url);
+
+      expect(result).toEqual({ exitCode: 0, stdout: '{"status":"ready"}' });
+      expect(backend.connections()).toBe(2);
+    },
+    20_000,
+  );
+
+  it("would give up after the first reset with connection-refused retries alone", async () => {
+    // The former installer options: a fresh VM aborted with curl exit 56
+    // because the backend listened about 0.3 s after `docker compose up -d`.
+    const backend = await startResettingBackend(1);
+    const formerOptions = readinessCurlOptions().BACKEND_READINESS.filter((option) => option !== "--retry-all-errors");
+
+    const result = await curl(formerOptions, backend.url);
+
+    expect(result.exitCode).toBe(56);
+    expect(backend.connections()).toBe(1);
+  }, 20_000);
+});

vitest.config.ts

@@ -23,6 +23,7 @@ const serializedHttpUnitTests = [
   "tests/server/cache-optimization.test.ts",
   "tests/server/cli-label-isolation.test.ts",
   "tests/server/dev-entrypoint.test.ts",
+  "tests/server/ubuntu-vbox-installer-readiness.test.ts",
   "tests/server/routes/compiler.routes.test.ts",
   "tests/server/routes/compiler-binary-payload.test.ts",
   "tests/server/routes/api-rate-limit-key.test.ts",