fix: a failed published-atlases fetch can half-generate a tracker atlas and still exit 0 (#3203)

apis/tracker/api.ts

@@ -3,22 +3,51 @@ import type {
   TrackerSourceDatasetResponse,
   TrackerSourceStudy,
 } from "@/types/network";
+import { readFile } from "fs/promises";
+import { PHASE_PRODUCTION_BUILD } from "next/constants";
+import { inspect } from "util";
+import { PUBLISHED_ATLASES_SNAPSHOT_ENV } from "./constants";
 import type { PublishedAtlas } from "./types";
 
 // Module-level cache for published atlases during build. Stores the in-flight
-// Promise so concurrent callers share a single fetch.
+// Promise so concurrent callers share a single load.
 let publishedAtlasesPromise: Promise<PublishedAtlas[]> | null = null;
 
+// Advice appended when `next build` runs without a snapshot path.
+const BUILD_HINT =
+  "(run `npm run build`, which snapshots the published atlases before `next build`)";
+
+// Advice appended when the build's snapshot file is gone, e.g. removed by a tmp
+// cleaner mid-build.
+const SNAPSHOT_MISSING_HINT =
+  "(the snapshot `scripts/build.sh` wrote for this build was removed; run the build again)";
+
+// Fields an item must carry as strings for the publication gate to match on
+// it. Items missing one are skipped, and a body where every item is missing one
+// fails the build, so a renamed field cannot make every atlas silently read as
+// unpublished.
+const PUBLISHED_ATLAS_MATCH_KEYS = [
+  "shortNameSlug",
+  "version",
+] as const satisfies readonly (keyof PublishedAtlas)[];
+
 /**
- * Returns the tracker base URL, read at call time to ensure env vars are loaded.
- * @returns tracker base URL.
+ * Fetches and validates the tracker's published atlases. Used directly by
+ * `next dev` and by the snapshot script that runs before `next build`.
+ * Failures (network error, non-2xx, malformed body, missing tracker URL) are
+ * rethrown with the reason in the message.
+ * @returns published atlases.
  */
-function getTrackerUrl(): string {
-  const url = process.env.NEXT_PUBLIC_ATLAS_TRACKER_URL;
-  if (!url) {
-    throw new Error("NEXT_PUBLIC_ATLAS_TRACKER_URL is not configured");
+export async function fetchPublishedAtlases(): Promise<PublishedAtlas[]> {
+  try {
+    const data = await fetchTrackerApi<unknown>(
+      "/api/published-atlases",
+      "published atlases"
+    );
+    return parsePublishedAtlases(data);
+  } catch (err) {
+    throw publishedAtlasesError(err);
   }
-  return url;
 }
 
 /**
@@ -80,38 +109,52 @@ export function fetchTrackerSourceStudies(
 }
 
 /**
- * Returns the cached list of published atlases, fetching on first call.
- * On failure (network error, non-2xx, malformed body), logs a warning and
- * resolves to an empty list so callers can fall back to "no atlases
- * published" — protecting non-tracker pages from tracker outages.
- * @returns list of published atlases.
+ * Finds the published atlas with the given slug and version. When two records
+ * match, the first is returned.
+ * @param shortNameSlug - Atlas short name slug (e.g., "gut").
+ * @param version - Atlas version (e.g., "v1.0").
+ * @returns the published atlas, or undefined when it is not published.
+ */
+async function findPublishedAtlas(
+  shortNameSlug: string,
+  version: string
+): Promise<PublishedAtlas | undefined> {
+  const atlases = await getPublishedAtlases();
+  return atlases.find(
+    (atlas) =>
+      atlas.shortNameSlug === shortNameSlug && atlas.version === version
+  );
+}
+
+/**
+ * Returns the cached published atlases, loading them on first call. Failures
+ * are rethrown so the build fails deterministically rather than silently
+ * omitting tracker atlases; the rejected promise is cleared so a long-lived
+ * `next dev` server can retry on the next request.
+ * @returns published atlases.
  */
 function getPublishedAtlases(): Promise<PublishedAtlas[]> {
   if (!publishedAtlasesPromise) {
-    publishedAtlasesPromise = fetchTrackerApi<unknown>(
-      "/api/published-atlases",
-      "published atlases"
-    )
-      .then((data) => {
-        if (!Array.isArray(data)) {
-          console.warn(
-            `Tracker /api/published-atlases returned a non-array body; treating as empty.`
-          );
-          return [];
-        }
-        return data as PublishedAtlas[];
-      })
-      .catch((err) => {
-        console.warn(
-          `Failed to fetch published atlases from tracker; treating as empty. ${err}`
-        );
-        publishedAtlasesPromise = null;
-        return [];
-      });
+    publishedAtlasesPromise = loadPublishedAtlases().catch((err) => {
+      publishedAtlasesPromise = null;
+      throw err;
+    });
   }
   return publishedAtlasesPromise;
 }
 
+/**
+ * Returns the tracker base URL, read at call time to ensure env vars are loaded.
+ * @returns tracker base URL.
+ */
+function getTrackerUrl(): string {
+  const url = process.env.NEXT_PUBLIC_ATLAS_TRACKER_URL;
+  if (!url) {
+    throw new Error("NEXT_PUBLIC_ATLAS_TRACKER_URL is not configured");
+  }
+  return url;
+}
+
 /**
  * Checks whether a tracker atlas is currently published.
  * @param shortNameSlug - Atlas short name slug (e.g., "gut").
@@ -122,10 +165,107 @@ export async function isTrackerAtlasPublished(
   shortNameSlug: string,
   version: string
 ): Promise<boolean> {
-  const atlases = await getPublishedAtlases();
-  return atlases.some(
-    (a) => a.shortNameSlug === shortNameSlug && a.version === version
-  );
+  return (await findPublishedAtlas(shortNameSlug, version)) !== undefined;
+}
+
+/**
+ * Loads the published atlases. `next build` reads the snapshot that
+ * `scripts/build.sh` writes before the build: Next runs getStaticPaths and
+ * getStaticProps across several worker processes, each with its own
+ * module-level cache, so fetching per worker could let pages disagree on which
+ * atlases are published if the tracker changed mid-build, shipping tabs that
+ * 404 (see #3203). A `next build` without the snapshot (e.g. a bare `npx next
+ * build`) therefore fails. Anything else, such as `next dev`, fetches live,
+ * even if the snapshot variable leaked into its environment.
+ * @returns published atlases.
+ */
+function loadPublishedAtlases(): Promise<PublishedAtlas[]> {
+  if (process.env.NEXT_PHASE !== PHASE_PRODUCTION_BUILD) {
+    return fetchPublishedAtlases();
+  }
+  const snapshotPath = process.env[PUBLISHED_ATLASES_SNAPSHOT_ENV];
+  if (!snapshotPath) {
+    return Promise.reject(
+      publishedAtlasesError(
+        `${PUBLISHED_ATLASES_SNAPSHOT_ENV} is not set`,
+        BUILD_HINT
+      )
+    );
+  }
+  return readPublishedAtlasesSnapshot(snapshotPath);
+}
+
+/**
+ * Validates the published-atlases response. The body must be an array. Items
+ * missing a string `shortNameSlug` or `version` are skipped with a warning, so
+ * a malformed record for an atlas this portal does not configure cannot fail
+ * the build, but a body where every item is malformed fails it. Fields used
+ * only once an atlas is resolved (e.g. `id`) are checked by
+ * `resolveTrackerAtlas`.
+ * @param data - Parsed response body.
+ * @returns the well-formed items, typed as published atlases.
+ */
+function parsePublishedAtlases(data: unknown): PublishedAtlas[] {
+  if (!Array.isArray(data)) {
+    throw new Error("Tracker /api/published-atlases returned a non-array body");
+  }
+  const atlases = data.filter((item, i) => {
+    const missingKey = PUBLISHED_ATLAS_MATCH_KEYS.find(
+      (key) => typeof item?.[key] !== "string"
+    );
+    if (missingKey === undefined) return true;
+    console.warn(
+      `[tracker] Skipping /api/published-atlases item ${i}: no string "${missingKey}"`
+    );
+    return false;
+  });
+  if (data.length > 0 && atlases.length === 0) {
+    throw new Error(
+      `Tracker /api/published-atlases has no item with string ${PUBLISHED_ATLAS_MATCH_KEYS.join(" and ")} fields`
+    );
+  }
+  return atlases;
+}
+
+/**
+ * Builds the error thrown when the published atlases cannot be loaded. The
+ * reason goes in the message, not `cause`: Next's build workers send only an
+ * error's name, message and stack back to the main process, so a `cause` would
+ * never reach the build log. A string reason is used as is; anything else is
+ * printed with `util.inspect`, which includes the underlying error's stack,
+ * Node error details (code, address, port), `cause` chain and AggregateError
+ * `errors`.
+ * @param reason - The underlying failure, or a description of it.
+ * @param hint - Optional advice on how to fix it.
+ * @returns error describing the failure.
+ */
+function publishedAtlasesError(reason: unknown, hint?: string): Error {
+  const text =
+    typeof reason === "string"
+      ? reason
+      : inspect(reason, { breakLength: Infinity, depth: 5 });
+  const parts = [`[tracker] Published atlases are unavailable: ${text}`, hint];
+  return new Error(parts.filter(Boolean).join("\n"));
+}
+
+/**
+ * Reads this build's published-atlases snapshot.
+ * @param snapshotPath - Path of the snapshot written by `scripts/build.sh`.
+ * @returns published atlases.
+ */
+async function readPublishedAtlasesSnapshot(
+  snapshotPath: string
+): Promise<PublishedAtlas[]> {
+  try {
+    const text = await readFile(snapshotPath, "utf8");
+    return parsePublishedAtlases(JSON.parse(text));
+  } catch (err) {
+    const missing = (err as NodeJS.ErrnoException).code === "ENOENT";
+    throw publishedAtlasesError(
+      err,
+      missing ? SNAPSHOT_MISSING_HINT : undefined
+    );
+  }
 }
 
 /**
@@ -138,15 +278,17 @@ export async function resolveTrackerAtlas(
   shortNameSlug: string,
   version: string
 ): Promise<PublishedAtlas> {
-  const atlases = await getPublishedAtlases();
-  const match = atlases.find(
-    (a) => a.shortNameSlug === shortNameSlug && a.version === version
-  );
+  const match = await findPublishedAtlas(shortNameSlug, version);
   if (!match) {
     throw new Error(
       `No published atlas found for slug="${shortNameSlug}" version="${version}"`
     );
   }
+  if (typeof match.id !== "string") {
+    throw new Error(
+      `Published atlas slug="${shortNameSlug}" version="${version}" has no string "id"`
+    );
+  }
   return match;
 }
 

apis/tracker/constants.ts

@@ -0,0 +1,6 @@
+// Environment variable holding the path of this build's published-atlases
+// snapshot. `scripts/build.sh` writes the snapshot once before `next build`, to
+// a path unique to the build, and deletes it however the build ends; every
+// build worker reads it, so all pages in a build agree on which atlases are
+// published (see #3203).
+export const PUBLISHED_ATLASES_SNAPSHOT_ENV = "PUBLISHED_ATLASES_SNAPSHOT";

contexts/atlasContext.ts

@@ -1,10 +1,12 @@
 import { NETWORKS, NETWORK_ATLAS_CONTENT } from "@/constants/networks";
-import { AtlasContext as AtlasContextType, AtlasModule } from "@/types/network";
+import type {
+  AtlasContext as AtlasContextType,
+  AtlasModule,
+} from "@/types/network";
 import { createContext, useContext } from "react";
+import { DEFAULT_NETWORK } from "./networkContext";
 
-const DEFAULT_NETWORK = NETWORKS[0];
-
-const DEFAULT_ATLAS = DEFAULT_NETWORK.atlases[0];
+const DEFAULT_ATLAS = NETWORKS[0].atlases[0];
 
 export const AtlasContext = createContext<AtlasContextType>({
   atlas: DEFAULT_ATLAS,

contexts/networkContext.ts

@@ -1,12 +1,18 @@
 import { NETWORKS, NETWORK_CONTENT } from "@/constants/networks";
-import {
+import type {
+  Network,
   NetworkContext as NetworkContextType,
   NetworkModule,
 } from "@/types/network";
 import { createContext, useContext } from "react";
 
+// Default network for contexts rendered without a provider. It lists no
+// atlases, so such a component cannot list unpublished tracker atlases whose
+// pages were never generated (see #3203).
+export const DEFAULT_NETWORK: Network = { ...NETWORKS[0], atlases: [] };
+
 export const NetworkContext = createContext<NetworkContextType>({
-  network: NETWORKS[0],
+  network: DEFAULT_NETWORK,
 });
 
 export const NetworkProvider = NetworkContext.Provider;

hooks/useNetworkList.ts

@@ -1,8 +1,10 @@
-import { NETWORKS } from "@/constants/networks";
 import type { Network } from "@/types/network";
 import { createContext, useContext } from "react";
 
-const NetworkListContext = createContext<Network[]>(NETWORKS);
+// Empty by default: pages must provide the publication-gated list from
+// `getAvailableNetworks`, so a page that forgets to cannot list unpublished
+// tracker atlases linking to pages that were never generated (see #3203).
+const NetworkListContext = createContext<Network[]>([]);
 
 export const NetworkListProvider = NetworkListContext.Provider;
 

package.json

@@ -4,9 +4,9 @@
   "private": true,
   "scripts": {
     "dev": "./scripts/dev.sh data-portal && ./scripts/set-version.sh dev && next dev --webpack",
-    "build-dev:data-portal": "./scripts/common-build.sh data-portal dev && ./scripts/set-version.sh && next build --webpack && npm run postbuild",
-    "build-prod:data-portal": "./scripts/common-build.sh data-portal prod && ./scripts/set-version.sh && next build --webpack && npm run postbuild",
-    "build": "next build --webpack",
+    "build-dev:data-portal": "./scripts/common-build.sh data-portal dev && ./scripts/set-version.sh && npm run build",
+    "build-prod:data-portal": "./scripts/common-build.sh data-portal prod && ./scripts/set-version.sh && npm run build",
+    "build": "./scripts/build.sh",
     "start": "npx serve out",
     "lint": "eslint .",
     "check-format": "prettier --check .",

scripts/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Snapshot the tracker's published atlases once, to a path unique to this
+# build, so every `next build` worker agrees on which atlases are published
+# (see #3203). The trap deletes the snapshot however the build ends.
+TMP_DIR="${TMPDIR:-/tmp}"
+PUBLISHED_ATLASES_SNAPSHOT=$(mktemp "${TMP_DIR%/}/published-atlases.XXXXXX")
+export PUBLISHED_ATLASES_SNAPSHOT
+trap 'rm -f "$PUBLISHED_ATLASES_SNAPSHOT"' EXIT
+
+npx --no-install esrun ./scripts/snapshot-published-atlases.ts
+npx --no-install next build --webpack "$@"

scripts/snapshot-published-atlases.ts

@@ -0,0 +1,38 @@
+import { fetchPublishedAtlases } from "@/apis/tracker/api";
+import { PUBLISHED_ATLASES_SNAPSHOT_ENV } from "@/apis/tracker/constants";
+import fsp from "fs/promises";
+import { createRequire } from "module";
+import path from "path";
+
+snapshotPublishedAtlases().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+
+/**
+ * Fetches the tracker's published atlases once and writes them to the
+ * snapshot every `next build` worker reads, so all pages in a build agree on
+ * which atlases are published (see #3203). Run by `scripts/build.sh`, which
+ * sets the snapshot path.
+ */
+async function snapshotPublishedAtlases(): Promise<void> {
+  const snapshotPath = process.env[PUBLISHED_ATLASES_SNAPSHOT_ENV];
+  if (!snapshotPath) {
+    throw new Error(
+      `${PUBLISHED_ATLASES_SNAPSHOT_ENV} is not set; run \`npm run build\`, which sets it`
+    );
+  }
+  // Load `@next/env` through `next`, so env files are read by the exact copy
+  // `next build` uses, in the same order, rather than a separately pinned one.
+  const requireFromRoot = createRequire(path.resolve("package.json"));
+  const requireFromNext = createRequire(
+    requireFromRoot.resolve("next/package.json")
+  );
+  const { loadEnvConfig } = requireFromNext("@next/env") as {
+    loadEnvConfig: (dir: string) => unknown;
+  };
+  loadEnvConfig(process.cwd());
+  const atlases = await fetchPublishedAtlases();
+  await fsp.writeFile(snapshotPath, JSON.stringify(atlases, undefined, 2));
+  console.log(`Wrote ${atlases.length} published atlases to ${snapshotPath}`);
+}