From 6b9d387e1632070060dfd146567749107c1dd284 Mon Sep 17 00:00:00 2001 From: Fran McDade <18710366+frano-m@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:38:31 +1000 Subject: [PATCH 1/5] fix: a failed published-atlases fetch can half-generate a tracker atlas and still exit 0 (#3203) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AsKeyxfv4ZT3FdQaqtGBdV --- apis/tracker/api.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/apis/tracker/api.ts b/apis/tracker/api.ts index 9d17db3a..edb03d1a 100644 --- a/apis/tracker/api.ts +++ b/apis/tracker/api.ts @@ -81,9 +81,14 @@ export function fetchTrackerSourceStudies( /** * Returns the cached list of published atlases, fetching on first call. - * On failure (network error, non-2xx, malformed body), logs a warning and - * resolves to an empty list so callers can fall back to "no atlases - * published" — protecting non-tracker pages from tracker outages. + * Failures (network error, non-2xx, malformed body, missing tracker URL) + * are rethrown so the build fails deterministically rather than silently + * omitting tracker atlases. Next.js runs getStaticPaths and getStaticProps + * across several worker processes, each with its own module-level cache, so + * degrading to an empty list could not be made consistent across callers — + * one page could treat an atlas as published while another omitted it, + * shipping tabs that 404 (see #3203). The rejected promise is cleared so a + * long-lived `next dev` server can retry on the next request. * @returns list of published atlases. */ function getPublishedAtlases(): Promise { @@ -94,19 +99,17 @@ function getPublishedAtlases(): Promise { ) .then((data) => { if (!Array.isArray(data)) { - console.warn( - `Tracker /api/published-atlases returned a non-array body; treating as empty.` + throw new Error( + "Tracker /api/published-atlases returned a non-array body" ); - return []; } return data as PublishedAtlas[]; }) .catch((err) => { - console.warn( - `Failed to fetch published atlases from tracker; treating as empty. ${err}` - ); publishedAtlasesPromise = null; - return []; + throw new Error( + `[tracker] Failed to fetch published atlases; aborting build. ${err}` + ); }); } return publishedAtlasesPromise; From 54982bc3305ceef6bc849ed3206a379a11e8fdb4 Mon Sep 17 00:00:00 2001 From: Fran McDade <18710366+frano-m@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:07:58 +1000 Subject: [PATCH 2/5] fix: validate published-atlas items, keep the error cause, and share the publish filter (#3203) check that every published-atlases item has string id, shortnameslug and version so a renamed field fails the build instead of hiding every atlas. rethrow with the original error as `cause` and a message that suits both build and dev. extract filterpublishedatlases so the network pages and the home page network list cannot disagree on which atlases are published. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AsKeyxfv4ZT3FdQaqtGBdV --- apis/tracker/api.ts | 52 ++++++++++++++++++++++++++++---------- utils/availableNetworks.ts | 42 +++++++++++++++++------------- utils/networkPages.ts | 12 ++------- 3 files changed, 66 insertions(+), 40 deletions(-) diff --git a/apis/tracker/api.ts b/apis/tracker/api.ts index edb03d1a..94f0f2b0 100644 --- a/apis/tracker/api.ts +++ b/apis/tracker/api.ts @@ -9,6 +9,37 @@ import type { PublishedAtlas } from "./types"; // Promise so concurrent callers share a single fetch. let publishedAtlasesPromise: Promise | null = null; +// Fields the publication gate matches on or resolves; each must be a string. +const PUBLISHED_ATLAS_REQUIRED_KEYS = [ + "id", + "shortNameSlug", + "version", +] as const satisfies readonly (keyof PublishedAtlas)[]; + +/** + * Validates the published-atlases response. Checks that the body is an array + * and that every item carries the fields the publication gate matches on, so + * a renamed or missing field fails the build instead of every atlas silently + * reading as unpublished. + * @param data - Parsed response body. + * @returns the body typed as published atlases. + */ +function assertPublishedAtlases(data: unknown): PublishedAtlas[] { + if (!Array.isArray(data)) { + throw new Error("Tracker /api/published-atlases returned a non-array body"); + } + data.forEach((item, i) => { + for (const key of PUBLISHED_ATLAS_REQUIRED_KEYS) { + if (typeof item?.[key] !== "string") { + throw new Error( + `Tracker /api/published-atlases item ${i} has no string "${key}"` + ); + } + } + }); + return data as PublishedAtlas[]; +} + /** * Returns the tracker base URL, read at call time to ensure env vars are loaded. * @returns tracker base URL. @@ -81,8 +112,8 @@ export function fetchTrackerSourceStudies( /** * Returns the cached list of published atlases, fetching on first call. - * Failures (network error, non-2xx, malformed body, missing tracker URL) - * are rethrown so the build fails deterministically rather than silently + * Failures (network error, non-2xx, malformed body or items, missing tracker + * URL) are rethrown so the build fails deterministically rather than silently * omitting tracker atlases. Next.js runs getStaticPaths and getStaticProps * across several worker processes, each with its own module-level cache, so * degrading to an empty list could not be made consistent across callers — @@ -97,19 +128,14 @@ function getPublishedAtlases(): Promise { "/api/published-atlases", "published atlases" ) - .then((data) => { - if (!Array.isArray(data)) { - throw new Error( - "Tracker /api/published-atlases returned a non-array body" - ); - } - return data as PublishedAtlas[]; - }) + .then(assertPublishedAtlases) .catch((err) => { publishedAtlasesPromise = null; - throw new Error( - `[tracker] Failed to fetch published atlases; aborting build. ${err}` - ); + // Keep the original error as `cause` so the underlying reason (e.g. + // ECONNREFUSED behind undici's "fetch failed") reaches the log. + throw new Error("[tracker] Published atlases are unavailable", { + cause: err, + }); }); } return publishedAtlasesPromise; diff --git a/utils/availableNetworks.ts b/utils/availableNetworks.ts index 7916fd22..0394fe72 100644 --- a/utils/availableNetworks.ts +++ b/utils/availableNetworks.ts @@ -1,6 +1,26 @@ import { isTrackerAtlasPublished } from "@/apis/tracker/api"; import { NETWORKS } from "@/constants/networks"; -import type { Network } from "@/types/network"; +import type { Atlas, Network } from "@/types/network"; + +/** + * Drops tracker atlases that are not currently published in the tracker; + * non-tracker atlases are always kept. Shared by every build-time caller that + * lists atlases so they cannot disagree on which atlases are published. + * @param atlases - Atlases to filter. + * @returns the atlases that should be shown, in their original order. + */ +export async function filterPublishedAtlases( + atlases: Atlas[] +): Promise { + const flags = await Promise.all( + atlases.map((a) => + a.tracker + ? isTrackerAtlasPublished(a.tracker.shortNameSlug, a.tracker.version) + : Promise.resolve(true) + ) + ); + return atlases.filter((_, i) => flags[i]); +} /** * Returns NETWORKS with unpublished tracker atlases filtered out. Used at @@ -11,21 +31,9 @@ import type { Network } from "@/types/network"; */ export async function getAvailableNetworks(): Promise { return Promise.all( - NETWORKS.map(async (network) => { - const flags = await Promise.all( - network.atlases.map((a) => - a.tracker - ? isTrackerAtlasPublished( - a.tracker.shortNameSlug, - a.tracker.version - ) - : Promise.resolve(true) - ) - ); - return { - ...network, - atlases: network.atlases.filter((_, i) => flags[i]), - }; - }) + NETWORKS.map(async (network) => ({ + ...network, + atlases: await filterPublishedAtlases(network.atlases), + })) ); } diff --git a/utils/networkPages.ts b/utils/networkPages.ts index ff806fae..50d36ca5 100644 --- a/utils/networkPages.ts +++ b/utils/networkPages.ts @@ -1,6 +1,5 @@ import { fetchTrackerComponentAtlases, - isTrackerAtlasPublished, resolveTrackerAtlasId, } from "@/apis/tracker/api"; import { NETWORKS } from "@/constants/networks"; @@ -10,6 +9,7 @@ import type { GetStaticPropsContext, GetStaticPropsResult, } from "next"; +import { filterPublishedAtlases } from "./availableNetworks"; import { fetchCXGDatasetsForAtlases, processNetwork } from "./network"; import { mapTrackerComponentAtlasToIntegratedAtlas } from "./trackerNetwork"; @@ -32,15 +32,7 @@ export async function getContentStaticProps( const network = NETWORKS.find(({ path }) => path === networkParam) as Network; - // Drop tracker atlases not currently published in the tracker. - const publishedFlags = await Promise.all( - network.atlases.map((a) => - a.tracker - ? isTrackerAtlasPublished(a.tracker.shortNameSlug, a.tracker.version) - : Promise.resolve(true) - ) - ); - const availableAtlases = network.atlases.filter((_, i) => publishedFlags[i]); + const availableAtlases = await filterPublishedAtlases(network.atlases); // Fetch CELLxGENE datasets for the network atlases. const cxgDatasets = await fetchCXGDatasetsForAtlases(availableAtlases); From 4c46b99b4ff9e5d40577505b7d45132df6e8f0df Mon Sep 17 00:00:00 2001 From: Fran McDade <18710366+frano-m@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:18:55 +1000 Subject: [PATCH 3/5] fix: log the tracker failure reason and gate every network prop in one place (#3203) Co-Authored-By: Claude Opus 5.5 (1M context) --- apis/tracker/api.ts | 107 ++++++++++++++++++++++--------- utils/atlasPages.ts | 33 +++++----- utils/availableNetworks.ts | 56 +++++++++++----- utils/networkPages.ts | 12 ++-- utils/networkPublicationPages.ts | 7 +- 5 files changed, 144 insertions(+), 71 deletions(-) diff --git a/apis/tracker/api.ts b/apis/tracker/api.ts index 94f0f2b0..0168e82a 100644 --- a/apis/tracker/api.ts +++ b/apis/tracker/api.ts @@ -5,13 +5,15 @@ import type { } from "@/types/network"; import type { PublishedAtlas } from "./types"; -// Module-level cache for published atlases during build. Stores the in-flight -// Promise so concurrent callers share a single fetch. -let publishedAtlasesPromise: Promise | null = null; +// Module-level cache for published atlases during build, keyed by +// `publishedAtlasKey`. Stores the in-flight Promise so concurrent callers share +// a single fetch. +let publishedAtlasesPromise: Promise> | null = null; -// Fields the publication gate matches on or resolves; each must be a string. -const PUBLISHED_ATLAS_REQUIRED_KEYS = [ - "id", +// Fields every item must carry as strings: the publication gate matches on +// them, so a renamed or missing field has to fail the build rather than make +// every atlas silently read as unpublished. +const PUBLISHED_ATLAS_MATCH_KEYS = [ "shortNameSlug", "version", ] as const satisfies readonly (keyof PublishedAtlas)[]; @@ -19,8 +21,10 @@ const PUBLISHED_ATLAS_REQUIRED_KEYS = [ /** * Validates the published-atlases response. Checks that the body is an array * and that every item carries the fields the publication gate matches on, so - * a renamed or missing field fails the build instead of every atlas silently - * reading as unpublished. + * any item missing a string `shortNameSlug` or `version` fails the build. + * Fields used only once an atlas is resolved (e.g. `id`) are checked by + * `resolveTrackerAtlas`, so a record for an atlas this portal does not + * configure cannot fail the build over those. * @param data - Parsed response body. * @returns the body typed as published atlases. */ @@ -29,7 +33,7 @@ function assertPublishedAtlases(data: unknown): PublishedAtlas[] { throw new Error("Tracker /api/published-atlases returned a non-array body"); } data.forEach((item, i) => { - for (const key of PUBLISHED_ATLAS_REQUIRED_KEYS) { + for (const key of PUBLISHED_ATLAS_MATCH_KEYS) { if (typeof item?.[key] !== "string") { throw new Error( `Tracker /api/published-atlases item ${i} has no string "${key}"` @@ -40,6 +44,17 @@ function assertPublishedAtlases(data: unknown): PublishedAtlas[] { return data as PublishedAtlas[]; } +/** + * Describes an error for a log message, appending its `cause` when it has one + * (e.g. the ECONNREFUSED behind undici's "fetch failed"). + * @param err - Error to describe. + * @returns the error and its cause as text. + */ +function describeError(err: unknown): string { + if (err instanceof Error && err.cause) return `${err} (cause: ${err.cause})`; + return String(err); +} + /** * Returns the tracker base URL, read at call time to ensure env vars are loaded. * @returns tracker base URL. @@ -111,31 +126,33 @@ export function fetchTrackerSourceStudies( } /** - * Returns the cached list of published atlases, fetching on first call. - * Failures (network error, non-2xx, malformed body or items, missing tracker - * URL) are rethrown so the build fails deterministically rather than silently - * omitting tracker atlases. Next.js runs getStaticPaths and getStaticProps - * across several worker processes, each with its own module-level cache, so - * degrading to an empty list could not be made consistent across callers — - * one page could treat an atlas as published while another omitted it, - * shipping tabs that 404 (see #3203). The rejected promise is cleared so a - * long-lived `next dev` server can retry on the next request. - * @returns list of published atlases. + * Returns the cached published atlases keyed by `publishedAtlasKey`, fetching + * on first call. Failures (network error, non-2xx, malformed body or items, + * missing tracker URL) are rethrown so the build fails deterministically rather + * than silently omitting tracker atlases. Next.js runs getStaticPaths and + * getStaticProps across several worker processes, each with its own + * module-level cache, so degrading to an empty list could not be made + * consistent across callers — one page could treat an atlas as published while + * another omitted it, shipping tabs that 404 (see #3203). The rejected promise + * is cleared so a long-lived `next dev` server can retry on the next request. + * @returns published atlases keyed by slug and version. */ -function getPublishedAtlases(): Promise { +function getPublishedAtlases(): Promise> { if (!publishedAtlasesPromise) { publishedAtlasesPromise = fetchTrackerApi( "/api/published-atlases", "published atlases" ) .then(assertPublishedAtlases) + .then(mapPublishedAtlases) .catch((err) => { publishedAtlasesPromise = null; - // Keep the original error as `cause` so the underlying reason (e.g. - // ECONNREFUSED behind undici's "fetch failed") reaches the log. - throw new Error("[tracker] Published atlases are unavailable", { - cause: err, - }); + // The reason goes in the message, not `cause`: Next's build workers + // send only an error's name, message and stack back to the main + // process, so a `cause` would never reach the build log. + throw new Error( + `[tracker] Published atlases are unavailable: ${describeError(err)}` + ); }); } return publishedAtlasesPromise; @@ -152,9 +169,34 @@ export async function isTrackerAtlasPublished( version: string ): Promise { const atlases = await getPublishedAtlases(); - return atlases.some( - (a) => a.shortNameSlug === shortNameSlug && a.version === version - ); + return atlases.has(publishedAtlasKey(shortNameSlug, version)); +} + +/** + * Keys published atlases by `publishedAtlasKey`. When two records share a + * slug and version the first is kept, as the previous list lookup did. + * @param atlases - Validated published atlases. + * @returns published atlases keyed by slug and version. + */ +function mapPublishedAtlases( + atlases: PublishedAtlas[] +): Map { + const byKey = new Map(); + for (const atlas of atlases) { + const key = publishedAtlasKey(atlas.shortNameSlug, atlas.version); + if (!byKey.has(key)) byKey.set(key, atlas); + } + return byKey; +} + +/** + * Returns the key a published atlas is cached under. + * @param shortNameSlug - Atlas short name slug (e.g., "gut"). + * @param version - Atlas version (e.g., "v1.0"). + * @returns key combining slug and version. + */ +function publishedAtlasKey(shortNameSlug: string, version: string): string { + return `${shortNameSlug}@${version}`; } /** @@ -168,14 +210,17 @@ export async function resolveTrackerAtlas( version: string ): Promise { const atlases = await getPublishedAtlases(); - const match = atlases.find( - (a) => a.shortNameSlug === shortNameSlug && a.version === version - ); + const match = atlases.get(publishedAtlasKey(shortNameSlug, version)); if (!match) { throw new Error( `No published atlas found for slug="${shortNameSlug}" version="${version}"` ); } + if (typeof match.id !== "string") { + throw new Error( + `Published atlas slug="${shortNameSlug}" version="${version}" has no string "id"` + ); + } return match; } diff --git a/utils/atlasPages.ts b/utils/atlasPages.ts index 376a8a0d..f2103ca2 100644 --- a/utils/atlasPages.ts +++ b/utils/atlasPages.ts @@ -1,7 +1,6 @@ import { filterProjectId } from "@/apis/azul/hca-dcp/common/filters"; import type { ProjectsResponse } from "@/apis/azul/hca-dcp/common/responses"; import { processEntityValue } from "@/apis/azul/hca-dcp/common/utils"; -import { isTrackerAtlasPublished } from "@/apis/tracker/api"; import { config } from "@/config/config"; import { NETWORKS } from "@/constants/networks"; import type { Atlas, AtlasContext, CXGDataset, Network } from "@/types/network"; @@ -14,6 +13,11 @@ import type { GetStaticPropsResult, } from "next"; import type { ParsedUrlQuery } from "querystring"; +import { + getAvailableNetwork, + isAtlasPublished, + isPublishedTrackerAtlas, +} from "./availableNetworks"; import { fetchCXGDatasetsForAtlases, processAtlas, @@ -35,9 +39,7 @@ export interface TrackerDataOptions { } export const getStaticPaths: GetStaticPaths = () => - buildStaticPaths((atlas) => - atlas.tracker ? isPublishedTrackerAtlas(atlas) : true - ); + buildStaticPaths(isAtlasPublished); /** * Static paths for non-tracker atlases only. Tracker-sourced atlases are @@ -70,10 +72,18 @@ export async function getContentStaticProps( ): Promise> { const { atlas: atlasParam, network: networkParam } = context.params ?? {}; - const network = NETWORKS.find(({ path }) => path === networkParam) as Network; - const atlas = network.atlases.find( + const configuredNetwork = NETWORKS.find( + ({ path }) => path === networkParam + ) as Network; + // Found in the configured network, not the filtered one, so an atlas that + // reads as unpublished here still reaches `resolveTrackerAtlas` and its + // clear error rather than crashing on an undefined atlas. + const atlas = configuredNetwork.atlases.find( ({ path }) => path === atlasParam ) as Atlas; + // Drop unpublished sibling atlases so the page's network agrees with the + // network and home pages. + const network = await getAvailableNetwork(configuredNetwork); // Delegate to tracker path if atlas has tracker config. if (atlas.tracker) { @@ -151,17 +161,6 @@ async function fetchAtlasProjects(atlas: Atlas): Promise { return hits; } -/** - * Returns true when the atlas is tracker-sourced and published in the tracker. - * @param atlas - Atlas to check. - * @returns true if the atlas is a published tracker atlas. - */ -async function isPublishedTrackerAtlas(atlas: Atlas): Promise { - if (!atlas.tracker) return false; - const { shortNameSlug, version } = atlas.tracker; - return isTrackerAtlasPublished(shortNameSlug, version); -} - /** * Sort datasets by cell count, descending. * @param d0 - First dataset to compare. diff --git a/utils/availableNetworks.ts b/utils/availableNetworks.ts index 0394fe72..d1133844 100644 --- a/utils/availableNetworks.ts +++ b/utils/availableNetworks.ts @@ -2,26 +2,35 @@ import { isTrackerAtlasPublished } from "@/apis/tracker/api"; import { NETWORKS } from "@/constants/networks"; import type { Atlas, Network } from "@/types/network"; +// The publication gate: every build-time decision about whether an atlas is +// shown or gets pages goes through this module, so callers cannot disagree on +// which atlases are published (see #3203). + /** * Drops tracker atlases that are not currently published in the tracker; - * non-tracker atlases are always kept. Shared by every build-time caller that - * lists atlases so they cannot disagree on which atlases are published. + * non-tracker atlases are always kept. * @param atlases - Atlases to filter. * @returns the atlases that should be shown, in their original order. */ export async function filterPublishedAtlases( atlases: Atlas[] ): Promise { - const flags = await Promise.all( - atlases.map((a) => - a.tracker - ? isTrackerAtlasPublished(a.tracker.shortNameSlug, a.tracker.version) - : Promise.resolve(true) - ) - ); + const flags = await Promise.all(atlases.map(isAtlasPublished)); return atlases.filter((_, i) => flags[i]); } +/** + * Returns the network with unpublished tracker atlases filtered out. + * @param network - Network to filter. + * @returns network with only available atlases. + */ +export async function getAvailableNetwork(network: Network): Promise { + return { + ...network, + atlases: await filterPublishedAtlases(network.atlases), + }; +} + /** * Returns NETWORKS with unpublished tracker atlases filtered out. Used at * build time by pages that surface atlas-level data (home page badges, @@ -30,10 +39,27 @@ export async function filterPublishedAtlases( * @returns networks with unpublished tracker atlases filtered out. */ export async function getAvailableNetworks(): Promise { - return Promise.all( - NETWORKS.map(async (network) => ({ - ...network, - atlases: await filterPublishedAtlases(network.atlases), - })) - ); + return Promise.all(NETWORKS.map(getAvailableNetwork)); +} + +/** + * Returns true when the atlas should be shown: non-tracker atlases always are, + * tracker atlases only while published in the tracker. + * @param atlas - Atlas to check. + * @returns true if the atlas is published. + */ +export async function isAtlasPublished(atlas: Atlas): Promise { + if (!atlas.tracker) return true; + const { shortNameSlug, version } = atlas.tracker; + return isTrackerAtlasPublished(shortNameSlug, version); +} + +/** + * Returns true when the atlas is tracker-sourced and published in the tracker. + * @param atlas - Atlas to check. + * @returns true if the atlas is a published tracker atlas. + */ +export async function isPublishedTrackerAtlas(atlas: Atlas): Promise { + if (!atlas.tracker) return false; + return isAtlasPublished(atlas); } diff --git a/utils/networkPages.ts b/utils/networkPages.ts index 50d36ca5..14328657 100644 --- a/utils/networkPages.ts +++ b/utils/networkPages.ts @@ -9,7 +9,7 @@ import type { GetStaticPropsContext, GetStaticPropsResult, } from "next"; -import { filterPublishedAtlases } from "./availableNetworks"; +import { getAvailableNetwork } from "./availableNetworks"; import { fetchCXGDatasetsForAtlases, processNetwork } from "./network"; import { mapTrackerComponentAtlasToIntegratedAtlas } from "./trackerNetwork"; @@ -30,16 +30,16 @@ export async function getContentStaticProps( ): Promise> { const { network: networkParam } = context.params ?? {}; - const network = NETWORKS.find(({ path }) => path === networkParam) as Network; - - const availableAtlases = await filterPublishedAtlases(network.atlases); + const network = await getAvailableNetwork( + NETWORKS.find(({ path }) => path === networkParam) as Network + ); // Fetch CELLxGENE datasets for the network atlases. - const cxgDatasets = await fetchCXGDatasetsForAtlases(availableAtlases); + const cxgDatasets = await fetchCXGDatasetsForAtlases(network.atlases); // Populate integrated atlases for tracker-sourced atlases immutably. const atlases = await Promise.all( - availableAtlases.map(async (atlas) => { + network.atlases.map(async (atlas) => { if (!atlas.tracker) return atlas; const { shortNameSlug, version } = atlas.tracker; const atlasId = await resolveTrackerAtlasId(shortNameSlug, version); diff --git a/utils/networkPublicationPages.ts b/utils/networkPublicationPages.ts index c71d5424..8360711d 100644 --- a/utils/networkPublicationPages.ts +++ b/utils/networkPublicationPages.ts @@ -2,6 +2,7 @@ import { NETWORKS } from "@/constants/networks"; import { Network, NetworkParam } from "@/types/network"; import { GetStaticPaths, GetStaticProps, GetStaticPropsContext } from "next"; import { GetStaticPathsResult } from "next/types"; +import { getAvailableNetwork } from "./availableNetworks"; export const getStaticPaths: GetStaticPaths = async () => { return { @@ -14,10 +15,12 @@ export const getStaticProps: GetStaticProps = async ( context: GetStaticPropsContext ) => { const { network: networkParam } = context.params ?? {}; - const network = NETWORKS.find(({ path }) => path === networkParam) as Network; + const network = await getAvailableNetwork( + NETWORKS.find(({ path }) => path === networkParam) as Network + ); return { props: { - network: network, + network, pageTitle: `${network.name} - BICCN Publications`, }, }; From b8dfe5a204c694a6c9e41e764d15a10a0144b84b Mon Sep 17 00:00:00 2001 From: Fran McDade <18710366+frano-m@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:06:18 +1000 Subject: [PATCH 4/5] fix: snapshot published atlases once per build and address review feedback (#3203) Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitignore | 3 + apis/tracker/api.ts | 290 ++++++++++++++++++-------- apis/tracker/constants.ts | 7 + hooks/useNetworkList.ts | 6 +- package-lock.json | 1 + package.json | 11 +- scripts/snapshot-published-atlases.ts | 33 +++ utils/availableNetworks.ts | 6 +- utils/networkPublicationPages.ts | 20 +- 9 files changed, 271 insertions(+), 106 deletions(-) create mode 100644 apis/tracker/constants.ts create mode 100644 scripts/snapshot-published-atlases.ts diff --git a/.gitignore b/.gitignore index e471f810..1fc0de03 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ # production /build +# tracker snapshot written before next build +/.tracker/ + # misc .DS_Store *.pem diff --git a/apis/tracker/api.ts b/apis/tracker/api.ts index 0168e82a..98e56dba 100644 --- a/apis/tracker/api.ts +++ b/apis/tracker/api.ts @@ -3,68 +3,96 @@ import type { TrackerSourceDatasetResponse, TrackerSourceStudy, } from "@/types/network"; +import { readFile } from "fs/promises"; +import { PUBLISHED_ATLASES_SNAPSHOT_PATH } from "./constants"; import type { PublishedAtlas } from "./types"; -// Module-level cache for published atlases during build, keyed by -// `publishedAtlasKey`. Stores the in-flight Promise so concurrent callers share -// a single fetch. -let publishedAtlasesPromise: Promise> | null = null; +// Module-level cache for published atlases during build. Stores the in-flight +// Promise so concurrent callers share a single load. +let publishedAtlasesPromise: Promise | null = null; -// Fields every item must carry as strings: the publication gate matches on -// them, so a renamed or missing field has to fail the build rather than make -// every atlas silently read as unpublished. +// Node socket-error fields worth logging, e.g. the host and port a fetch could +// not reach. +const ERROR_DETAIL_KEYS = [ + "code", + "syscall", + "hostname", + "address", + "port", +] as const; + +// How deep `describeError` follows `cause` and AggregateError `errors`, so a +// cyclic cause cannot recurse forever. +const MAX_ERROR_DEPTH = 5; + +// Fields an item must carry as strings for the publication gate to match on +// it. Items missing one are skipped, and a body where every item is missing one +// fails the build, so a renamed field cannot make every atlas silently read as +// unpublished. const PUBLISHED_ATLAS_MATCH_KEYS = [ "shortNameSlug", "version", ] as const satisfies readonly (keyof PublishedAtlas)[]; /** - * Validates the published-atlases response. Checks that the body is an array - * and that every item carries the fields the publication gate matches on, so - * any item missing a string `shortNameSlug` or `version` fails the build. - * Fields used only once an atlas is resolved (e.g. `id`) are checked by - * `resolveTrackerAtlas`, so a record for an atlas this portal does not - * configure cannot fail the build over those. - * @param data - Parsed response body. - * @returns the body typed as published atlases. + * Describes an error for a log message: its name and message, any Node + * socket-error details (code, host, port), the errors inside an AggregateError + * (e.g. undici's "fetch failed" to a dual-stack host, one per address tried), + * and its `cause` chain. + * @param err - Error to describe. + * @param depth - Current nesting depth; callers omit it. + * @returns the error, its details and its causes as text. */ -function assertPublishedAtlases(data: unknown): PublishedAtlas[] { - if (!Array.isArray(data)) { - throw new Error("Tracker /api/published-atlases returned a non-array body"); +function describeError(err: unknown, depth = 0): string { + if (!(err instanceof Error)) return describeValue(err); + const fields = err as unknown as Record; + const details = ERROR_DETAIL_KEYS.filter( + (key) => fields[key] !== undefined + ).map((key) => `${key}=${fields[key]}`); + let text = details.length ? `${err} [${details.join(" ")}]` : String(err); + if (depth >= MAX_ERROR_DEPTH) return text; + if (err instanceof AggregateError && err.errors.length > 0) { + const errors = err.errors.map((e) => describeError(e, depth + 1)); + text += ` (errors: ${errors.join("; ")})`; } - data.forEach((item, i) => { - for (const key of PUBLISHED_ATLAS_MATCH_KEYS) { - if (typeof item?.[key] !== "string") { - throw new Error( - `Tracker /api/published-atlases item ${i} has no string "${key}"` - ); - } - } - }); - return data as PublishedAtlas[]; + if (err.cause !== undefined) { + text += ` (cause: ${describeError(err.cause, depth + 1)})`; + } + return text; } /** - * Describes an error for a log message, appending its `cause` when it has one - * (e.g. the ECONNREFUSED behind undici's "fetch failed"). - * @param err - Error to describe. - * @returns the error and its cause as text. + * Describes a non-Error thrown value or cause, printing plain objects as JSON + * rather than "[object Object]". + * @param value - Value to describe. + * @returns the value as text. */ -function describeError(err: unknown): string { - if (err instanceof Error && err.cause) return `${err} (cause: ${err.cause})`; - return String(err); +function describeValue(value: unknown): string { + if (typeof value !== "object" || value === null) return String(value); + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } } /** - * Returns the tracker base URL, read at call time to ensure env vars are loaded. - * @returns tracker base URL. + * Fetches and validates the tracker's published atlases. Used directly by + * `next dev` and by the snapshot script that runs before `next build`. + * Failures (network error, non-2xx, malformed body, missing tracker URL) are + * rethrown with the reason in the message. + * @returns published atlases. */ -function getTrackerUrl(): string { - const url = process.env.NEXT_PUBLIC_ATLAS_TRACKER_URL; - if (!url) { - throw new Error("NEXT_PUBLIC_ATLAS_TRACKER_URL is not configured"); +export async function fetchPublishedAtlases(): Promise { + try { + const data = await fetchTrackerApi( + "/api/published-atlases", + "published atlases" + ); + return parsePublishedAtlases(data); + } catch (err) { + throw publishedAtlasesError(err); } - return url; } /** @@ -126,38 +154,61 @@ export function fetchTrackerSourceStudies( } /** - * Returns the cached published atlases keyed by `publishedAtlasKey`, fetching - * on first call. Failures (network error, non-2xx, malformed body or items, - * missing tracker URL) are rethrown so the build fails deterministically rather - * than silently omitting tracker atlases. Next.js runs getStaticPaths and - * getStaticProps across several worker processes, each with its own - * module-level cache, so degrading to an empty list could not be made - * consistent across callers — one page could treat an atlas as published while - * another omitted it, shipping tabs that 404 (see #3203). The rejected promise - * is cleared so a long-lived `next dev` server can retry on the next request. - * @returns published atlases keyed by slug and version. + * Finds the published atlas with the given slug and version. When two records + * match, the first is returned. + * @param shortNameSlug - Atlas short name slug (e.g., "gut"). + * @param version - Atlas version (e.g., "v1.0"). + * @returns the published atlas, or undefined when it is not published. */ -function getPublishedAtlases(): Promise> { +async function findPublishedAtlas( + shortNameSlug: string, + version: string +): Promise { + const atlases = await getPublishedAtlases(); + return atlases.find( + (atlas) => + atlas.shortNameSlug === shortNameSlug && atlas.version === version + ); +} + +/** + * Returns the cached published atlases, loading them on first call. `next + * build` reads the snapshot written before the build: Next runs getStaticPaths + * and getStaticProps across several worker processes, each with its own + * module-level cache, so fetching per worker could let pages disagree on which + * atlases are published if the tracker changed mid-build, shipping tabs that + * 404 (see #3203). `next dev` fetches live. Failures are rethrown so the build + * fails deterministically rather than silently omitting tracker atlases; the + * rejected promise is cleared so a long-lived `next dev` server can retry on + * the next request. + * @returns published atlases. + */ +function getPublishedAtlases(): Promise { if (!publishedAtlasesPromise) { - publishedAtlasesPromise = fetchTrackerApi( - "/api/published-atlases", - "published atlases" - ) - .then(assertPublishedAtlases) - .then(mapPublishedAtlases) - .catch((err) => { - publishedAtlasesPromise = null; - // The reason goes in the message, not `cause`: Next's build workers - // send only an error's name, message and stack back to the main - // process, so a `cause` would never reach the build log. - throw new Error( - `[tracker] Published atlases are unavailable: ${describeError(err)}` - ); - }); + const load = + process.env.NODE_ENV === "production" + ? readPublishedAtlasesSnapshot + : fetchPublishedAtlases; + publishedAtlasesPromise = load().catch((err) => { + publishedAtlasesPromise = null; + throw err; + }); } return publishedAtlasesPromise; } +/** + * Returns the tracker base URL, read at call time to ensure env vars are loaded. + * @returns tracker base URL. + */ +function getTrackerUrl(): string { + const url = process.env.NEXT_PUBLIC_ATLAS_TRACKER_URL; + if (!url) { + throw new Error("NEXT_PUBLIC_ATLAS_TRACKER_URL is not configured"); + } + return url; +} + /** * Checks whether a tracker atlas is currently published. * @param shortNameSlug - Atlas short name slug (e.g., "gut"). @@ -168,35 +219,75 @@ export async function isTrackerAtlasPublished( shortNameSlug: string, version: string ): Promise { - const atlases = await getPublishedAtlases(); - return atlases.has(publishedAtlasKey(shortNameSlug, version)); + return (await findPublishedAtlas(shortNameSlug, version)) !== undefined; } /** - * Keys published atlases by `publishedAtlasKey`. When two records share a - * slug and version the first is kept, as the previous list lookup did. - * @param atlases - Validated published atlases. - * @returns published atlases keyed by slug and version. + * Validates the published-atlases response. The body must be an array. Items + * missing a string `shortNameSlug` or `version` are skipped with a warning, so + * a malformed record for an atlas this portal does not configure cannot fail + * the build, but a body where every item is malformed fails it. Fields used + * only once an atlas is resolved (e.g. `id`) are checked by + * `resolveTrackerAtlas`. + * @param data - Parsed response body. + * @returns the well-formed items, typed as published atlases. */ -function mapPublishedAtlases( - atlases: PublishedAtlas[] -): Map { - const byKey = new Map(); - for (const atlas of atlases) { - const key = publishedAtlasKey(atlas.shortNameSlug, atlas.version); - if (!byKey.has(key)) byKey.set(key, atlas); +function parsePublishedAtlases(data: unknown): PublishedAtlas[] { + if (!Array.isArray(data)) { + throw new Error("Tracker /api/published-atlases returned a non-array body"); + } + const atlases = data.filter((item, i) => { + const missingKey = PUBLISHED_ATLAS_MATCH_KEYS.find( + (key) => typeof item?.[key] !== "string" + ); + if (missingKey === undefined) return true; + console.warn( + `[tracker] Skipping /api/published-atlases item ${i}: no string "${missingKey}"` + ); + return false; + }); + if (data.length > 0 && atlases.length === 0) { + throw new Error( + `Tracker /api/published-atlases has no item with string ${PUBLISHED_ATLAS_MATCH_KEYS.join(" and ")} fields` + ); } - return byKey; + return atlases; } /** - * Returns the key a published atlas is cached under. - * @param shortNameSlug - Atlas short name slug (e.g., "gut"). - * @param version - Atlas version (e.g., "v1.0"). - * @returns key combining slug and version. + * Builds the error thrown when the published atlases cannot be loaded. The + * reason goes in the message, not `cause`: Next's build workers send only an + * error's name, message and stack back to the main process, so a `cause` would + * never reach the build log. This error's own stack points at the handler, so + * the failing step's first stack frame in app code is appended too. + * @param err - The underlying failure. + * @param hint - Optional advice on how to fix it. + * @returns error describing the failure. */ -function publishedAtlasKey(shortNameSlug: string, version: string): string { - return `${shortNameSlug}@${version}`; +function publishedAtlasesError(err: unknown, hint?: string): Error { + const frame = topAppStackFrame(err); + const parts = [ + `[tracker] Published atlases are unavailable: ${describeError(err)}`, + frame, + hint, + ]; + return new Error(parts.filter(Boolean).join(" ")); +} + +/** + * Reads the published-atlases snapshot written before `next build`. + * @returns published atlases. + */ +async function readPublishedAtlasesSnapshot(): Promise { + try { + const text = await readFile(PUBLISHED_ATLASES_SNAPSHOT_PATH, "utf8"); + return parsePublishedAtlases(JSON.parse(text)); + } catch (err) { + throw publishedAtlasesError( + err, + `(write the snapshot with \`npm run snapshot-published-atlases\` before \`next build\`)` + ); + } } /** @@ -209,8 +300,7 @@ export async function resolveTrackerAtlas( shortNameSlug: string, version: string ): Promise { - const atlases = await getPublishedAtlases(); - const match = atlases.get(publishedAtlasKey(shortNameSlug, version)); + const match = await findPublishedAtlas(shortNameSlug, version); if (!match) { throw new Error( `No published atlas found for slug="${shortNameSlug}" version="${version}"` @@ -237,3 +327,23 @@ export async function resolveTrackerAtlasId( const { id } = await resolveTrackerAtlas(shortNameSlug, version); return id; } + +/** + * Returns the first stack frame of an error that is in app code, skipping Node + * internals, dependencies and native frames (e.g. undici or `JSON.parse`). + * @param err - Error to inspect. + * @returns the frame (e.g. "at async fetchTrackerApi (...)"), or undefined. + */ +function topAppStackFrame(err: unknown): string | undefined { + if (!(err instanceof Error) || !err.stack) return undefined; + return err.stack + .split("\n") + .map((line) => line.trim()) + .find( + (line) => + line.startsWith("at ") && + !line.includes("node:") && + !line.includes("node_modules") && + !line.includes("") + ); +} diff --git a/apis/tracker/constants.ts b/apis/tracker/constants.ts new file mode 100644 index 00000000..e3a19c64 --- /dev/null +++ b/apis/tracker/constants.ts @@ -0,0 +1,7 @@ +// Snapshot of the tracker's published atlases, written once by +// `scripts/snapshot-published-atlases.ts` before `next build` and read by every +// build worker, so all pages in a build agree on which atlases are published. +// `postbuild` deletes it, so a later bare `next build` fails instead of reading +// a stale one. +export const PUBLISHED_ATLASES_SNAPSHOT_PATH = + "./.tracker/published-atlases.json"; diff --git a/hooks/useNetworkList.ts b/hooks/useNetworkList.ts index 3b06d7eb..611fde10 100644 --- a/hooks/useNetworkList.ts +++ b/hooks/useNetworkList.ts @@ -1,8 +1,10 @@ -import { NETWORKS } from "@/constants/networks"; import type { Network } from "@/types/network"; import { createContext, useContext } from "react"; -const NetworkListContext = createContext(NETWORKS); +// Empty by default: pages must provide the publication-gated list from +// `getAvailableNetworks`, so a page that forgets to cannot list unpublished +// tracker atlases linking to pages that were never generated (see #3203). +const NetworkListContext = createContext([]); export const NetworkListProvider = NetworkListContext.Provider; diff --git a/package-lock.json b/package-lock.json index b52aa7c6..b477237c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,6 +50,7 @@ "@eslint-community/eslint-plugin-eslint-comments": "^4.7.1", "@eslint/eslintrc": "^3", "@eslint/js": "^9", + "@next/env": "^16.4.0", "@next/eslint-plugin-next": "^16.4.0", "@types/mdx": "^2.0.11", "@types/node": "22.13.0", diff --git a/package.json b/package.json index 4581450d..b653db23 100644 --- a/package.json +++ b/package.json @@ -4,17 +4,19 @@ "private": true, "scripts": { "dev": "./scripts/dev.sh data-portal && ./scripts/set-version.sh dev && next dev --webpack", - "build-dev:data-portal": "./scripts/common-build.sh data-portal dev && ./scripts/set-version.sh && next build --webpack && npm run postbuild", - "build-prod:data-portal": "./scripts/common-build.sh data-portal prod && ./scripts/set-version.sh && next build --webpack && npm run postbuild", + "build-dev:data-portal": "./scripts/common-build.sh data-portal dev && ./scripts/set-version.sh && npm run build", + "build-prod:data-portal": "./scripts/common-build.sh data-portal prod && ./scripts/set-version.sh && npm run build", + "prebuild": "npm run snapshot-published-atlases", "build": "next build --webpack", "start": "npx serve out", "lint": "eslint .", "check-format": "prettier --check .", "format": "prettier --write .", - "postbuild": "next-sitemap --config next-sitemap.config.mjs", + "postbuild": "next-sitemap --config next-sitemap.config.mjs && rm -rf .tracker", "prepare": "husky", "complete-biccn-publications": "esrun ./scripts/complete-biccn-publications.ts", - "get-cellxgene-projects": "esrun ./scripts/get-cellxgene-projects.ts" + "get-cellxgene-projects": "esrun ./scripts/get-cellxgene-projects.ts", + "snapshot-published-atlases": "esrun ./scripts/snapshot-published-atlases.ts" }, "dependencies": { "@databiosphere/findable-ui": "^57.0.0", @@ -59,6 +61,7 @@ "@eslint-community/eslint-plugin-eslint-comments": "^4.7.1", "@eslint/eslintrc": "^3", "@eslint/js": "^9", + "@next/env": "^16.4.0", "@next/eslint-plugin-next": "^16.4.0", "@types/mdx": "^2.0.11", "@types/node": "22.13.0", diff --git a/scripts/snapshot-published-atlases.ts b/scripts/snapshot-published-atlases.ts new file mode 100644 index 00000000..a80376e2 --- /dev/null +++ b/scripts/snapshot-published-atlases.ts @@ -0,0 +1,33 @@ +import { fetchPublishedAtlases } from "@/apis/tracker/api"; +import { PUBLISHED_ATLASES_SNAPSHOT_PATH } from "@/apis/tracker/constants"; +import nextEnv from "@next/env"; +import fsp from "fs/promises"; +import path from "path"; + +snapshotPublishedAtlases().catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); +}); + +/** + * Fetches the tracker's published atlases once and writes them to the + * snapshot every `next build` worker reads, so all pages in a build agree on + * which atlases are published (see #3203). Any previous snapshot is removed + * first, so a failed fetch cannot leave a stale one behind. + */ +async function snapshotPublishedAtlases(): Promise { + // Load the same env files, in the same order, as `next build`. + nextEnv.loadEnvConfig(process.cwd()); + await fsp.rm(PUBLISHED_ATLASES_SNAPSHOT_PATH, { force: true }); + const atlases = await fetchPublishedAtlases(); + await fsp.mkdir(path.dirname(PUBLISHED_ATLASES_SNAPSHOT_PATH), { + recursive: true, + }); + await fsp.writeFile( + PUBLISHED_ATLASES_SNAPSHOT_PATH, + JSON.stringify(atlases, undefined, 2) + ); + console.log( + `Wrote ${atlases.length} published atlases to ${PUBLISHED_ATLASES_SNAPSHOT_PATH}` + ); +} diff --git a/utils/availableNetworks.ts b/utils/availableNetworks.ts index d1133844..2c28a1b4 100644 --- a/utils/availableNetworks.ts +++ b/utils/availableNetworks.ts @@ -3,8 +3,10 @@ import { NETWORKS } from "@/constants/networks"; import type { Atlas, Network } from "@/types/network"; // The publication gate: every build-time decision about whether an atlas is -// shown or gets pages goes through this module, so callers cannot disagree on -// which atlases are published (see #3203). +// shown or gets pages should go through this module, so callers cannot +// disagree on which atlases are published (see #3203). Pages that list atlases +// pass these results to `NetworkListProvider`, whose default is empty so a page +// that skips the gate lists nothing rather than unpublished atlases. /** * Drops tracker atlases that are not currently published in the tracker; diff --git a/utils/networkPublicationPages.ts b/utils/networkPublicationPages.ts index 8360711d..430208ee 100644 --- a/utils/networkPublicationPages.ts +++ b/utils/networkPublicationPages.ts @@ -1,8 +1,11 @@ import { NETWORKS } from "@/constants/networks"; -import { Network, NetworkParam } from "@/types/network"; -import { GetStaticPaths, GetStaticProps, GetStaticPropsContext } from "next"; -import { GetStaticPathsResult } from "next/types"; -import { getAvailableNetwork } from "./availableNetworks"; +import type { Network, NetworkParam } from "@/types/network"; +import type { + GetStaticPaths, + GetStaticProps, + GetStaticPropsContext, +} from "next"; +import type { GetStaticPathsResult } from "next/types"; export const getStaticPaths: GetStaticPaths = async () => { return { @@ -15,12 +18,13 @@ export const getStaticProps: GetStaticProps = async ( context: GetStaticPropsContext ) => { const { network: networkParam } = context.params ?? {}; - const network = await getAvailableNetwork( - NETWORKS.find(({ path }) => path === networkParam) as Network - ); + const network = NETWORKS.find(({ path }) => path === networkParam) as Network; return { props: { - network, + // The page renders only network-level fields, never atlases, so the + // atlas list is dropped rather than publication-gated: that keeps the + // page off the tracker and out of its serialized props. + network: { ...network, atlases: [] }, pageTitle: `${network.name} - BICCN Publications`, }, }; From 05b28d9a77686d37f0649b4075a114c5ca8c1228 Mon Sep 17 00:00:00 2001 From: Fran McDade <18710366+frano-m@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:03:40 +1000 Subject: [PATCH 5/5] fix: snapshot published atlases per build and simplify tracker error logging (#3203) write the published-atlases snapshot from scripts/build.sh to a per-build temp path passed via published_atlases_snapshot, delete it however the build ends, and read it only during next build. load @next/env through next, print errors with util.inspect, default network contexts to no atlases, and inline the tracker-only static paths predicate. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitignore | 3 - apis/tracker/api.ts | 166 ++++++++++---------------- apis/tracker/constants.ts | 13 +- contexts/atlasContext.ts | 10 +- contexts/networkContext.ts | 10 +- package-lock.json | 1 - package.json | 9 +- scripts/build.sh | 13 ++ scripts/snapshot-published-atlases.ts | 41 ++++--- utils/atlasPages.ts | 10 +- utils/availableNetworks.ts | 10 -- 11 files changed, 125 insertions(+), 161 deletions(-) create mode 100755 scripts/build.sh diff --git a/.gitignore b/.gitignore index 1fc0de03..e471f810 100644 --- a/.gitignore +++ b/.gitignore @@ -15,9 +15,6 @@ # production /build -# tracker snapshot written before next build -/.tracker/ - # misc .DS_Store *.pem diff --git a/apis/tracker/api.ts b/apis/tracker/api.ts index 98e56dba..146f87ad 100644 --- a/apis/tracker/api.ts +++ b/apis/tracker/api.ts @@ -4,26 +4,23 @@ import type { TrackerSourceStudy, } from "@/types/network"; import { readFile } from "fs/promises"; -import { PUBLISHED_ATLASES_SNAPSHOT_PATH } from "./constants"; +import { PHASE_PRODUCTION_BUILD } from "next/constants"; +import { inspect } from "util"; +import { PUBLISHED_ATLASES_SNAPSHOT_ENV } from "./constants"; import type { PublishedAtlas } from "./types"; // Module-level cache for published atlases during build. Stores the in-flight // Promise so concurrent callers share a single load. let publishedAtlasesPromise: Promise | null = null; -// Node socket-error fields worth logging, e.g. the host and port a fetch could -// not reach. -const ERROR_DETAIL_KEYS = [ - "code", - "syscall", - "hostname", - "address", - "port", -] as const; +// Advice appended when `next build` runs without a snapshot path. +const BUILD_HINT = + "(run `npm run build`, which snapshots the published atlases before `next build`)"; -// How deep `describeError` follows `cause` and AggregateError `errors`, so a -// cyclic cause cannot recurse forever. -const MAX_ERROR_DEPTH = 5; +// Advice appended when the build's snapshot file is gone, e.g. removed by a tmp +// cleaner mid-build. +const SNAPSHOT_MISSING_HINT = + "(the snapshot `scripts/build.sh` wrote for this build was removed; run the build again)"; // Fields an item must carry as strings for the publication gate to match on // it. Items missing one are skipped, and a body where every item is missing one @@ -34,48 +31,6 @@ const PUBLISHED_ATLAS_MATCH_KEYS = [ "version", ] as const satisfies readonly (keyof PublishedAtlas)[]; -/** - * Describes an error for a log message: its name and message, any Node - * socket-error details (code, host, port), the errors inside an AggregateError - * (e.g. undici's "fetch failed" to a dual-stack host, one per address tried), - * and its `cause` chain. - * @param err - Error to describe. - * @param depth - Current nesting depth; callers omit it. - * @returns the error, its details and its causes as text. - */ -function describeError(err: unknown, depth = 0): string { - if (!(err instanceof Error)) return describeValue(err); - const fields = err as unknown as Record; - const details = ERROR_DETAIL_KEYS.filter( - (key) => fields[key] !== undefined - ).map((key) => `${key}=${fields[key]}`); - let text = details.length ? `${err} [${details.join(" ")}]` : String(err); - if (depth >= MAX_ERROR_DEPTH) return text; - if (err instanceof AggregateError && err.errors.length > 0) { - const errors = err.errors.map((e) => describeError(e, depth + 1)); - text += ` (errors: ${errors.join("; ")})`; - } - if (err.cause !== undefined) { - text += ` (cause: ${describeError(err.cause, depth + 1)})`; - } - return text; -} - -/** - * Describes a non-Error thrown value or cause, printing plain objects as JSON - * rather than "[object Object]". - * @param value - Value to describe. - * @returns the value as text. - */ -function describeValue(value: unknown): string { - if (typeof value !== "object" || value === null) return String(value); - try { - return JSON.stringify(value) ?? String(value); - } catch { - return String(value); - } -} - /** * Fetches and validates the tracker's published atlases. Used directly by * `next dev` and by the snapshot script that runs before `next build`. @@ -172,24 +127,15 @@ async function findPublishedAtlas( } /** - * Returns the cached published atlases, loading them on first call. `next - * build` reads the snapshot written before the build: Next runs getStaticPaths - * and getStaticProps across several worker processes, each with its own - * module-level cache, so fetching per worker could let pages disagree on which - * atlases are published if the tracker changed mid-build, shipping tabs that - * 404 (see #3203). `next dev` fetches live. Failures are rethrown so the build - * fails deterministically rather than silently omitting tracker atlases; the - * rejected promise is cleared so a long-lived `next dev` server can retry on - * the next request. + * Returns the cached published atlases, loading them on first call. Failures + * are rethrown so the build fails deterministically rather than silently + * omitting tracker atlases; the rejected promise is cleared so a long-lived + * `next dev` server can retry on the next request. * @returns published atlases. */ function getPublishedAtlases(): Promise { if (!publishedAtlasesPromise) { - const load = - process.env.NODE_ENV === "production" - ? readPublishedAtlasesSnapshot - : fetchPublishedAtlases; - publishedAtlasesPromise = load().catch((err) => { + publishedAtlasesPromise = loadPublishedAtlases().catch((err) => { publishedAtlasesPromise = null; throw err; }); @@ -222,6 +168,33 @@ export async function isTrackerAtlasPublished( return (await findPublishedAtlas(shortNameSlug, version)) !== undefined; } +/** + * Loads the published atlases. `next build` reads the snapshot that + * `scripts/build.sh` writes before the build: Next runs getStaticPaths and + * getStaticProps across several worker processes, each with its own + * module-level cache, so fetching per worker could let pages disagree on which + * atlases are published if the tracker changed mid-build, shipping tabs that + * 404 (see #3203). A `next build` without the snapshot (e.g. a bare `npx next + * build`) therefore fails. Anything else, such as `next dev`, fetches live, + * even if the snapshot variable leaked into its environment. + * @returns published atlases. + */ +function loadPublishedAtlases(): Promise { + if (process.env.NEXT_PHASE !== PHASE_PRODUCTION_BUILD) { + return fetchPublishedAtlases(); + } + const snapshotPath = process.env[PUBLISHED_ATLASES_SNAPSHOT_ENV]; + if (!snapshotPath) { + return Promise.reject( + publishedAtlasesError( + `${PUBLISHED_ATLASES_SNAPSHOT_ENV} is not set`, + BUILD_HINT + ) + ); + } + return readPublishedAtlasesSnapshot(snapshotPath); +} + /** * Validates the published-atlases response. The body must be an array. Items * missing a string `shortNameSlug` or `version` are skipped with a warning, so @@ -258,34 +231,39 @@ function parsePublishedAtlases(data: unknown): PublishedAtlas[] { * Builds the error thrown when the published atlases cannot be loaded. The * reason goes in the message, not `cause`: Next's build workers send only an * error's name, message and stack back to the main process, so a `cause` would - * never reach the build log. This error's own stack points at the handler, so - * the failing step's first stack frame in app code is appended too. - * @param err - The underlying failure. + * never reach the build log. A string reason is used as is; anything else is + * printed with `util.inspect`, which includes the underlying error's stack, + * Node error details (code, address, port), `cause` chain and AggregateError + * `errors`. + * @param reason - The underlying failure, or a description of it. * @param hint - Optional advice on how to fix it. * @returns error describing the failure. */ -function publishedAtlasesError(err: unknown, hint?: string): Error { - const frame = topAppStackFrame(err); - const parts = [ - `[tracker] Published atlases are unavailable: ${describeError(err)}`, - frame, - hint, - ]; - return new Error(parts.filter(Boolean).join(" ")); +function publishedAtlasesError(reason: unknown, hint?: string): Error { + const text = + typeof reason === "string" + ? reason + : inspect(reason, { breakLength: Infinity, depth: 5 }); + const parts = [`[tracker] Published atlases are unavailable: ${text}`, hint]; + return new Error(parts.filter(Boolean).join("\n")); } /** - * Reads the published-atlases snapshot written before `next build`. + * Reads this build's published-atlases snapshot. + * @param snapshotPath - Path of the snapshot written by `scripts/build.sh`. * @returns published atlases. */ -async function readPublishedAtlasesSnapshot(): Promise { +async function readPublishedAtlasesSnapshot( + snapshotPath: string +): Promise { try { - const text = await readFile(PUBLISHED_ATLASES_SNAPSHOT_PATH, "utf8"); + const text = await readFile(snapshotPath, "utf8"); return parsePublishedAtlases(JSON.parse(text)); } catch (err) { + const missing = (err as NodeJS.ErrnoException).code === "ENOENT"; throw publishedAtlasesError( err, - `(write the snapshot with \`npm run snapshot-published-atlases\` before \`next build\`)` + missing ? SNAPSHOT_MISSING_HINT : undefined ); } } @@ -327,23 +305,3 @@ export async function resolveTrackerAtlasId( const { id } = await resolveTrackerAtlas(shortNameSlug, version); return id; } - -/** - * Returns the first stack frame of an error that is in app code, skipping Node - * internals, dependencies and native frames (e.g. undici or `JSON.parse`). - * @param err - Error to inspect. - * @returns the frame (e.g. "at async fetchTrackerApi (...)"), or undefined. - */ -function topAppStackFrame(err: unknown): string | undefined { - if (!(err instanceof Error) || !err.stack) return undefined; - return err.stack - .split("\n") - .map((line) => line.trim()) - .find( - (line) => - line.startsWith("at ") && - !line.includes("node:") && - !line.includes("node_modules") && - !line.includes("") - ); -} diff --git a/apis/tracker/constants.ts b/apis/tracker/constants.ts index e3a19c64..c7f7567c 100644 --- a/apis/tracker/constants.ts +++ b/apis/tracker/constants.ts @@ -1,7 +1,6 @@ -// Snapshot of the tracker's published atlases, written once by -// `scripts/snapshot-published-atlases.ts` before `next build` and read by every -// build worker, so all pages in a build agree on which atlases are published. -// `postbuild` deletes it, so a later bare `next build` fails instead of reading -// a stale one. -export const PUBLISHED_ATLASES_SNAPSHOT_PATH = - "./.tracker/published-atlases.json"; +// Environment variable holding the path of this build's published-atlases +// snapshot. `scripts/build.sh` writes the snapshot once before `next build`, to +// a path unique to the build, and deletes it however the build ends; every +// build worker reads it, so all pages in a build agree on which atlases are +// published (see #3203). +export const PUBLISHED_ATLASES_SNAPSHOT_ENV = "PUBLISHED_ATLASES_SNAPSHOT"; diff --git a/contexts/atlasContext.ts b/contexts/atlasContext.ts index 4a2affc7..6acac060 100644 --- a/contexts/atlasContext.ts +++ b/contexts/atlasContext.ts @@ -1,10 +1,12 @@ import { NETWORKS, NETWORK_ATLAS_CONTENT } from "@/constants/networks"; -import { AtlasContext as AtlasContextType, AtlasModule } from "@/types/network"; +import type { + AtlasContext as AtlasContextType, + AtlasModule, +} from "@/types/network"; import { createContext, useContext } from "react"; +import { DEFAULT_NETWORK } from "./networkContext"; -const DEFAULT_NETWORK = NETWORKS[0]; - -const DEFAULT_ATLAS = DEFAULT_NETWORK.atlases[0]; +const DEFAULT_ATLAS = NETWORKS[0].atlases[0]; export const AtlasContext = createContext({ atlas: DEFAULT_ATLAS, diff --git a/contexts/networkContext.ts b/contexts/networkContext.ts index 2caa8a4a..fb0601b7 100644 --- a/contexts/networkContext.ts +++ b/contexts/networkContext.ts @@ -1,12 +1,18 @@ import { NETWORKS, NETWORK_CONTENT } from "@/constants/networks"; -import { +import type { + Network, NetworkContext as NetworkContextType, NetworkModule, } from "@/types/network"; import { createContext, useContext } from "react"; +// Default network for contexts rendered without a provider. It lists no +// atlases, so such a component cannot list unpublished tracker atlases whose +// pages were never generated (see #3203). +export const DEFAULT_NETWORK: Network = { ...NETWORKS[0], atlases: [] }; + export const NetworkContext = createContext({ - network: NETWORKS[0], + network: DEFAULT_NETWORK, }); export const NetworkProvider = NetworkContext.Provider; diff --git a/package-lock.json b/package-lock.json index b477237c..b52aa7c6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,6 @@ "@eslint-community/eslint-plugin-eslint-comments": "^4.7.1", "@eslint/eslintrc": "^3", "@eslint/js": "^9", - "@next/env": "^16.4.0", "@next/eslint-plugin-next": "^16.4.0", "@types/mdx": "^2.0.11", "@types/node": "22.13.0", diff --git a/package.json b/package.json index b653db23..3875e32d 100644 --- a/package.json +++ b/package.json @@ -6,17 +6,15 @@ "dev": "./scripts/dev.sh data-portal && ./scripts/set-version.sh dev && next dev --webpack", "build-dev:data-portal": "./scripts/common-build.sh data-portal dev && ./scripts/set-version.sh && npm run build", "build-prod:data-portal": "./scripts/common-build.sh data-portal prod && ./scripts/set-version.sh && npm run build", - "prebuild": "npm run snapshot-published-atlases", - "build": "next build --webpack", + "build": "./scripts/build.sh", "start": "npx serve out", "lint": "eslint .", "check-format": "prettier --check .", "format": "prettier --write .", - "postbuild": "next-sitemap --config next-sitemap.config.mjs && rm -rf .tracker", + "postbuild": "next-sitemap --config next-sitemap.config.mjs", "prepare": "husky", "complete-biccn-publications": "esrun ./scripts/complete-biccn-publications.ts", - "get-cellxgene-projects": "esrun ./scripts/get-cellxgene-projects.ts", - "snapshot-published-atlases": "esrun ./scripts/snapshot-published-atlases.ts" + "get-cellxgene-projects": "esrun ./scripts/get-cellxgene-projects.ts" }, "dependencies": { "@databiosphere/findable-ui": "^57.0.0", @@ -61,7 +59,6 @@ "@eslint-community/eslint-plugin-eslint-comments": "^4.7.1", "@eslint/eslintrc": "^3", "@eslint/js": "^9", - "@next/env": "^16.4.0", "@next/eslint-plugin-next": "^16.4.0", "@types/mdx": "^2.0.11", "@types/node": "22.13.0", diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 00000000..e178d40e --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +# Snapshot the tracker's published atlases once, to a path unique to this +# build, so every `next build` worker agrees on which atlases are published +# (see #3203). The trap deletes the snapshot however the build ends. +TMP_DIR="${TMPDIR:-/tmp}" +PUBLISHED_ATLASES_SNAPSHOT=$(mktemp "${TMP_DIR%/}/published-atlases.XXXXXX") +export PUBLISHED_ATLASES_SNAPSHOT +trap 'rm -f "$PUBLISHED_ATLASES_SNAPSHOT"' EXIT + +npx --no-install esrun ./scripts/snapshot-published-atlases.ts +npx --no-install next build --webpack "$@" diff --git a/scripts/snapshot-published-atlases.ts b/scripts/snapshot-published-atlases.ts index a80376e2..f31d6e6e 100644 --- a/scripts/snapshot-published-atlases.ts +++ b/scripts/snapshot-published-atlases.ts @@ -1,33 +1,38 @@ import { fetchPublishedAtlases } from "@/apis/tracker/api"; -import { PUBLISHED_ATLASES_SNAPSHOT_PATH } from "@/apis/tracker/constants"; -import nextEnv from "@next/env"; +import { PUBLISHED_ATLASES_SNAPSHOT_ENV } from "@/apis/tracker/constants"; import fsp from "fs/promises"; +import { createRequire } from "module"; import path from "path"; snapshotPublishedAtlases().catch((err) => { - console.error(err instanceof Error ? err.message : err); + console.error(err); process.exit(1); }); /** * Fetches the tracker's published atlases once and writes them to the * snapshot every `next build` worker reads, so all pages in a build agree on - * which atlases are published (see #3203). Any previous snapshot is removed - * first, so a failed fetch cannot leave a stale one behind. + * which atlases are published (see #3203). Run by `scripts/build.sh`, which + * sets the snapshot path. */ async function snapshotPublishedAtlases(): Promise { - // Load the same env files, in the same order, as `next build`. - nextEnv.loadEnvConfig(process.cwd()); - await fsp.rm(PUBLISHED_ATLASES_SNAPSHOT_PATH, { force: true }); - const atlases = await fetchPublishedAtlases(); - await fsp.mkdir(path.dirname(PUBLISHED_ATLASES_SNAPSHOT_PATH), { - recursive: true, - }); - await fsp.writeFile( - PUBLISHED_ATLASES_SNAPSHOT_PATH, - JSON.stringify(atlases, undefined, 2) - ); - console.log( - `Wrote ${atlases.length} published atlases to ${PUBLISHED_ATLASES_SNAPSHOT_PATH}` + const snapshotPath = process.env[PUBLISHED_ATLASES_SNAPSHOT_ENV]; + if (!snapshotPath) { + throw new Error( + `${PUBLISHED_ATLASES_SNAPSHOT_ENV} is not set; run \`npm run build\`, which sets it` + ); + } + // Load `@next/env` through `next`, so env files are read by the exact copy + // `next build` uses, in the same order, rather than a separately pinned one. + const requireFromRoot = createRequire(path.resolve("package.json")); + const requireFromNext = createRequire( + requireFromRoot.resolve("next/package.json") ); + const { loadEnvConfig } = requireFromNext("@next/env") as { + loadEnvConfig: (dir: string) => unknown; + }; + loadEnvConfig(process.cwd()); + const atlases = await fetchPublishedAtlases(); + await fsp.writeFile(snapshotPath, JSON.stringify(atlases, undefined, 2)); + console.log(`Wrote ${atlases.length} published atlases to ${snapshotPath}`); } diff --git a/utils/atlasPages.ts b/utils/atlasPages.ts index f2103ca2..6bf4f60d 100644 --- a/utils/atlasPages.ts +++ b/utils/atlasPages.ts @@ -13,11 +13,7 @@ import type { GetStaticPropsResult, } from "next"; import type { ParsedUrlQuery } from "querystring"; -import { - getAvailableNetwork, - isAtlasPublished, - isPublishedTrackerAtlas, -} from "./availableNetworks"; +import { getAvailableNetwork, isAtlasPublished } from "./availableNetworks"; import { fetchCXGDatasetsForAtlases, processAtlas, @@ -55,7 +51,9 @@ export const getNonTrackerStaticPaths: GetStaticPaths = () => * @returns static paths for published tracker atlases. */ export const getTrackerStaticPaths: GetStaticPaths = () => - buildStaticPaths(isPublishedTrackerAtlas); + buildStaticPaths( + async (atlas) => Boolean(atlas.tracker) && isAtlasPublished(atlas) + ); /** * Builds static props for an atlas page, delegating to the tracker builder when diff --git a/utils/availableNetworks.ts b/utils/availableNetworks.ts index 2c28a1b4..35a1c420 100644 --- a/utils/availableNetworks.ts +++ b/utils/availableNetworks.ts @@ -55,13 +55,3 @@ export async function isAtlasPublished(atlas: Atlas): Promise { const { shortNameSlug, version } = atlas.tracker; return isTrackerAtlasPublished(shortNameSlug, version); } - -/** - * Returns true when the atlas is tracker-sourced and published in the tracker. - * @param atlas - Atlas to check. - * @returns true if the atlas is a published tracker atlas. - */ -export async function isPublishedTrackerAtlas(atlas: Atlas): Promise { - if (!atlas.tracker) return false; - return isAtlasPublished(atlas); -}