v3: load metadata on demand from the Public API's v2 routes, unused until the switch

.github/workflows/pr.yaml

@@ -112,6 +112,26 @@ jobs:
         working-directory: ./website
         run: bun run test
 
+  public-api-client:
+    name: Public API client is current
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Bun
+        uses: oven-sh/setup-bun@v2
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      # Regenerates the types from the checked-in OpenAPI document and fails
+      # when src/api/publicApi/schema.d.ts differs from the result.
+      - name: Check the generated client against its OpenAPI document
+        working-directory: ./app
+        run: bun run public-api:check
+
   check-embeds:
     name: Check embedded site URLs
     runs-on: ubuntu-latest

app/package.json

@@ -7,6 +7,9 @@
     "evaluate-search": "tsx scripts/evaluate-parameter-search.ts",
     "refresh-search-eval-cases": "tsx scripts/refresh-parameter-search-cases.ts",
     "build-parameter-dependencies": "python3 scripts/build-parameter-dependency-map.py",
+    "public-api:generate": "tsx scripts/public-api-client.ts generate",
+    "public-api:check": "tsx scripts/public-api-client.ts check",
+    "public-api:refresh": "tsx scripts/public-api-client.ts refresh",
     "typecheck": "tsc --noEmit",
     "lint": "bun run eslint && bun run stylelint",
     "eslint": "eslint . --cache --max-warnings 0",
@@ -39,6 +42,7 @@
     "fuse.js": "^7.0.0",
     "html-to-image": "^1.11.13",
     "lucide-react": "^0.575.0",
+    "openapi-fetch": "0.17.0",
     "policyengine-household-wizard": "^0.1.0",
     "radix-ui": "^1.4.3",
     "react": "^19.1.0",
@@ -78,6 +82,7 @@
     "fast-check": "^4.10.2",
     "identity-obj-proxy": "^3.0.0",
     "jsdom": "^26.1.0",
+    "openapi-typescript": "7.13.0",
     "postcss": "^8.5.5",
     "postcss-simple-vars": "^7.0.1",
     "prettier": "^3.5.3",

app/scripts/public-api-client.ts

@@ -0,0 +1,144 @@
+/**
+ * TypeScript types for the Public API's v2 routes (policyengine-api),
+ * generated from its OpenAPI document (`GET /v2/openapi.json`).
+ *
+ * The document is checked in beside the generated types, so a build never
+ * calls the API, and CI can check that the types match the document:
+ *
+ *   bun run public-api:generate            writes schema.d.ts from openapi.json
+ *   bun run public-api:check               fails when schema.d.ts is out of date
+ *   bun run public-api:refresh <source>    copies the document from a URL or a
+ *                                          file, then writes schema.d.ts
+ *
+ * <source> is an API origin (https://api.policyengine.org), a full URL of
+ * the document, or a path to a saved copy of it.
+ */
+import { readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import openapiTS, { astToString, type OpenAPI3 } from 'openapi-typescript';
+import prettier from 'prettier';
+
+const SCRIPT_DIRECTORY = path.dirname(fileURLToPath(import.meta.url));
+const CLIENT_DIRECTORY = path.resolve(SCRIPT_DIRECTORY, '../src/api/publicApi');
+
+export const OPENAPI_DOCUMENT_PATH = path.join(CLIENT_DIRECTORY, 'openapi.json');
+export const GENERATED_SCHEMA_PATH = path.join(CLIENT_DIRECTORY, 'schema.d.ts');
+
+const GENERATED_HEADER = `/**
+ * Generated from src/api/publicApi/openapi.json by scripts/public-api-client.ts.
+ * Do not edit: run \`bun run public-api:generate\` (or \`public-api:refresh\`).
+ */
+`;
+
+/** The path the API serves its v2 OpenAPI document at. */
+const OPENAPI_DOCUMENT_ROUTE = '/v2/openapi.json';
+
+/** The generated, formatted types for an OpenAPI document. */
+export async function generatePublicApiTypes(openapiDocument: OpenAPI3): Promise<string> {
+  const syntaxTree = await openapiTS(openapiDocument);
+  const source = `${GENERATED_HEADER}\n${astToString(syntaxTree)}`;
+  const prettierConfig = (await prettier.resolveConfig(GENERATED_SCHEMA_PATH)) ?? {};
+  return prettier.format(source, { ...prettierConfig, filepath: GENERATED_SCHEMA_PATH });
+}
+
+export interface SchemaComparison {
+  upToDate: boolean;
+  expected: string;
+}
+
+/** Whether checked-in types equal what the document generates. */
+export async function compareGeneratedTypes(
+  openapiDocument: OpenAPI3,
+  checkedInTypes: string
+): Promise<SchemaComparison> {
+  const expected = await generatePublicApiTypes(openapiDocument);
+  return { upToDate: expected === checkedInTypes, expected };
+}
+
+/** Where to read the document from: an origin gets the v2 route appended. */
+export function openapiDocumentLocation(source: string): string {
+  if (!/^https?:\/\//.test(source)) {
+    return source;
+  }
+  const url = new URL(source);
+  if (url.pathname === '/' || url.pathname === '') {
+    url.pathname = OPENAPI_DOCUMENT_ROUTE;
+  }
+  return url.toString();
+}
+
+/** The document as checked in: two-space JSON with a final newline. */
+export function serializeOpenapiDocument(openapiDocument: OpenAPI3): string {
+  return `${JSON.stringify(openapiDocument, null, 2)}\n`;
+}
+
+async function readOpenapiDocument(location: string): Promise<OpenAPI3> {
+  if (/^https?:\/\//.test(location)) {
+    const response = await fetch(location, { headers: { Accept: 'application/json' } });
+    if (!response.ok) {
+      throw new Error(`GET ${location} returned ${response.status}`);
+    }
+    return (await response.json()) as OpenAPI3;
+  }
+  return JSON.parse(await readFile(location, 'utf8')) as OpenAPI3;
+}
+
+async function writeGeneratedTypes(): Promise<void> {
+  const openapiDocument = await readOpenapiDocument(OPENAPI_DOCUMENT_PATH);
+  await writeFile(GENERATED_SCHEMA_PATH, await generatePublicApiTypes(openapiDocument));
+  console.info(`Wrote ${path.relative(process.cwd(), GENERATED_SCHEMA_PATH)}`);
+}
+
+async function checkGeneratedTypes(): Promise<boolean> {
+  const openapiDocument = await readOpenapiDocument(OPENAPI_DOCUMENT_PATH);
+  const checkedInTypes = await readFile(GENERATED_SCHEMA_PATH, 'utf8');
+  const { upToDate } = await compareGeneratedTypes(openapiDocument, checkedInTypes);
+  if (upToDate) {
+    console.info('The generated Public API types match openapi.json.');
+    return true;
+  }
+  console.error(
+    'src/api/publicApi/schema.d.ts does not match src/api/publicApi/openapi.json.\n' +
+      'Run `bun run public-api:generate` in app/ and commit the result.'
+  );
+  return false;
+}
+
+async function refreshOpenapiDocument(source: string | undefined): Promise<void> {
+  if (!source) {
+    throw new Error('Name the document to copy: an API origin, a URL or a file path.');
+  }
+  const location = openapiDocumentLocation(source);
+  const openapiDocument = await readOpenapiDocument(location);
+  await writeFile(OPENAPI_DOCUMENT_PATH, serializeOpenapiDocument(openapiDocument));
+  console.info(`Copied ${location} to ${path.relative(process.cwd(), OPENAPI_DOCUMENT_PATH)}`);
+  await writeGeneratedTypes();
+}
+
+async function main(commandLineArguments: string[]): Promise<number> {
+  const [command, source] = commandLineArguments;
+  switch (command) {
+    case 'generate':
+      await writeGeneratedTypes();
+      return 0;
+    case 'check':
+      return (await checkGeneratedTypes()) ? 0 : 1;
+    case 'refresh':
+      await refreshOpenapiDocument(source);
+      return 0;
+    default:
+      console.error('Usage: public-api-client.ts generate | check | refresh <source>');
+      return 2;
+  }
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  main(process.argv.slice(2)).then(
+    (exitCode) => process.exit(exitCode),
+    (error: unknown) => {
+      console.error(error instanceof Error ? error.message : error);
+      process.exit(1);
+    }
+  );
+}

app/src/api/publicApi/catalogParameters.ts

@@ -0,0 +1,110 @@
+import {
+  createPublicApiClient,
+  publicApiCountryId,
+  requireResult,
+  type PublicApiClient,
+} from '@/api/publicApi/client';
+import { parameterValueFromApi } from '@/api/publicApi/jsonValues';
+import type { components } from '@/api/publicApi/schema';
+import { v1FallbackLabel } from '@/libs/flagship/v2ParameterSearch';
+import type { ParameterMetadataCollection } from '@/types/metadata/parameterMetadata';
+import type { ValuesList } from '@/types/subIngredients/valueInterval';
+
+/**
+ * Parameters fetched by name from the Public API's
+ * `GET /v2/model-catalog/parameters`, with their current-law values and the
+ * labels of every node above them (for breadcrumbs), in the shape v1's
+ * metadata gave each parameter, so code that reads the metadata store
+ * reads them unchanged.
+ */
+
+type Schemas = components['schemas'];
+
+/** How many names one request takes (the route's own bound). */
+export const CATALOG_PARAMETER_BATCH = 100;
+
+export interface LoadedCatalogParameters {
+  /** The parameters found, and every node above them. */
+  parameters: ParameterMetadataCollection;
+  /** Each found parameter's catalog ID, which `/v2/policies` takes. */
+  parameterIds: Record<string, string>;
+  /** Names the catalog version does not have. */
+  missingNames: string[];
+}
+
+/** v1 dated a value in force since the earliest date 0000-01-01; Python's earliest is year 1. */
+const V2_EARLIEST_DATE = '0001-01-01';
+const V1_EARLIEST_DATE = '0000-01-01';
+
+function v1Values(parameterName: string, values: Schemas['MetadataDatedValue'][]): ValuesList {
+  return Object.fromEntries(
+    values.map((dated) => {
+      const date = dated.start_date.slice(0, 10);
+      return [
+        date === V2_EARLIEST_DATE ? V1_EARLIEST_DATE : date,
+        parameterValueFromApi(dated.value, parameterName),
+      ];
+    })
+  );
+}
+
+function addResult(
+  loaded: LoadedCatalogParameters,
+  result: Schemas['MetadataCatalogParametersResult']
+): void {
+  for (const [path, label] of Object.entries(result.node_labels)) {
+    loaded.parameters[path] = {
+      type: 'parameterNode',
+      parameter: path,
+      label: label ?? v1FallbackLabel(path),
+    };
+  }
+  for (const parameter of result.parameters) {
+    loaded.parameters[parameter.name] = {
+      type: 'parameter',
+      parameter: parameter.name,
+      label: parameter.label ?? v1FallbackLabel(parameter.name),
+      description: parameter.description,
+      unit: parameter.unit,
+      values: v1Values(parameter.name, parameter.values),
+      economy: parameter.economy ?? true,
+      household: parameter.household ?? true,
+    };
+    loaded.parameterIds[parameter.name] = parameter.id;
+  }
+  loaded.missingNames.push(...result.missing_names);
+}
+
+/** The named parameters of a catalog version, in batches the route takes. */
+export async function fetchCatalogParameters(
+  countryId: string,
+  policyengineVersion: string,
+  names: readonly string[],
+  client: PublicApiClient = createPublicApiClient()
+): Promise<LoadedCatalogParameters> {
+  const unique = [...new Set(names)].sort();
+  const batches: string[][] = [];
+  for (let start = 0; start < unique.length; start += CATALOG_PARAMETER_BATCH) {
+    batches.push(unique.slice(start, start + CATALOG_PARAMETER_BATCH));
+  }
+  const results = await Promise.all(
+    batches.map((batch) =>
+      requireResult(
+        client.GET('/v2/model-catalog/parameters', {
+          params: {
+            query: {
+              country_id: publicApiCountryId(countryId),
+              policyengine_version: policyengineVersion,
+              name: batch,
+            },
+          },
+        })
+      )
+    )
+  );
+  const loaded: LoadedCatalogParameters = { parameters: {}, parameterIds: {}, missingNames: [] };
+  for (const result of results) {
+    addResult(loaded, result);
+  }
+  return loaded;
+}