v3 variables on the v2 routes: the household builder and breakdown from /v2/model-catalog/variables, unused until the switch

app/src/api/publicApi/catalogParameters.ts

@@ -4,11 +4,10 @@ import {
   requireResult,
   type PublicApiClient,
 } from '@/api/publicApi/client';
-import { parameterValueFromApi } from '@/api/publicApi/jsonValues';
+import { v1DatedValues } from '@/api/publicApi/datedValues';
 import type { components } from '@/api/publicApi/schema';
 import { v1FallbackLabel } from '@/libs/flagship/v2ParameterSearch';
 import type { ParameterMetadataCollection } from '@/types/metadata/parameterMetadata';
-import type { ValuesList } from '@/types/subIngredients/valueInterval';
 
 /**
  * Parameters fetched by name from the Public API's
@@ -32,22 +31,6 @@ export interface LoadedCatalogParameters {
   missingNames: string[];
 }
 
-/** v1 dated a value in force since the earliest date 0000-01-01; Python's earliest is year 1. */
-const V2_EARLIEST_DATE = '0001-01-01';
-const V1_EARLIEST_DATE = '0000-01-01';
-
-function v1Values(parameterName: string, values: Schemas['MetadataDatedValue'][]): ValuesList {
-  return Object.fromEntries(
-    values.map((dated) => {
-      const date = dated.start_date.slice(0, 10);
-      return [
-        date === V2_EARLIEST_DATE ? V1_EARLIEST_DATE : date,
-        parameterValueFromApi(dated.value, parameterName),
-      ];
-    })
-  );
-}
-
 function addResult(
   loaded: LoadedCatalogParameters,
   result: Schemas['MetadataCatalogParametersResult']
@@ -66,7 +49,7 @@ function addResult(
       label: parameter.label ?? v1FallbackLabel(parameter.name),
       description: parameter.description,
       unit: parameter.unit,
-      values: v1Values(parameter.name, parameter.values),
+      values: v1DatedValues(parameter.name, parameter.values),
       economy: parameter.economy ?? true,
       household: parameter.household ?? true,
     };

app/src/api/publicApi/catalogVariables.ts

@@ -0,0 +1,148 @@
+import {
+  createPublicApiClient,
+  publicApiCountryId,
+  requireResult,
+  type PublicApiClient,
+} from '@/api/publicApi/client';
+import { v1DatedValues } from '@/api/publicApi/datedValues';
+import type { components } from '@/api/publicApi/schema';
+import type { ValuesList } from '@/types/subIngredients/valueInterval';
+
+/**
+ * Every variable of a catalog version, from the Public API's
+ * `GET /v2/model-catalog/variables`, in the shape v1's
+ * `GET /{country}/metadata` gave each variable, so the household builder,
+ * its variable picker and the household breakdown read them unchanged.
+ *
+ * Where a variable's `adds` or `subtracts` names a parameter, v1 gave the
+ * parameter's path as the value, and the breakdown read that parameter
+ * from the whole parameter list for the report's year. On v2 the path is
+ * kept the same way, and the parameter's dated values come with the
+ * variables (`part_parameter_values`), so the breakdown never needs the
+ * whole parameter list.
+ *
+ * The catalog does not publish a variable's category or position in its
+ * module, so those stay unset.
+ */
+
+type Schemas = components['schemas'];
+
+export interface LoadedCatalogVariables {
+  /** Each variable by name, as v1's metadata described it. */
+  variables: Record<string, V1ShapedVariable>;
+  /** The dated values of each parameter a variable's adds or subtracts names. */
+  partParameterValues: Record<string, ValuesList>;
+}
+
+/** The fields of a v1 metadata variable that the catalog publishes. */
+export interface V1ShapedVariable {
+  name: string;
+  label: string;
+  entity: string;
+  documentation: string | null;
+  valueType: string | null;
+  /** Such as 'currency-USD' or '/1'; null for a variable without one, as on v1. */
+  unit: string | null;
+  /** 'year', 'month', 'day' or 'eternity'; null for versions before catalog format 2. */
+  definitionPeriod: string | null;
+  defaultValue: unknown;
+  /** A list of variable names, or the path of a parameter whose value is that list. */
+  adds: string[] | string | null;
+  subtracts: string[] | string | null;
+  isInputVariable: boolean;
+  hidden_input: boolean;
+  moduleName: string | null;
+  possibleValues?: Array<{ value: string; label: Schemas['MetadataEnumValue']['label'] }>;
+}
+
+/**
+ * v1's type name for a variable. The catalog stores the Python type's name,
+ * and .py publishes every Enum variable as "str" with its values, so a
+ * variable with enum values is v1's "Enum".
+ */
+function v1ValueType(variable: Schemas['MetadataVariable']): string | null {
+  return variable.enum_values ? 'Enum' : variable.data_type;
+}
+
+/**
+ * v1 gave a default only for numbers, yes-or-no values and enums (the
+ * default value's name); a text or date variable had none.
+ */
+function v1DefaultValue(valueType: string | null, defaultValue: unknown): unknown {
+  if (valueType === 'Enum') {
+    return defaultValue;
+  }
+  return typeof defaultValue === 'number' || typeof defaultValue === 'boolean'
+    ? defaultValue
+    : null;
+}
+
+export function v1ShapedVariable(variable: Schemas['MetadataVariable']): V1ShapedVariable {
+  const valueType = v1ValueType(variable);
+  const shaped: V1ShapedVariable = {
+    name: variable.name,
+    // v1 always had a label; a variable without one reads as its name.
+    label: variable.label ?? variable.name,
+    entity: variable.entity,
+    documentation: variable.description,
+    valueType,
+    unit: variable.unit,
+    definitionPeriod: variable.definition_period,
+    defaultValue: v1DefaultValue(valueType, variable.default_value),
+    adds: variable.adds_parameter_path ?? variable.adds,
+    subtracts: variable.subtracts_parameter_path ?? variable.subtracts,
+    isInputVariable: variable.is_input_variable ?? false,
+    hidden_input: variable.hidden_input ?? false,
+    moduleName: variable.module_name ?? null,
+  };
+  if (variable.enum_values) {
+    shaped.possibleValues = variable.enum_values.map((enumValue) => ({
+      value: enumValue.value,
+      label: enumValue.label,
+    }));
+  }
+  return shaped;
+}
+
+/** The catalog format whose variables carry the input flags, enum labels, modules and parts v3 reads. */
+const CATALOG_FORMAT_WITH_VARIABLE_FIELDS = 2;
+
+/**
+ * All of a catalog version's variables, with the values their
+ * parameter-named parts read. A version published before catalog format 2
+ * fails, rather than giving variables without what v3 reads.
+ */
+export async function fetchCatalogVariables(
+  countryId: string,
+  policyengineVersion: string,
+  client: PublicApiClient = createPublicApiClient()
+): Promise<LoadedCatalogVariables> {
+  const result = await requireResult(
+    client.GET('/v2/model-catalog/variables', {
+      params: {
+        query: {
+          country_id: publicApiCountryId(countryId),
+          policyengine_version: policyengineVersion,
+        },
+      },
+    })
+  );
+  if (result.catalog_format !== CATALOG_FORMAT_WITH_VARIABLE_FIELDS) {
+    // Without format 2's fields there are no input flags, enum labels or
+    // parameter-named parts, so v3's builder and breakdown cannot use them.
+    throw new Error(
+      `The catalog version ${result.policyengine_version} was published before catalog format 2, so it has no variable details v3 can use.`
+    );
+  }
+  return {
+    variables: Object.fromEntries(
+      result.variables.map((variable) => [variable.name, v1ShapedVariable(variable)])
+    ),
+    partParameterValues: Object.fromEntries(
+      Object.entries(result.part_parameter_values).map(([path, values]) => [
+        path,
+        v1DatedValues(path, values),
+      ])
+    ),
+  };
+}

app/src/api/publicApi/datedValues.ts

@@ -0,0 +1,29 @@
+import { parameterValueFromApi } from '@/api/publicApi/jsonValues';
+import type { components } from '@/api/publicApi/schema';
+import type { ValuesList } from '@/types/subIngredients/valueInterval';
+
+/** v1 dated a value in force since the earliest date 0000-01-01; Python's earliest is year 1. */
+const V2_EARLIEST_DATE = '0001-01-01';
+const V1_EARLIEST_DATE = '0000-01-01';
+
+/**
+ * A parameter's dated values from the Public API's v2 catalog routes, as
+ * v1's metadata dated them: each value under the date it starts, the
+ * earliest date written as v1 wrote it. Each value is checked where it
+ * arrives (`parameterValueFromApi`), so a malformed response fails here,
+ * naming the parameter.
+ */
+export function v1DatedValues(
+  parameterName: string,
+  values: components['schemas']['MetadataDatedValue'][]
+): ValuesList {
+  return Object.fromEntries(
+    values.map((dated) => {
+      const date = dated.start_date.slice(0, 10);
+      return [
+        date === V2_EARLIEST_DATE ? V1_EARLIEST_DATE : date,
+        parameterValueFromApi(dated.value, parameterName),
+      ];
+    })
+  );
+}

app/src/api/publicApi/modelCatalog.ts

@@ -16,7 +16,9 @@ import type { MetadataApiPayload, MetadataRegionEntry } from '@/types/metadata';
  *
  * Parameters and variables are not part of it. v3 loads them on demand
  * (the parameters of what is open, the parameter tree one level at a time,
- * and a search file in the background), so they start empty here. v1's
+ * and a search file in the background) and loads every variable once in the
+ * background after a page renders (`useV3VariablesInBackground`), so they
+ * start empty here. v1's
  * variable modules, modelled policies and SPM settings are not published
  * by v2 and stay empty too.
  */

app/src/components/flagship/CalibrationMatches.tsx

@@ -1,9 +1,11 @@
 import { useEffect, useState } from 'react';
 import { useSelector } from 'react-redux';
 import { STATE_NAMES } from '@/api/billFeed';
+import { VariableNamesFailed } from '@/components/flagship/V3VariablesGate';
 import { Text } from '@/components/ui';
 import { Dialog, DialogContent, DialogTitle, DialogTrigger } from '@/components/ui/dialog';
 import { colors, spacing, typography } from '@/designTokens';
+import { useV3Variables } from '@/hooks/useV3Variables';
 import {
   CalibrationCountry,
   calibrationFitForGeography,
@@ -325,8 +327,26 @@ export function GeographyFitSection({
 }) {
   const fit = useGeographyCalibrationFit(geography, countryId);
   const variables = useSelector((state: RootState) => state.metadata.variables);
+  // On v2 the variables (target names) load in the background; wait for them.
+  const variablesStatus = useV3Variables();
   const name = geographyName(geography);
   const heading = <ValidationSubheading>{name} data fit</ValidationSubheading>;
+  if (variablesStatus.status === 'loading') {
+    return (
+      <>
+        {heading}
+        <ValidationLoading label="Loading variable names" />
+      </>
+    );
+  }
+  if (variablesStatus.status === 'error') {
+    return (
+      <>
+        {heading}
+        <VariableNamesFailed onRetry={variablesStatus.retry} />
+      </>
+    );
+  }
   if (fit === undefined) {
     return (
       <>
@@ -403,6 +423,8 @@ export function CalibrationMatchSection({
   pin?: CalibrationPin;
 }) {
   const variables = useSelector((state: RootState) => state.metadata.variables);
+  // On v2 the variables (target names) load in the background; wait for them.
+  const variablesStatus = useV3Variables();
   const countryId: CalibrationCountry = matches?.countryId ?? 'us';
   const uk = countryId === 'uk';
   // UK nations and the UK itself are the dashboard's "country" level.
@@ -433,6 +455,12 @@ export function CalibrationMatchSection({
   if (matches === undefined) {
     return card(<ValidationLoading label="Checking calibration targets" />);
   }
+  if (variablesStatus.status === 'loading') {
+    return card(<ValidationLoading label="Loading variable names" />);
+  }
+  if (variablesStatus.status === 'error') {
+    return card(<VariableNamesFailed onRetry={variablesStatus.retry} />);
+  }
   const placeName = geographyName(matches.geography);
   const related = matches.matches.length === 0 ? (matches.related ?? []) : [];
   if (matches.reachedCount === 0 && uk) {

app/src/components/flagship/ProvisionList.tsx

@@ -1,6 +1,8 @@
 import { useSelector } from 'react-redux';
+import { VariableNamesFailed, VariableNamesLoading } from '@/components/flagship/V3VariablesGate';
 import { Stack, Text } from '@/components/ui';
 import { colors, spacing, typography } from '@/designTokens';
+import { useV3Variables } from '@/hooks/useV3Variables';
 import { describeChange } from '@/libs/flagship/draftLabels';
 import { RootState } from '@/store';
 import { formatCurrentLaw, formatValue } from '@/utils/parameterValues';
@@ -31,6 +33,8 @@ export default function ProvisionList({
 }) {
   const variables = useSelector((state: RootState) => state.metadata.variables);
   const variableLabel = (name: string) => variables?.[name]?.label ?? name;
+  // On v2 the variables load in the background: a list of them waits for their labels.
+  const variablesStatus = useV3Variables();
   return (
     <div
       style={{
@@ -99,7 +103,11 @@ export default function ProvisionList({
                 textAlign: 'right',
               }}
             >
-              {listChange ? (
+              {listChange && variablesStatus.status === 'loading' ? (
+                <VariableNamesLoading />
+              ) : listChange && variablesStatus.status === 'error' ? (
+                <VariableNamesFailed onRetry={variablesStatus.retry} />
+              ) : listChange ? (
                 <span
                   style={{ color: colors.primary[700], fontWeight: typography.fontWeight.semibold }}
                 >

app/src/components/flagship/V3VariablesGate.tsx

@@ -0,0 +1,68 @@
+import { ErrorState } from '@/components/flagship/states';
+import { Skeleton } from '@/components/ui';
+import { spacing } from '@/designTokens';
+import { useV3Variables } from '@/hooks/useV3Variables';
+
+interface V3VariablesGateProps {
+  children: React.ReactNode;
+  /** What is waiting, for screen readers ("Loading the household builder"). */
+  label: string;
+  /** Placeholder while the variables load; a few skeleton rows by default. */
+  placeholder?: React.ReactNode;
+}
+
+function DefaultPlaceholder({ label }: { label: string }) {
+  return (
+    <div
+      role="status"
+      aria-busy="true"
+      aria-label={label}
+      style={{ display: 'flex', flexDirection: 'column', gap: spacing.sm }}
+    >
+      <Skeleton style={{ height: 16, width: '35%' }} />
+      <Skeleton style={{ height: 12, width: '80%' }} />
+      <Skeleton style={{ height: 12, width: '65%' }} />
+    </div>
+  );
+}
+
+/**
+ * An inline placeholder for text made of variable names (a list a reform
+ * changes), while the variables load on the v2 routes.
+ */
+export function VariableNamesLoading() {
+  return (
+    <span role="status" aria-busy="true" aria-label="Loading variable names">
+      <Skeleton style={{ display: 'inline-block', height: 12, width: 120 }} />
+    </span>
+  );
+}
+
+/** Where variable names should be, when loading them failed on the v2 routes. */
+export function VariableNamesFailed({ onRetry }: { onRetry: () => unknown }) {
+  return <ErrorState compact title="Could not load the variable names" onRetry={onRetry} />;
+}
+
+/**
+ * Content that reads the model's variables (the household builder and
+ * breakdown, programme names), shown once `useV3Variables` says they are
+ * 'ready'. On the v2 routes it waits behind a placeholder while they load in
+ * the background, with no time limit, and a failure says so and asks again;
+ * on the v1 routes they are always ready, so the content shows as before.
+ */
+export default function V3VariablesGate({ children, label, placeholder }: V3VariablesGateProps) {
+  const variables = useV3Variables();
+  if (variables.status === 'error') {
+    return (
+      <ErrorState
+        title="Could not load the model's variables"
+        description={variables.message ?? undefined}
+        onRetry={variables.retry}
+      />
+    );
+  }
+  if (variables.status === 'loading') {
+    return <>{placeholder ?? <DefaultPlaceholder label={label} />}</>;
+  }
+  return <>{children}</>;
+}