From c3df8541197c3da3bdf47a8f51e14b495d2f2edf Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Fri, 26 Jun 2026 15:29:21 -0400 Subject: [PATCH 1/5] chore(deps): update @codaco packages and migrate collected categorical data Bump all @codaco packages to latest, including @codaco/protocol-utilities 1.x -> 2.0.0. Adapt the generateNetwork call sites in the test-interview generator to the new options-object signature (seed moved into options). @codaco/interview 1.1.0 stores categorical attribute values as arrays of selected option values, dropping the scalar-handling fallbacks. Add a one-time, idempotent migration that wraps scalar categorical values in collected interview networks into single-element arrays, resolving categorical variables from the protocol codebook. Wired into the build:platform migration step after the v8 protocol migration. Closes #795 --- app/api/generate-test-interviews/route.ts | 3 +- package.json | 12 +- pnpm-lock.yaml | 310 ++++++++++++----- .../migrate-interview-categoricals.test.ts | 315 ++++++++++++++++++ scripts/migrate-interview-categoricals.ts | 186 +++++++++++ scripts/setup-database.ts | 2 + 6 files changed, 743 insertions(+), 85 deletions(-) create mode 100644 scripts/__tests__/migrate-interview-categoricals.test.ts create mode 100644 scripts/migrate-interview-categoricals.ts diff --git a/app/api/generate-test-interviews/route.ts b/app/api/generate-test-interviews/route.ts index 9c4cd36c7..ccddf6273 100644 --- a/app/api/generate-test-interviews/route.ts +++ b/app/api/generate-test-interviews/route.ts @@ -67,7 +67,7 @@ export async function POST(request: Request) { for (let i = 0; i < count; i++) { const { network, stageMetadata, currentStep, droppedOut } = - generateNetwork(typedCodebook, typedStages, undefined, genOptions); + generateNetwork(typedCodebook, typedStages, genOptions); const isCompleted = !droppedOut; if (isCompleted) { @@ -133,7 +133,6 @@ export async function POST(request: Request) { const { network, stageMetadata, currentStep } = generateNetwork( typedCodebook, typedStages, - undefined, { ...genOptions, simulateDropOut: false, diff --git a/package.json b/package.json index a02cfd6df..7c4082754 100644 --- a/package.json +++ b/package.json @@ -28,12 +28,12 @@ "@aws-sdk/client-s3": "^3.1068.0", "@aws-sdk/s3-request-presigner": "^3.1068.0", "@base-ui/react": "^1.5.0", - "@codaco/fresco-ui": "^2.13.0", - "@codaco/interview": "^1.0.1", - "@codaco/network-exporters": "^1.0.3", - "@codaco/protocol-utilities": "^1.0.0", - "@codaco/protocol-validation": "^11.6.1", - "@codaco/shared-consts": "5.2.0", + "@codaco/fresco-ui": "^2.14.0", + "@codaco/interview": "^1.1.0", + "@codaco/network-exporters": "^1.1.0", + "@codaco/protocol-utilities": "^2.0.0", + "@codaco/protocol-validation": "^11.7.0", + "@codaco/shared-consts": "5.3.0", "@codaco/tailwind-config": "^1.0.1", "@paralleldrive/cuid2": "^3.3.0", "@posthog/nextjs-config": "^1.9.66", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4435c1167..71b8b7efa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,23 +24,23 @@ importers: specifier: ^1.5.0 version: 1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) '@codaco/fresco-ui': - specifier: ^2.13.0 - version: 2.13.0(@base-ui/react@1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@codaco/shared-consts@5.2.0)(@codaco/tailwind-config@1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1))(@floating-ui/dom@1.7.6)(@tanstack/react-table@8.21.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@tiptap/extension-list@3.24.0(@tiptap/core@3.24.0(@tiptap/pm@3.24.0))(@tiptap/pm@3.24.0))(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(motion@12.40.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1)(typescript@6.0.3)(zod@4.4.3)(zustand@5.0.14(@types/react@19.2.17)(immer@11.1.8)(react@19.2.7)(use-sync-external-store@1.6.0(react@19.2.7))) + specifier: ^2.14.0 + version: 2.14.0(@base-ui/react@1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@codaco/shared-consts@5.3.0)(@codaco/tailwind-config@1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1))(@floating-ui/dom@1.7.6)(@tanstack/react-table@8.21.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@tiptap/extension-list@3.24.0(@tiptap/core@3.24.0(@tiptap/pm@3.24.0))(@tiptap/pm@3.24.0))(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(motion@12.40.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1)(typescript@6.0.3)(zod@4.4.3)(zustand@5.0.14(@types/react@19.2.17)(immer@11.1.8)(react@19.2.7)(use-sync-external-store@1.6.0(react@19.2.7))) '@codaco/interview': - specifier: ^1.0.1 - version: 1.0.1(dd9ec96e0023f7ec0f33bb20963f8bbd) + specifier: ^1.1.0 + version: 1.1.0(1b402db64fdf29c167eb1ba29139ca9d) '@codaco/network-exporters': - specifier: ^1.0.3 - version: 1.0.3 + specifier: ^1.1.0 + version: 1.1.0 '@codaco/protocol-utilities': - specifier: ^1.0.0 - version: 1.0.0 + specifier: ^2.0.0 + version: 2.0.0 '@codaco/protocol-validation': - specifier: ^11.6.1 - version: 11.6.1 + specifier: ^11.7.0 + version: 11.7.0 '@codaco/shared-consts': - specifier: 5.2.0 - version: 5.2.0 + specifier: 5.3.0 + version: 5.3.0 '@codaco/tailwind-config': specifier: ^1.0.1 version: 1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1) @@ -574,11 +574,11 @@ packages: peerDependencies: storybook: ^0.0.0-0 || ^10.1.0 || ^10.1.0-0 || ^10.2.0-0 || ^10.3.0-0 || ^10.4.0-0 || ^10.5.0-0 || ^10.6.0-0 - '@codaco/fresco-ui@2.13.0': - resolution: {integrity: sha512-RuPz395kZiN7ic41EPk9R7NC8aUXXFDVOOr4KGu7qTuG14UnbjWpB8YLISdG13PdslviMUsN7x/QT9sbaYQKVQ==} + '@codaco/fresco-ui@2.14.0': + resolution: {integrity: sha512-X2ntMiYwQRccMsW79bENOmsKMoV187crAn7G9jqZjv/SOneNAskNEX8QW/YWkbBYK7Dwi2dSVoL6JxVPcPMjMw==} peerDependencies: '@base-ui/react': ^1.4.0 - '@codaco/shared-consts': ^5.2.0 + '@codaco/shared-consts': ^5.3.0 '@codaco/tailwind-config': ^1.0.1 '@tanstack/react-table': ^8.21.3 motion: ^12.40.0 @@ -588,45 +588,33 @@ packages: zod: ^4.4.3 zustand: ^5.0.14 - '@codaco/interview@1.0.1': - resolution: {integrity: sha512-wIKKaaI2fq2GAAURSano+63oMIIQvC90XzB31EdFgmNWL5WTNylHkh2j8APkS2WzTl3+FPdUWVBtkpch6aatKA==} + '@codaco/interview@1.1.0': + resolution: {integrity: sha512-Y+gqo7WVFhqI4UsHKh5mIQ4T42XmonBLcp94IQM08kCwfI+cHhRiA0oXdvMpygKUoz4/HY8I61Ai+b1BMjUGLA==} peerDependencies: - '@codaco/fresco-ui': ^2.13.0 - '@codaco/protocol-validation': ^11.6.1 - '@codaco/shared-consts': ^5.2.0 + '@codaco/fresco-ui': ^2.14.0 + '@codaco/protocol-validation': ^11.7.0 + '@codaco/shared-consts': ^5.3.0 '@codaco/tailwind-config': ^1.0.1 motion: ^12.40.0 react: ^19.2.6 react-dom: ^19.2.6 tailwindcss: ^4.3.0 - '@codaco/network-exporters@1.0.3': - resolution: {integrity: sha512-1r72LZ6aKikWr2ZH+8bPwr4/uII7f6tlLGaZS10q/AbPRp2DY0SyvbTmOKv8cJ0Pis4RqjIQWOjYhEk0J8wl3A==} + '@codaco/network-exporters@1.1.0': + resolution: {integrity: sha512-83KFLiD/0+xb9Ikzmc/OjuUi5ChC0o6lTWYg1dLwojixGYYOdZWfmPIL7RrFHrJsYKGq/AtWIovHEbJrynLTWg==} - '@codaco/network-query@1.0.1': - resolution: {integrity: sha512-v93+t6ZM3HykaN9MjsFQc4abUlQPJ5gEgGxpa5XAaE6ZCKoJG95EKBI1llBTvwid25FfTSWAZYgHYY036i2CUA==} + '@codaco/network-query@1.1.0': + resolution: {integrity: sha512-EBG+unaiH56RdpJQMgyXxbekCzL5vJXJsCVzMmoX6D2+BU91G78LTsIb+JnG2b9QCOktYqa6eTwYATcXpDKCXA==} - '@codaco/protocol-utilities@1.0.0': - resolution: {integrity: sha512-S2A+ZXXLnGR4hsuxLd02u+uo3duyagsD0jn+RMGRIo+CtnimFIxX5FDEQyYQKOkRRtvbWkcwr0z20JerY2df0A==} - - '@codaco/protocol-validation@11.4.0': - resolution: {integrity: sha512-mfoZqpPc9YW+qk9rPULrkyrUNlgJnaOgDBCb9gPzklpgV6NQDazala9DUvneBHTuBFlGq0hIfrIKBwkkyxNC2A==} - hasBin: true + '@codaco/protocol-utilities@2.0.0': + resolution: {integrity: sha512-31vh5/Gt9N93Djf1XIr+8elF60ixl+k+45T5vFIQ/fbNH5VE3RkTdsR2s8AZCPHC6SnKuZGSS+lRZqxvLaiLDg==} - '@codaco/protocol-validation@11.6.1': - resolution: {integrity: sha512-Vbj9BBlL4GrtBzrvfkkuvGYdZue2s4xuSI35Ejn50riEYKasS2ADm+EuUqklVVd6V4qagG5DQf0cPd7hDlgI9w==} + '@codaco/protocol-validation@11.7.0': + resolution: {integrity: sha512-MSSoo+vp9Kfysg2hvU9rU/gjKo2MPJbTYtSCUz52dNVMjtsOdiEM4ii/RcwJhbOZi0lnOJoXZdzW116CqqvkuQ==} hasBin: true - '@codaco/shared-consts@5.0.0': - resolution: {integrity: sha512-7neTWHNjw6Y7FdVOU8htO2OshADlpS9vI+5mQEDWOdtlL8BdHmBflqQlZE8lFjzBin8tUI9jIlxsLlMYOpjm/w==} - engines: {node: '>=20.0.0'} - - '@codaco/shared-consts@5.1.0': - resolution: {integrity: sha512-lPWbwcsaGKUUe3grkhk2USdA7dMZGiEgGxef0SwgTsXCVGyNxyBPWq6Vgt+w2HSU2X317aqcUwQBtXXs9BAXOQ==} - engines: {node: '>=20.0.0'} - - '@codaco/shared-consts@5.2.0': - resolution: {integrity: sha512-N1GZ8nDqW0jxICp6QeNLc8tA8Yley4GtZ1xLoH5qNqJM1dm7VCbKjPBhYcCquS0rJqUw5+gRdzWCE7Vm2XESrA==} + '@codaco/shared-consts@5.3.0': + resolution: {integrity: sha512-lZMGWKLI1d7OxNZSA3GzkgmNJupAB13nfeq0G27yxF4eJrEKmmUHUGT0fX9evcrrK4VJZKr/gD4aOQruIPY+cQ==} engines: {node: '>=20.0.0'} '@codaco/tailwind-config@1.0.1': @@ -5062,6 +5050,9 @@ packages: mapbox-gl@3.24.0: resolution: {integrity: sha512-R+FdFUB3DnoE5FYASV7lGSiRyMkSblcZ2UEy7b2pt7s5ZbCxFIUPXd0E6iAFd8OdvdA2VtbvZZVylzAZNaurjA==} + markdown-table@3.0.4: + resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} + martinez-polygon-clipping@0.8.1: resolution: {integrity: sha512-9PLLMzMPI6ihHox4Ns6LpVBLpRc7sbhULybZ/wyaY8sY3ECNe2+hxm1hA2/9bEEpRrdpjoeduBuZLg2aq1cSIQ==} @@ -5075,6 +5066,24 @@ packages: mdast-util-from-markdown@2.0.3: resolution: {integrity: sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==} + mdast-util-gfm-autolink-literal@2.0.1: + resolution: {integrity: sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==} + + mdast-util-gfm-footnote@2.1.0: + resolution: {integrity: sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==} + + mdast-util-gfm-strikethrough@2.0.0: + resolution: {integrity: sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==} + + mdast-util-gfm-table@2.0.0: + resolution: {integrity: sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==} + + mdast-util-gfm-task-list-item@2.0.0: + resolution: {integrity: sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==} + + mdast-util-gfm@3.1.0: + resolution: {integrity: sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==} + mdast-util-mdx-expression@2.0.1: resolution: {integrity: sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==} @@ -5112,6 +5121,27 @@ packages: micromark-core-commonmark@2.0.3: resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} + micromark-extension-gfm-autolink-literal@2.1.0: + resolution: {integrity: sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==} + + micromark-extension-gfm-footnote@2.1.0: + resolution: {integrity: sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==} + + micromark-extension-gfm-strikethrough@2.1.0: + resolution: {integrity: sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==} + + micromark-extension-gfm-table@2.1.1: + resolution: {integrity: sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==} + + micromark-extension-gfm-tagfilter@2.0.0: + resolution: {integrity: sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==} + + micromark-extension-gfm-task-list-item@2.1.0: + resolution: {integrity: sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==} + + micromark-extension-gfm@3.0.0: + resolution: {integrity: sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==} + micromark-factory-destination@2.0.1: resolution: {integrity: sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==} @@ -5882,12 +5912,18 @@ packages: remark-gemoji@8.0.0: resolution: {integrity: sha512-/fL9rc72FYwFGtOKcT+QeQdx9Q9t5v4N6KLXSDOTEgaedzK85I9judBqB2eqz+g4b0ERMejlwSOuPK+wket6aA==} + remark-gfm@4.0.1: + resolution: {integrity: sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==} + remark-parse@11.0.0: resolution: {integrity: sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==} remark-rehype@11.1.2: resolution: {integrity: sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw==} + remark-stringify@11.0.0: + resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==} + remeda@2.33.4: resolution: {integrity: sha512-ygHswjlc/opg2VrtiYvUOPLjxjtdKvjGz1/plDhkG66hjNjFr1xmfrs2ClNFo/E6TyUFiwYNh53bKV26oBoMGQ==} @@ -7445,10 +7481,10 @@ snapshots: - '@chromatic-com/playwright' - '@chromatic-com/vitest' - '@codaco/fresco-ui@2.13.0(@base-ui/react@1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@codaco/shared-consts@5.2.0)(@codaco/tailwind-config@1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1))(@floating-ui/dom@1.7.6)(@tanstack/react-table@8.21.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@tiptap/extension-list@3.24.0(@tiptap/core@3.24.0(@tiptap/pm@3.24.0))(@tiptap/pm@3.24.0))(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(motion@12.40.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1)(typescript@6.0.3)(zod@4.4.3)(zustand@5.0.14(@types/react@19.2.17)(immer@11.1.8)(react@19.2.7)(use-sync-external-store@1.6.0(react@19.2.7)))': + '@codaco/fresco-ui@2.14.0(@base-ui/react@1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@codaco/shared-consts@5.3.0)(@codaco/tailwind-config@1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1))(@floating-ui/dom@1.7.6)(@tanstack/react-table@8.21.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@tiptap/extension-list@3.24.0(@tiptap/core@3.24.0(@tiptap/pm@3.24.0))(@tiptap/pm@3.24.0))(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(motion@12.40.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1)(typescript@6.0.3)(zod@4.4.3)(zustand@5.0.14(@types/react@19.2.17)(immer@11.1.8)(react@19.2.7)(use-sync-external-store@1.6.0(react@19.2.7)))': dependencies: '@base-ui/react': 1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - '@codaco/shared-consts': 5.2.0 + '@codaco/shared-consts': 5.3.0 '@codaco/tailwind-config': 1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1) '@radix-ui/react-slot': 1.2.4(@types/react@19.2.17)(react@19.2.7) '@tanstack/react-table': 8.21.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7) @@ -7476,6 +7512,7 @@ snapshots: rehype-raw: 7.0.0 rehype-sanitize: 6.0.0 remark-gemoji: 8.0.0 + remark-gfm: 4.0.1 tailwind-merge: 3.6.0 tailwindcss: 4.3.1 usehooks-ts: 3.1.1(react@19.2.7) @@ -7489,13 +7526,13 @@ snapshots: - supports-color - typescript - '@codaco/interview@1.0.1(dd9ec96e0023f7ec0f33bb20963f8bbd)': + '@codaco/interview@1.1.0(1b402db64fdf29c167eb1ba29139ca9d)': dependencies: '@base-ui/react': 1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - '@codaco/fresco-ui': 2.13.0(@base-ui/react@1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@codaco/shared-consts@5.2.0)(@codaco/tailwind-config@1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1))(@floating-ui/dom@1.7.6)(@tanstack/react-table@8.21.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@tiptap/extension-list@3.24.0(@tiptap/core@3.24.0(@tiptap/pm@3.24.0))(@tiptap/pm@3.24.0))(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(motion@12.40.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1)(typescript@6.0.3)(zod@4.4.3)(zustand@5.0.14(@types/react@19.2.17)(immer@11.1.8)(react@19.2.7)(use-sync-external-store@1.6.0(react@19.2.7))) - '@codaco/network-query': 1.0.1 - '@codaco/protocol-validation': 11.6.1 - '@codaco/shared-consts': 5.2.0 + '@codaco/fresco-ui': 2.14.0(@base-ui/react@1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@codaco/shared-consts@5.3.0)(@codaco/tailwind-config@1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1))(@floating-ui/dom@1.7.6)(@tanstack/react-table@8.21.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@tiptap/extension-list@3.24.0(@tiptap/core@3.24.0(@tiptap/pm@3.24.0))(@tiptap/pm@3.24.0))(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(motion@12.40.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1)(typescript@6.0.3)(zod@4.4.3)(zustand@5.0.14(@types/react@19.2.17)(immer@11.1.8)(react@19.2.7)(use-sync-external-store@1.6.0(react@19.2.7))) + '@codaco/network-query': 1.1.0 + '@codaco/protocol-validation': 11.7.0 + '@codaco/shared-consts': 5.3.0 '@codaco/tailwind-config': 1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1) '@faker-js/faker': 10.4.0 '@mapbox/search-js-react': 1.5.1(mapbox-gl@3.24.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) @@ -7525,10 +7562,10 @@ snapshots: - redux - use-sync-external-store - '@codaco/network-exporters@1.0.3': + '@codaco/network-exporters@1.1.0': dependencies: - '@codaco/protocol-validation': 11.6.1 - '@codaco/shared-consts': 5.1.0 + '@codaco/protocol-validation': 11.7.0 + '@codaco/shared-consts': 5.3.0 '@xmldom/xmldom': 0.9.10 effect: 3.21.3 es-toolkit: 1.47.1 @@ -7537,43 +7574,28 @@ snapshots: sanitize-filename: 1.6.4 zod: 4.4.3 - '@codaco/network-query@1.0.1': + '@codaco/network-query@1.1.0': dependencies: - '@codaco/protocol-validation': 11.4.0 - '@codaco/shared-consts': 5.0.0 + '@codaco/protocol-validation': 11.7.0 + '@codaco/shared-consts': 5.3.0 es-toolkit: 1.47.1 - '@codaco/protocol-utilities@1.0.0': + '@codaco/protocol-utilities@2.0.0': dependencies: - '@codaco/network-query': 1.0.1 - '@codaco/protocol-validation': 11.6.1 - '@codaco/shared-consts': 5.2.0 + '@codaco/network-query': 1.1.0 + '@codaco/protocol-validation': 11.7.0 + '@codaco/shared-consts': 5.3.0 '@faker-js/faker': 10.4.0 es-toolkit: 1.47.1 uuid: 14.0.0 - '@codaco/protocol-validation@11.4.0': + '@codaco/protocol-validation@11.7.0': dependencies: - '@codaco/shared-consts': 5.0.0 - '@faker-js/faker': 10.4.0 - zod: 4.4.3 - - '@codaco/protocol-validation@11.6.1': - dependencies: - '@codaco/shared-consts': 5.1.0 - '@faker-js/faker': 10.4.0 + '@codaco/shared-consts': 5.3.0 ohash: 2.0.11 zod: 4.4.3 - '@codaco/shared-consts@5.0.0': - dependencies: - zod: 4.4.3 - - '@codaco/shared-consts@5.1.0': - dependencies: - zod: 4.4.3 - - '@codaco/shared-consts@5.2.0': + '@codaco/shared-consts@5.3.0': dependencies: zod: 4.4.3 @@ -11777,6 +11799,8 @@ snapshots: supercluster: 8.0.1 tinyqueue: 3.0.0 + markdown-table@3.0.4: {} + martinez-polygon-clipping@0.8.1: dependencies: robust-predicates: 2.0.4 @@ -11809,6 +11833,63 @@ snapshots: transitivePeerDependencies: - supports-color + mdast-util-gfm-autolink-literal@2.0.1: + dependencies: + '@types/mdast': 4.0.4 + ccount: 2.0.1 + devlop: 1.1.0 + mdast-util-find-and-replace: 3.0.2 + micromark-util-character: 2.1.1 + + mdast-util-gfm-footnote@2.1.0: + dependencies: + '@types/mdast': 4.0.4 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + micromark-util-normalize-identifier: 2.0.1 + transitivePeerDependencies: + - supports-color + + mdast-util-gfm-strikethrough@2.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-gfm-table@2.0.0: + dependencies: + '@types/mdast': 4.0.4 + devlop: 1.1.0 + markdown-table: 3.0.4 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-gfm-task-list-item@2.0.0: + dependencies: + '@types/mdast': 4.0.4 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + + mdast-util-gfm@3.1.0: + dependencies: + mdast-util-from-markdown: 2.0.3 + mdast-util-gfm-autolink-literal: 2.0.1 + mdast-util-gfm-footnote: 2.1.0 + mdast-util-gfm-strikethrough: 2.0.0 + mdast-util-gfm-table: 2.0.0 + mdast-util-gfm-task-list-item: 2.0.0 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + mdast-util-mdx-expression@2.0.1: dependencies: '@types/estree-jsx': 1.0.5 @@ -11908,6 +11989,64 @@ snapshots: micromark-util-symbol: 2.0.1 micromark-util-types: 2.0.2 + micromark-extension-gfm-autolink-literal@2.1.0: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm-footnote@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-core-commonmark: 2.0.3 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm-strikethrough@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.1 + micromark-util-classify-character: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm-table@2.1.1: + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm-tagfilter@2.0.0: + dependencies: + micromark-util-types: 2.0.2 + + micromark-extension-gfm-task-list-item@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-gfm@3.0.0: + dependencies: + micromark-extension-gfm-autolink-literal: 2.1.0 + micromark-extension-gfm-footnote: 2.1.0 + micromark-extension-gfm-strikethrough: 2.1.0 + micromark-extension-gfm-table: 2.1.1 + micromark-extension-gfm-tagfilter: 2.0.0 + micromark-extension-gfm-task-list-item: 2.1.0 + micromark-util-combine-extensions: 2.0.1 + micromark-util-types: 2.0.2 + micromark-factory-destination@2.0.1: dependencies: micromark-util-character: 2.1.1 @@ -12817,6 +12956,17 @@ snapshots: gemoji: 8.1.0 mdast-util-find-and-replace: 3.0.2 + remark-gfm@4.0.1: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-gfm: 3.1.0 + micromark-extension-gfm: 3.0.0 + remark-parse: 11.0.0 + remark-stringify: 11.0.0 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + remark-parse@11.0.0: dependencies: '@types/mdast': 4.0.4 @@ -12834,6 +12984,12 @@ snapshots: unified: 11.0.5 vfile: 6.0.3 + remark-stringify@11.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-to-markdown: 2.1.2 + unified: 11.0.5 + remeda@2.33.4: {} require-directory@2.1.1: {} diff --git a/scripts/__tests__/migrate-interview-categoricals.test.ts b/scripts/__tests__/migrate-interview-categoricals.test.ts new file mode 100644 index 000000000..e3073f332 --- /dev/null +++ b/scripts/__tests__/migrate-interview-categoricals.test.ts @@ -0,0 +1,315 @@ +import { type Codebook } from '@codaco/protocol-validation'; +import { type NcNetwork } from '@codaco/shared-consts'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { + migrateInterviewCategoricals, + migrateNetworkCategoricals, +} from '~/scripts/migrate-interview-categoricals'; + +/** + * A codebook with one categorical node variable, one ordinal node variable, one + * categorical edge variable, and one categorical ego variable. The variable ids + * double as the network attribute keys (the Network Canvas contract). + */ +const CAT_NODE = 'cat-node-var'; +const ORD_NODE = 'ord-node-var'; +const TEXT_NODE = 'text-node-var'; +const CAT_EDGE = 'cat-edge-var'; +const CAT_EGO = 'cat-ego-var'; + +function makeCodebook(): Codebook { + return { + node: { + person: { + name: 'Person', + color: 'node-color-seq-1', + shape: { default: 'circle' }, + variables: { + [CAT_NODE]: { + name: 'closeness', + type: 'categorical', + component: 'CheckboxGroup', + options: [ + { label: 'Family', value: 'family' }, + { label: 'Friend', value: 'friend' }, + ], + }, + [ORD_NODE]: { + name: 'frequency', + type: 'ordinal', + component: 'LikertScale', + options: [ + { label: 'Low', value: 1 }, + { label: 'High', value: 2 }, + ], + }, + [TEXT_NODE]: { + name: 'nickname', + type: 'text', + component: 'Text', + }, + }, + }, + }, + edge: { + friend: { + name: 'Friend', + color: 'edge-color-seq-1', + variables: { + [CAT_EDGE]: { + name: 'context', + type: 'categorical', + component: 'ToggleButtonGroup', + options: [ + { label: 'Work', value: 'work' }, + { label: 'School', value: 'school' }, + ], + }, + }, + }, + }, + ego: { + variables: { + [CAT_EGO]: { + name: 'identity', + type: 'categorical', + component: 'CheckboxGroup', + options: [ + { label: 'A', value: 'a' }, + { label: 'B', value: 'b' }, + ], + }, + }, + }, + }; +} + +function makeNetwork(overrides?: Partial): NcNetwork { + return { + nodes: [ + { + _uid: 'node-1', + type: 'person', + attributes: { + [CAT_NODE]: 'family', + [ORD_NODE]: 1, + [TEXT_NODE]: 'Bob', + }, + }, + ], + edges: [ + { + _uid: 'edge-1', + type: 'friend', + from: 'node-1', + to: 'node-1', + attributes: { [CAT_EDGE]: 'work' }, + }, + ], + ego: { + _uid: 'ego-1', + attributes: { [CAT_EGO]: 'a' }, + }, + ...overrides, + }; +} + +describe('migrateNetworkCategoricals', () => { + it('wraps scalar categorical values in single-element arrays', () => { + const { network, changed } = migrateNetworkCategoricals( + makeNetwork(), + makeCodebook(), + ); + + expect(changed).toBe(true); + expect(network.nodes[0]?.attributes[CAT_NODE]).toEqual(['family']); + expect(network.edges[0]?.attributes[CAT_EDGE]).toEqual(['work']); + expect(network.ego.attributes[CAT_EGO]).toEqual(['a']); + }); + + it('leaves ordinal, text, and number values untouched', () => { + const { network } = migrateNetworkCategoricals( + makeNetwork(), + makeCodebook(), + ); + + expect(network.nodes[0]?.attributes[ORD_NODE]).toBe(1); + expect(network.nodes[0]?.attributes[TEXT_NODE]).toBe('Bob'); + }); + + it('wraps scalar number and boolean categorical values', () => { + const network = makeNetwork({ + nodes: [ + { + _uid: 'node-1', + type: 'person', + attributes: { [CAT_NODE]: 2 }, + }, + ], + }); + + const result = migrateNetworkCategoricals(network, makeCodebook()); + expect(result.changed).toBe(true); + expect(result.network.nodes[0]?.attributes[CAT_NODE]).toEqual([2]); + }); + + it('is idempotent: already-array categorical values are unchanged', () => { + const network = makeNetwork({ + nodes: [ + { + _uid: 'node-1', + type: 'person', + attributes: { [CAT_NODE]: ['family'] }, + }, + ], + edges: [], + ego: { _uid: 'ego-1', attributes: {} }, + }); + + const result = migrateNetworkCategoricals(network, makeCodebook()); + expect(result.changed).toBe(false); + expect(result.network.nodes[0]?.attributes[CAT_NODE]).toEqual(['family']); + }); + + it('leaves unanswered (null) categorical values untouched', () => { + const network = makeNetwork({ + nodes: [ + { + _uid: 'node-1', + type: 'person', + attributes: { [CAT_NODE]: null }, + }, + ], + edges: [], + ego: { _uid: 'ego-1', attributes: {} }, + }); + + const result = migrateNetworkCategoricals(network, makeCodebook()); + expect(result.changed).toBe(false); + expect(result.network.nodes[0]?.attributes[CAT_NODE]).toBeNull(); + }); + + it('reports no change when a network has no categorical scalars', () => { + const network = makeNetwork({ + nodes: [ + { + _uid: 'node-1', + type: 'person', + attributes: { [ORD_NODE]: 1, [TEXT_NODE]: 'Bob' }, + }, + ], + edges: [], + ego: { _uid: 'ego-1', attributes: {} }, + }); + + const result = migrateNetworkCategoricals(network, makeCodebook()); + expect(result.changed).toBe(false); + }); +}); + +type MockPrisma = { + interview: { + findMany: ReturnType; + update: ReturnType; + }; +}; + +function makeMockPrisma(): MockPrisma { + return { + interview: { + findMany: vi.fn().mockResolvedValue([]), + update: vi.fn().mockResolvedValue({}), + }, + }; +} + +function makeRow(id: string, attributes: Record) { + return { + id, + protocolId: 'protocol-1', + protocol: { codebook: makeCodebook() }, + network: { + nodes: [{ _uid: 'node-1', type: 'person', attributes }], + edges: [], + ego: { _uid: 'ego-1', attributes: {} }, + }, + }; +} + +describe('migrateInterviewCategoricals', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('is a no-op when there are no interviews', async () => { + const prisma = makeMockPrisma(); + + await migrateInterviewCategoricals( + prisma as unknown as Parameters[0], + ); + + expect(prisma.interview.update).not.toHaveBeenCalled(); + }); + + it('rewrites only interviews whose categorical values are scalars', async () => { + const prisma = makeMockPrisma(); + prisma.interview.findMany.mockResolvedValueOnce([ + makeRow('needs-migration', { [CAT_NODE]: 'family' }), + makeRow('already-array', { [CAT_NODE]: ['friend'] }), + ]); + + await migrateInterviewCategoricals( + prisma as unknown as Parameters[0], + ); + + expect(prisma.interview.update).toHaveBeenCalledTimes(1); + const call = prisma.interview.update.mock.calls[0]?.[0] as { + where: { id: string }; + data: { network: NcNetwork }; + }; + expect(call.where).toEqual({ id: 'needs-migration' }); + expect(call.data.network.nodes[0]?.attributes[CAT_NODE]).toEqual([ + 'family', + ]); + }); + + it('skips an interview whose network fails to parse without aborting', async () => { + const prisma = makeMockPrisma(); + prisma.interview.findMany.mockResolvedValueOnce([ + { ...makeRow('broken', {}), network: 'not-a-network' }, + makeRow('valid', { [CAT_NODE]: 'family' }), + ]); + + await migrateInterviewCategoricals( + prisma as unknown as Parameters[0], + ); + + expect(prisma.interview.update).toHaveBeenCalledTimes(1); + const call = prisma.interview.update.mock.calls[0]?.[0] as { + where: { id: string }; + }; + expect(call.where).toEqual({ id: 'valid' }); + }); + + it('pages through interviews until a short batch is returned', async () => { + const prisma = makeMockPrisma(); + const fullBatch = Array.from({ length: 200 }, (_, i) => + makeRow(`id-${String(i).padStart(3, '0')}`, { [CAT_NODE]: ['friend'] }), + ); + prisma.interview.findMany + .mockResolvedValueOnce(fullBatch) + .mockResolvedValueOnce([makeRow('last', { [CAT_NODE]: ['friend'] })]); + + await migrateInterviewCategoricals( + prisma as unknown as Parameters[0], + ); + + expect(prisma.interview.findMany).toHaveBeenCalledTimes(2); + const secondCall = prisma.interview.findMany.mock.calls[1]?.[0] as { + cursor?: { id: string }; + skip?: number; + }; + expect(secondCall.cursor).toEqual({ id: 'id-199' }); + expect(secondCall.skip).toBe(1); + }); +}); diff --git a/scripts/migrate-interview-categoricals.ts b/scripts/migrate-interview-categoricals.ts new file mode 100644 index 000000000..15a056cf2 --- /dev/null +++ b/scripts/migrate-interview-categoricals.ts @@ -0,0 +1,186 @@ +/* eslint-disable no-console */ +import { + type Codebook, + CodebookSchema, + type Variable, +} from '@codaco/protocol-validation'; +import { + type NcNetwork, + NcNetworkSchema, + type VariableValue, +} from '@codaco/shared-consts'; +import { type PrismaClient } from '~/lib/db/generated/client'; + +/** + * `@codaco/interview` >= 1.1.0 stores categorical attribute values as arrays of + * selected option values (a single selection is a one-element array). Networks + * collected by earlier versions may hold a bare scalar (written by CategoricalBin), + * which the new package's array-only readers no longer match. This migration wraps + * those scalars in single-element arrays. See Fresco#795. + */ + +function categoricalVariableIds( + variables: Record | undefined, +): Set { + const ids = new Set(); + if (!variables) return ids; + for (const [id, variable] of Object.entries(variables)) { + if (variable.type === 'categorical') ids.add(id); + } + return ids; +} + +/** + * Wrap any scalar value held by a categorical variable in a single-element array. + * Already-array values, unanswered (`null`/`undefined`) values, and the `{x, y}` + * layout object are left untouched, so the transform is idempotent. + */ +function wrapEntityCategoricals( + attributes: Record, + categoricalIds: Set, +): boolean { + let changed = false; + for (const id of categoricalIds) { + if (!Object.prototype.hasOwnProperty.call(attributes, id)) continue; + const value = attributes[id]; + if (value === null || value === undefined) continue; + if (Array.isArray(value)) continue; + if (typeof value === 'object') continue; + attributes[id] = [value]; + changed = true; + } + return changed; +} + +/** + * Convert scalar categorical attribute values to single-element arrays across a + * single interview network, resolving categorical variables from the protocol + * codebook. Mutates and returns the passed network; `changed` reports whether any + * value was rewritten. + */ +export function migrateNetworkCategoricals( + network: NcNetwork, + codebook: Codebook, +): { network: NcNetwork; changed: boolean } { + let changed = false; + + for (const node of network.nodes) { + const ids = categoricalVariableIds(codebook.node?.[node.type]?.variables); + if (ids.size > 0 && wrapEntityCategoricals(node.attributes, ids)) { + changed = true; + } + } + + for (const edge of network.edges) { + const ids = categoricalVariableIds(codebook.edge?.[edge.type]?.variables); + if (ids.size > 0 && wrapEntityCategoricals(edge.attributes, ids)) { + changed = true; + } + } + + const egoIds = categoricalVariableIds(codebook.ego?.variables); + if ( + egoIds.size > 0 && + wrapEntityCategoricals(network.ego.attributes, egoIds) + ) { + changed = true; + } + + return { network, changed }; +} + +const BATCH_SIZE = 200; + +/** + * Migrate scalar categorical attribute values to single-element arrays across all + * stored interview networks. + * + * Idempotent: networks already on the array contract are read but not rewritten. + * Codebooks are parsed once per protocol and cached. An interview whose network or + * codebook fails to parse is logged and skipped rather than aborting the run, so a + * single malformed row can't block a deploy. + */ +export async function migrateInterviewCategoricals( + prisma: PrismaClient, +): Promise { + const codebookCache = new Map(); + + const getCodebook = (protocolId: string, raw: unknown): Codebook | null => { + const cached = codebookCache.get(protocolId); + if (cached !== undefined) return cached; + + const result = CodebookSchema.safeParse(raw); + const codebook = result.success ? result.data : null; + if (!codebook) { + console.warn( + `Skipping interviews for protocol ${protocolId}: codebook failed to parse.`, + ); + } + codebookCache.set(protocolId, codebook); + return codebook; + }; + + let cursor: string | undefined; + let scanned = 0; + let migrated = 0; + let skipped = 0; + + for (;;) { + const batch = await prisma.interview.findMany({ + take: BATCH_SIZE, + ...(cursor ? { skip: 1, cursor: { id: cursor } } : {}), + orderBy: { id: 'asc' }, + select: { + id: true, + network: true, + protocolId: true, + protocol: { select: { codebook: true } }, + }, + }); + + if (batch.length === 0) break; + + for (const row of batch) { + scanned++; + + const codebook = getCodebook(row.protocolId, row.protocol.codebook); + if (!codebook) { + skipped++; + continue; + } + + const parsedNetwork = NcNetworkSchema.safeParse(row.network); + if (!parsedNetwork.success) { + console.warn(`Skipping interview ${row.id}: network failed to parse.`); + skipped++; + continue; + } + + const { network, changed } = migrateNetworkCategoricals( + parsedNetwork.data, + codebook, + ); + + if (changed) { + await prisma.interview.update({ + where: { id: row.id }, + data: { network }, + }); + migrated++; + } + } + + cursor = batch[batch.length - 1]?.id; + if (batch.length < BATCH_SIZE) break; + } + + if (scanned === 0) { + console.log('No interviews to scan for categorical migration.'); + return; + } + + console.log( + `Categorical migration: scanned ${scanned} interviews, ` + + `rewrote ${migrated}, skipped ${skipped}.`, + ); +} diff --git a/scripts/setup-database.ts b/scripts/setup-database.ts index 262b6c109..94dd5250f 100644 --- a/scripts/setup-database.ts +++ b/scripts/setup-database.ts @@ -6,6 +6,7 @@ dotenv.config(); import { PrismaPg } from '@prisma/adapter-pg'; import { execSync, spawnSync } from 'child_process'; import { PrismaClient } from '~/lib/db/generated/client'; +import { migrateInterviewCategoricals } from './migrate-interview-categoricals'; import { migrateProtocolsToV8 } from './migrate-protocols-to-v8'; // CLI scripts must use the PG adapter directly because the Neon serverless @@ -110,6 +111,7 @@ async function handleMigrations(): Promise { try { await handleMigrations(); await migrateProtocolsToV8(prisma); + await migrateInterviewCategoricals(prisma); } catch (error) { console.error('Error during database setup:', error); process.exit(1); From 171f5cf28147cd9c0bdcdd3901045370792d09f9 Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Fri, 26 Jun 2026 15:38:17 -0400 Subject: [PATCH 2/5] fix(migrations): run data migrations atomically in a transaction Wrap the protocol-v8 and interview-categorical data migrations in a single interactive transaction so a failure during setup-database rolls them all back. A partially-migrated database is dangerous on a failed 3.x -> 4.x upgrade: the previous Fresco version keeps serving and would read half-converted protocols/networks written for the new interview module. All-or-nothing keeps the old version working on the old data. --- scripts/migrate-interview-categoricals.ts | 4 ++-- scripts/migrate-protocols-to-v8.ts | 6 +++--- scripts/setup-database.ts | 20 ++++++++++++++++++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/migrate-interview-categoricals.ts b/scripts/migrate-interview-categoricals.ts index 15a056cf2..d28d5c6ee 100644 --- a/scripts/migrate-interview-categoricals.ts +++ b/scripts/migrate-interview-categoricals.ts @@ -9,7 +9,7 @@ import { NcNetworkSchema, type VariableValue, } from '@codaco/shared-consts'; -import { type PrismaClient } from '~/lib/db/generated/client'; +import { type Prisma } from '~/lib/db/generated/client'; /** * `@codaco/interview` >= 1.1.0 stores categorical attribute values as arrays of @@ -101,7 +101,7 @@ const BATCH_SIZE = 200; * single malformed row can't block a deploy. */ export async function migrateInterviewCategoricals( - prisma: PrismaClient, + prisma: Prisma.TransactionClient, ): Promise { const codebookCache = new Map(); diff --git a/scripts/migrate-protocols-to-v8.ts b/scripts/migrate-protocols-to-v8.ts index 42d5f6a34..c21f7ea5d 100644 --- a/scripts/migrate-protocols-to-v8.ts +++ b/scripts/migrate-protocols-to-v8.ts @@ -1,9 +1,9 @@ /* eslint-disable no-console */ import { hashProtocol, migrateProtocol } from '@codaco/protocol-validation'; -import { Prisma, type PrismaClient } from '~/lib/db/generated/client'; +import { Prisma } from '~/lib/db/generated/client'; async function migrateOneProtocol( - prisma: PrismaClient, + prisma: Prisma.TransactionClient, row: { id: string; name: string; @@ -71,7 +71,7 @@ async function migrateOneProtocol( * Idempotent. Hard-fails per protocol on migration errors. */ export async function migrateProtocolsToV8( - prisma: PrismaClient, + prisma: Prisma.TransactionClient, ): Promise { const v7Protocols = await prisma.protocol.findMany({ where: { schemaVersion: { lt: 8 } }, diff --git a/scripts/setup-database.ts b/scripts/setup-database.ts index 94dd5250f..1ce15c6b3 100644 --- a/scripts/setup-database.ts +++ b/scripts/setup-database.ts @@ -108,10 +108,26 @@ async function handleMigrations(): Promise { } } +// Generous ceiling for the one-time data migrations, which run at deploy time +// with no concurrent app traffic. If a deployment exceeds it, raise this rather +// than splitting the migrations across transactions. +const DATA_MIGRATION_TIMEOUT_MS = 1000 * 60 * 30; + try { await handleMigrations(); - await migrateProtocolsToV8(prisma); - await migrateInterviewCategoricals(prisma); + + // Run the in-place data migrations together in a single transaction so a + // failure rolls them all back. A partially-migrated database is dangerous: + // when the deploy aborts, the previous Fresco version keeps serving and would + // read half-converted protocols/networks written for the new interview + // module. All-or-nothing keeps the old version working on the old data. + await prisma.$transaction( + async (tx) => { + await migrateProtocolsToV8(tx); + await migrateInterviewCategoricals(tx); + }, + { timeout: DATA_MIGRATION_TIMEOUT_MS }, + ); } catch (error) { console.error('Error during database setup:', error); process.exit(1); From 550374f3636181497958577697ec088378e7ae61 Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Fri, 26 Jun 2026 15:47:47 -0400 Subject: [PATCH 3/5] fix(migrations): reconstruct asset manifest for v8 protocol validation protocol-validation 11.7.0 added strict validation that a NameGeneratorRoster dataSource (and Geospatial asset references) resolve to an entry in the protocol's assetManifest. migrateProtocolsToV8 reconstructed the protocol from only name/schemaVersion/stages/codebook, so the manifest was absent and migration failed for any protocol with a roster: Roster dataSource "..." does not reference an asset in the manifest. Fresco stores assets in a separate table, so rebuild the assetManifest from the protocol's linked Asset rows and include it in the document passed to migrateProtocol. The manifest is excluded from the protocol hash, so this does not change computed hashes. Verified against the real validator with a roster protocol. --- .../__tests__/migrate-protocols-to-v8.test.ts | 56 ++++++++++++++++++- scripts/migrate-protocols-to-v8.ts | 46 +++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/scripts/__tests__/migrate-protocols-to-v8.test.ts b/scripts/__tests__/migrate-protocols-to-v8.test.ts index 94feaf55a..8cb553c9c 100644 --- a/scripts/__tests__/migrate-protocols-to-v8.test.ts +++ b/scripts/__tests__/migrate-protocols-to-v8.test.ts @@ -1,6 +1,9 @@ import { hashProtocol, migrateProtocol } from '@codaco/protocol-validation'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { migrateProtocolsToV8 } from '~/scripts/migrate-protocols-to-v8'; +import { + buildAssetManifest, + migrateProtocolsToV8, +} from '~/scripts/migrate-protocols-to-v8'; /** * A minimal v7 protocol JSON containing the fields the v7→v8 migration @@ -85,6 +88,7 @@ describe('migrateProtocolsToV8', () => { prisma.protocol.findMany.mockResolvedValue([ { id: 'cm-protocol-1', + assets: [], name: 'Test Protocol.netcanvas', schemaVersion: 7, stages: v7.stages, @@ -146,6 +150,7 @@ describe('migrateProtocolsToV8', () => { prisma.protocol.findMany.mockResolvedValue([ { id: 'cm-x', + assets: [], name: 'My Protocol.netcanvas', schemaVersion: 7, stages: v7.stages, @@ -181,6 +186,7 @@ describe('migrateProtocolsToV8', () => { prisma.protocol.findMany.mockResolvedValue([ { id: 'cm-broken', + assets: [], name: 'Broken Protocol.netcanvas', schemaVersion: 7, // `stages` is required to be an array — passing a non-array makes @@ -212,6 +218,7 @@ describe('migrateProtocolsToV8', () => { prisma.protocol.findMany.mockResolvedValue([ { id: 'cm-collide', + assets: [], name: 'Colliding.netcanvas', schemaVersion: 7, stages: v7.stages, @@ -248,3 +255,50 @@ describe('migrateProtocolsToV8', () => { expect(message).toMatch(/Existing\.netcanvas/); }); }); + +describe('buildAssetManifest', () => { + it('reconstructs a file-asset manifest entry from a stored Asset row', () => { + const manifest = buildAssetManifest([ + { + assetId: 'asset-network-1', + name: 'roster-source.csv', + type: 'network', + value: null, + }, + ]); + + expect(manifest['asset-network-1']).toEqual({ + id: 'asset-network-1', + name: 'roster-source.csv', + type: 'network', + source: 'roster-source.csv', + }); + }); + + it('reconstructs an apikey manifest entry using value, not source', () => { + const manifest = buildAssetManifest([ + { + assetId: 'asset-key-1', + name: 'Mapbox token', + type: 'apikey', + value: 'pk.secret', + }, + ]); + + expect(manifest['asset-key-1']).toEqual({ + id: 'asset-key-1', + name: 'Mapbox token', + type: 'apikey', + value: 'pk.secret', + }); + }); + + it('keys every asset by its assetId', () => { + const manifest = buildAssetManifest([ + { assetId: 'a', name: 'a.png', type: 'image', value: null }, + { assetId: 'b', name: 'b.geojson', type: 'geojson', value: null }, + ]); + + expect(Object.keys(manifest)).toEqual(['a', 'b']); + }); +}); diff --git a/scripts/migrate-protocols-to-v8.ts b/scripts/migrate-protocols-to-v8.ts index c21f7ea5d..611da3b68 100644 --- a/scripts/migrate-protocols-to-v8.ts +++ b/scripts/migrate-protocols-to-v8.ts @@ -2,6 +2,47 @@ import { hashProtocol, migrateProtocol } from '@codaco/protocol-validation'; import { Prisma } from '~/lib/db/generated/client'; +type ProtocolAssetRow = { + assetId: string; + name: string; + type: string; + value: string | null; +}; + +/** + * Rebuild a protocol's `assetManifest` from its linked Asset rows. Fresco stores + * assets in a separate table rather than inline on the protocol, but v8 validation + * resolves NameGeneratorRoster/Geospatial asset references against the manifest, so + * it must be present for migration to validate. The manifest is excluded from the + * protocol hash, so reconstructing it here does not affect the computed hash. + */ +export function buildAssetManifest(assets: ProtocolAssetRow[]) { + const manifest: Record< + string, + | { id: string; name: string; type: string; source: string } + | { id: string; name: string; type: 'apikey'; value: string } + > = {}; + + for (const asset of assets) { + manifest[asset.assetId] = + asset.type === 'apikey' + ? { + id: asset.assetId, + name: asset.name, + type: 'apikey', + value: asset.value ?? '', + } + : { + id: asset.assetId, + name: asset.name, + type: asset.type, + source: asset.name, + }; + } + + return manifest; +} + async function migrateOneProtocol( prisma: Prisma.TransactionClient, row: { @@ -10,6 +51,7 @@ async function migrateOneProtocol( schemaVersion: number; stages: unknown; codebook: unknown; + assets: ProtocolAssetRow[]; }, ): Promise { const cleanName = row.name.replace(/\.netcanvas$/i, ''); @@ -19,6 +61,7 @@ async function migrateOneProtocol( schemaVersion: row.schemaVersion, stages: row.stages, codebook: row.codebook, + assetManifest: buildAssetManifest(row.assets), }; let migrated: ReturnType; @@ -81,6 +124,9 @@ export async function migrateProtocolsToV8( schemaVersion: true, stages: true, codebook: true, + assets: { + select: { assetId: true, name: true, type: true, value: true }, + }, }, }); From 7e49760aeec209888bb8767976295c16a2e97aff Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Fri, 26 Jun 2026 16:23:02 -0400 Subject: [PATCH 4/5] refactor(interview): import contract utils from @codaco/interview/contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit createInitialNetwork and isValidAssetType were imported from the @codaco/interview package root, whose single bundle evaluates React createContext at module load and so crashes the RSC production build when pulled into server code (server action / server component). Import them instead from the new server-safe @codaco/interview/contract entry (complexdatacollective/network-canvas-monorepo#705), and drop the temporary local reimplementations so the factory stays single-sourced in the package. NOTE: blocked on publishing @codaco/interview@1.2.0 with the ./contract export. Until then the build/install is red (the subpath does not resolve against 1.1.0). Once published, refresh the lockfile (pnpm install) to go green — no code change. --- actions/__tests__/createInterview.test.ts | 2 +- actions/interviews.ts | 2 +- app/(interview)/interview/[interviewId]/mapInterviewPayload.ts | 2 +- package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/actions/__tests__/createInterview.test.ts b/actions/__tests__/createInterview.test.ts index cddce5992..c025cc799 100644 --- a/actions/__tests__/createInterview.test.ts +++ b/actions/__tests__/createInterview.test.ts @@ -87,7 +87,7 @@ vi.mock('~/lib/posthog-server', () => ({ shutdownPostHog: vi.fn(), })); -vi.mock('@codaco/interview', () => ({ +vi.mock('@codaco/interview/contract', () => ({ createInitialNetwork: vi.fn(() => ({ nodes: [], edges: [], diff --git a/actions/interviews.ts b/actions/interviews.ts index c143a124e..f4c5c6553 100644 --- a/actions/interviews.ts +++ b/actions/interviews.ts @@ -5,7 +5,7 @@ import { after } from 'next/server'; import { requireApiAuth } from '~/lib/auth/guards'; import { safeRevalidateTag, safeUpdateTag } from '~/lib/cache'; import { prisma } from '~/lib/db'; -import { createInitialNetwork } from '@codaco/interview'; +import { createInitialNetwork } from '@codaco/interview/contract'; import { captureException, shutdownPostHog } from '~/lib/posthog-server'; import { getAppSetting } from '~/queries/appSettings'; import { getInterviewIdsMatching } from '~/queries/interviews'; diff --git a/app/(interview)/interview/[interviewId]/mapInterviewPayload.ts b/app/(interview)/interview/[interviewId]/mapInterviewPayload.ts index 729b8799b..a4568bb78 100644 --- a/app/(interview)/interview/[interviewId]/mapInterviewPayload.ts +++ b/app/(interview)/interview/[interviewId]/mapInterviewPayload.ts @@ -2,7 +2,7 @@ import { isValidAssetType, type InterviewPayload, type ResolvedAsset, -} from '@codaco/interview'; +} from '@codaco/interview/contract'; import type { GetInterviewByIdQuery } from '~/queries/interviews'; export function mapInterviewPayload( diff --git a/package.json b/package.json index 7c4082754..8a25b21cc 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "@aws-sdk/s3-request-presigner": "^3.1068.0", "@base-ui/react": "^1.5.0", "@codaco/fresco-ui": "^2.14.0", - "@codaco/interview": "^1.1.0", + "@codaco/interview": "^1.2.0", "@codaco/network-exporters": "^1.1.0", "@codaco/protocol-utilities": "^2.0.0", "@codaco/protocol-validation": "^11.7.0", From 9be35493db254b71df8aaad3cf3f3cd8030397ea Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Fri, 26 Jun 2026 17:05:02 -0400 Subject: [PATCH 5/5] chore(deps): lock @codaco/interview to 1.2.0 (server-safe ./contract export) @codaco/interview@1.2.0 is published with the ./contract entry. Refresh the lockfile so the @codaco/interview/contract imports resolve, unblocking the RSC production build. --- pnpm-lock.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 71b8b7efa..fda2f6022 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,8 +27,8 @@ importers: specifier: ^2.14.0 version: 2.14.0(@base-ui/react@1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@codaco/shared-consts@5.3.0)(@codaco/tailwind-config@1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1))(@floating-ui/dom@1.7.6)(@tanstack/react-table@8.21.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@tiptap/extension-list@3.24.0(@tiptap/core@3.24.0(@tiptap/pm@3.24.0))(@tiptap/pm@3.24.0))(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(motion@12.40.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1)(typescript@6.0.3)(zod@4.4.3)(zustand@5.0.14(@types/react@19.2.17)(immer@11.1.8)(react@19.2.7)(use-sync-external-store@1.6.0(react@19.2.7))) '@codaco/interview': - specifier: ^1.1.0 - version: 1.1.0(1b402db64fdf29c167eb1ba29139ca9d) + specifier: ^1.2.0 + version: 1.2.0(1b402db64fdf29c167eb1ba29139ca9d) '@codaco/network-exporters': specifier: ^1.1.0 version: 1.1.0 @@ -588,8 +588,8 @@ packages: zod: ^4.4.3 zustand: ^5.0.14 - '@codaco/interview@1.1.0': - resolution: {integrity: sha512-Y+gqo7WVFhqI4UsHKh5mIQ4T42XmonBLcp94IQM08kCwfI+cHhRiA0oXdvMpygKUoz4/HY8I61Ai+b1BMjUGLA==} + '@codaco/interview@1.2.0': + resolution: {integrity: sha512-flUTBDbSAIrlACJzZPALf+2XE0NPJOVIS4zUOnD18nT1ET8eqEhQyXaOAFWhTwhBrGEvIrjF//e/jmSZmU9Orw==} peerDependencies: '@codaco/fresco-ui': ^2.14.0 '@codaco/protocol-validation': ^11.7.0 @@ -7526,7 +7526,7 @@ snapshots: - supports-color - typescript - '@codaco/interview@1.1.0(1b402db64fdf29c167eb1ba29139ca9d)': + '@codaco/interview@1.2.0(1b402db64fdf29c167eb1ba29139ca9d)': dependencies: '@base-ui/react': 1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) '@codaco/fresco-ui': 2.14.0(@base-ui/react@1.5.0(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@codaco/shared-consts@5.3.0)(@codaco/tailwind-config@1.0.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1))(@floating-ui/dom@1.7.6)(@tanstack/react-table@8.21.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(@tiptap/extension-list@3.24.0(@tiptap/core@3.24.0(@tiptap/pm@3.24.0))(@tiptap/pm@3.24.0))(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(motion@12.40.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(tailwindcss@4.3.1)(typescript@6.0.3)(zod@4.4.3)(zustand@5.0.14(@types/react@19.2.17)(immer@11.1.8)(react@19.2.7)(use-sync-external-store@1.6.0(react@19.2.7)))