fix/df-1631: Fix optionals within repeater on 'copy' form

src/server/routes/copy-form.js

@@ -172,7 +172,7 @@ export default [
           .takeover()
       }
 
-      await setCacheState(request, submissionToCopy, form)
+      await setCacheState(request, submissionToCopy, form, def)
 
       const formHref = getFormHref(
         form.slug,
@@ -276,7 +276,7 @@ export default [
       const { formVersionHasChanged } = getFormDetails(def, submissionToCopy)
 
       // Set state and dispatch user into the form
-      await setCacheState(request, submissionToCopy, form)
+      await setCacheState(request, submissionToCopy, form, def)
 
       // If the form version has changed, redirect to the form start page and walkthrough,
       // otherwise redirect to the summary page and allow the engine to handle the walkthrough
@@ -408,10 +408,11 @@ function getFormDetails(def, submission) {
  * @param {Request<{ Params: CopyFormParams }>} request
  * @param {FormSubmissionDocument} submissionToCopy
  * @param {FormMetadata} form
+ * @param {FormDefinition} definition
  */
-async function setCacheState(request, submissionToCopy, form) {
+async function setCacheState(request, submissionToCopy, form, definition) {
   const cacheService = getCacheService(request.server)
-  const copiedState = mapValueToState(submissionToCopy)
+  const copiedState = mapValueToState(submissionToCopy, definition)
   const state = getPreviewStatus(submissionToCopy)
   const req = { yar: request.yar, params: { slug: form.slug, state } }
 

src/server/utils/map-submission-to-state.js

@@ -1,5 +1,8 @@
 import { randomUUID } from 'node:crypto'
 
+import { createComponent } from '@defra/forms-engine-plugin/engine/components/helpers/components.js'
+import { hasFormComponents } from '@defra/forms-model'
+
 /**
  * Handles sub fields like address.addressLine1 and date.day
  * @param {Record<string, FormStateValue>} subfieldObject
@@ -44,8 +47,9 @@ export const isString = (value) => typeof value === 'string'
 /**
  * Turns a form submission document into data suitable for session state
  * @param {FormSubmissionDocument} formSubmissionMessage
+ * @param {FormDefinition} definition
  */
-export function mapValueToState(formSubmissionMessage) {
+export function mapValueToState(formSubmissionMessage, definition) {
   const mainEntries = Object.entries(formSubmissionMessage.data.main)
   const main = mainEntries.reduce(
     (acc, entry) => handleSubfields(acc, entry),
@@ -73,11 +77,62 @@ export function mapValueToState(formSubmissionMessage) {
 
   return {
     ...main,
-    ...repeaters
+    // Rehydrate any missing null-value fields from repeaters
+    ...rehydrateRepeaters(repeaters, definition)
+  }
+}
+
+/**
+ * Rehydrates repeater fields with missing null-value fields based on the form definition.
+ * These are required (even though they may be option fields) for repeater validation to work correctly.
+ * @param {Record<string, Record<string, RichFormValue>[]>} repeaters
+ * @param {FormDefinition} definition
+ */
+export function rehydrateRepeaters(repeaters, definition) {
+  const repeaterPages = definition.pages.filter(
+    (p) => hasFormComponents(p) && 'repeat' in p
+  )
+
+  /** @type {Record<string, Record<string, RichFormValue | null>[]>} */
+  const rehydrated = { ...repeaters }
+
+  for (const page of repeaterPages) {
+    // @ts-expect-error: page.repeat may be undefined, but we handle it with optional chaining
+    const name = page.repeat?.options.name
+    const items = name ? repeaters[name] : undefined
+
+    if (!name || !Array.isArray(items)) {
+      continue
+    }
+
+    const components = page.components
+
+    rehydrated[name] = items.map((item) => {
+      /** @type {Record<string, RichFormValue | null>} */
+      const rehydratedItem = { ...item }
+
+      for (const component of components) {
+        const field = createComponent(component, {
+          // @ts-expect-error: partial implementation of model as extra data not required for simple component creation
+          model: { getList: () => undefined }
+        })
+        const fieldNames = (field.collection?.fields ?? [field]).map(
+          (f) => f.name
+        )
+        for (const fieldName of fieldNames) {
+          rehydratedItem[fieldName] ??= null
+        }
+      }
+
+      return rehydratedItem
+    })
   }
+
+  return rehydrated
 }
 
 /**
+ * @import { FormDefinition } from '@defra/forms-model'
  * @import { RichFormValue, FormStateValue } from '@defra/forms-engine-plugin/engine/types.js'
  * @import { FormSubmissionDocument } from '~/src/server/services/submissionService.js'
  */

src/server/utils/map-submission-to-state.test.js

@@ -1,10 +1,19 @@
 import { FormStatus } from '@defra/forms-engine-plugin/types'
+import { ComponentType } from '@defra/forms-model'
 
 import {
   isString,
-  mapValueToState
+  mapValueToState,
+  rehydrateRepeaters
 } from '~/src/server/utils/map-submission-to-state.js'
 
+const emptyDefinition = /** @type {FormDefinition} */ ({
+  pages: [],
+  lists: [],
+  conditions: [],
+  sections: []
+})
+
 describe('mapValueToState', () => {
   it('should map adaptor v2 message to state2', () => {
     /**
@@ -308,7 +317,7 @@ describe('mapValueToState', () => {
       expireAt: new Date('2028-05-02T12:23:28.461Z')
     }
 
-    expect(mapValueToState(message)).toEqual({
+    expect(mapValueToState(message, emptyDefinition)).toEqual({
       textField: 'Bob',
       multilineTextField: 'Blah',
       numberField: 2,
@@ -569,7 +578,7 @@ describe('mapValueToState', () => {
     }
 
     // @ts-expect-error - exercises omission of undefined values in a malformed submission
-    expect(mapValueToState(message)).toEqual({
+    expect(mapValueToState(message, emptyDefinition)).toEqual({
       compound__defined: 'value',
       list: ['item'],
       empty: null,
@@ -590,3 +599,208 @@ describe('mapValueToState', () => {
     expect(isString(null)).toBe(false)
   })
 })
+
+describe('rehydrateRepeaters', () => {
+  /**
+   * @param {object[]} components
+   * @param {string | undefined} repeatName
+   */
+  const repeatPage = (components, repeatName = 'people') => ({
+    title: 'Repeater page',
+    path: '/people',
+    components,
+    next: [],
+    repeat: {
+      options: { name: repeatName, title: 'Person' },
+      schema: { min: 1, max: 5 }
+    }
+  })
+
+  /**
+   * @function textComponent
+   * @param {string} name
+   * @returns {object}
+   */
+  const textComponent = (name) => ({
+    id: `id-${name}`,
+    type: ComponentType.TextField,
+    name,
+    title: name,
+    options: {},
+    schema: {}
+  })
+
+  /**
+   * @param {object[]} pages
+   * @returns {FormDefinition}
+   */
+  const definitionWith = (pages) =>
+    /** @type {FormDefinition} */ (
+      /** @type {unknown} */ ({
+        name: 'Test',
+        pages,
+        lists: [],
+        sections: [],
+        conditions: []
+      })
+    )
+
+  it('should add null for missing component values in each repeater item', () => {
+    const definition = definitionWith([
+      repeatPage([textComponent('sTUBVF'), textComponent('pUcoFr')])
+    ])
+
+    expect(
+      rehydrateRepeaters(
+        { people: [{ sTUBVF: 'SO 1111 1111' }, { pUcoFr: 'x' }, {}] },
+        definition
+      )
+    ).toEqual({
+      people: [
+        { sTUBVF: 'SO 1111 1111', pUcoFr: null },
+        { sTUBVF: null, pUcoFr: 'x' },
+        { sTUBVF: null, pUcoFr: null }
+      ]
+    })
+  })
+
+  it('should not overwrite existing values, including falsy ones', () => {
+    const definition = definitionWith([
+      repeatPage([textComponent('a'), textComponent('b'), textComponent('c')])
+    ])
+
+    expect(
+      rehydrateRepeaters({ people: [{ a: '', b: 0, c: false }] }, definition)
+    ).toEqual({ people: [{ a: '', b: 0, c: false }] })
+  })
+
+  it('should keep an explicit null value as null', () => {
+    const definition = definitionWith([repeatPage([textComponent('a')])])
+
+    // @ts-expect-error: explicit null value is allowed
+    expect(rehydrateRepeaters({ people: [{ a: null }] }, definition)).toEqual({
+      people: [{ a: null }]
+    })
+  })
+
+  it('should preserve extra keys such as itemId', () => {
+    const definition = definitionWith([repeatPage([textComponent('a')])])
+
+    expect(
+      rehydrateRepeaters({ people: [{ itemId: 'abc' }] }, definition)
+    ).toEqual({ people: [{ itemId: 'abc', a: null }] })
+  })
+
+  it('should expand composite components into their sub field names', () => {
+    const definition = definitionWith([
+      repeatPage([
+        {
+          id: 'date-id',
+          type: ComponentType.DatePartsField,
+          name: 'dob',
+          title: 'Date of birth',
+          options: {},
+          schema: {}
+        }
+      ])
+    ])
+
+    const [item] = rehydrateRepeaters({ people: [{}] }, definition).people
+
+    expect(Object.keys(item).sort()).toEqual([
+      'dob__day',
+      'dob__month',
+      'dob__year'
+    ])
+    expect(Object.values(item).every((v) => v === null)).toBe(true)
+  })
+
+  it('should rehydrate multiple repeater pages independently', () => {
+    const definition = definitionWith([
+      repeatPage([textComponent('a')], 'first'),
+      repeatPage([textComponent('b')], 'second')
+    ])
+
+    expect(
+      rehydrateRepeaters({ first: [{}], second: [{ b: 'x' }, {}] }, definition)
+    ).toEqual({
+      first: [{ a: null }],
+      second: [{ b: 'x' }, { b: null }]
+    })
+  })
+
+  it('should ignore non-repeater pages and leave unrelated state untouched', () => {
+    const definition = definitionWith([
+      {
+        title: 'Plain',
+        path: '/plain',
+        components: [textComponent('plain')],
+        next: []
+      },
+      repeatPage([textComponent('a')])
+    ])
+    const other = [{ keep: 'me' }]
+
+    expect(rehydrateRepeaters({ people: [{}], other }, definition)).toEqual({
+      people: [{ a: null }],
+      other: [{ keep: 'me' }]
+    })
+  })
+
+  it('should skip repeater pages with no matching or non-array data', () => {
+    const definition = definitionWith([
+      repeatPage([textComponent('a')], 'missing'),
+      repeatPage([textComponent('b')], 'notArray')
+    ])
+    const repeaters = { notArray: 'value' }
+
+    // @ts-expect-error - exercises malformed repeater data
+    expect(rehydrateRepeaters(repeaters, definition)).toEqual({
+      notArray: 'value'
+    })
+  })
+
+  it('should return empty arrays unchanged', () => {
+    const definition = definitionWith([repeatPage([textComponent('a')])])
+
+    expect(rehydrateRepeaters({ people: [] }, definition)).toEqual({
+      people: []
+    })
+  })
+
+  it('should not mutate the input repeaters', () => {
+    const definition = definitionWith([repeatPage([textComponent('a')])])
+    const repeaters = { people: [{ x: 1 }] }
+
+    const result = rehydrateRepeaters(repeaters, definition)
+
+    expect(result).not.toBe(repeaters)
+    expect(repeaters).toEqual({ people: [{ x: 1 }] })
+  })
+
+  it('should return the repeaters as-is when there are no repeater pages', () => {
+    expect(
+      rehydrateRepeaters({ people: [{ a: 1 }] }, definitionWith([]))
+    ).toEqual({ people: [{ a: 1 }] })
+  })
+
+  it('should be applied by mapValueToState for missing repeater fields', () => {
+    const definition = definitionWith([
+      repeatPage([textComponent('sTUBVF'), textComponent('pUcoFr')])
+    ])
+    const message = {
+      data: { main: {}, repeaters: { people: [{ sTUBVF: 'SO 1111 1111' }] } }
+    }
+
+    // @ts-expect-error - partial submission document
+    expect(mapValueToState(message, definition)).toEqual({
+      people: [
+        { sTUBVF: 'SO 1111 1111', pUcoFr: null, itemId: expect.any(String) }
+      ]
+    })
+  })
+})
+
+/**
+ * @import { FormDefinition } from '@defra/forms-model'
+ */