Detect and expand JSON-serialized nested columns

.changeset/expand-nested-json-columns.md

@@ -0,0 +1,6 @@
+---
+"@jspsych/metadata": minor
+"@jspsych/metadata-cli": minor
+---
+
+Detect and expand JSON-serialized nested columns in `generate()`. Flat JSON objects (e.g. `response: {"Q0":4,"Q1":3}`) are expanded into dotted sub-variables (`response.Q0`, `response.Q1`) in `variableMeasured` with correct types and min/max tracking. JSON arrays of objects are extracted into separate Psych-DS compliant CSV files (`{stem}_measure-{col}_data.csv`) with `trial_index` and `element_index` as join keys.

packages/cli/src/data.ts

@@ -1,7 +1,7 @@
 import fs from "fs";
 import path from "path";
 import JsPsychMetadata from "@jspsych/metadata";
-import { expandHomeDir } from "./utils";
+import { expandHomeDir, deriveArrayFilename, objectsToCSV } from "./utils";
 
 // creating path -> handles the absolute vs non-absolute paths
 export const generatePath = (inputPath: string): string => {
@@ -54,7 +54,18 @@ const processFile = async (metadata: JsPsychMetadata, directoryPath: string, fil
         return false;
     }
 
-    if (targetDirectoryPath) await copyFileWithStructure(filePath, verbose, targetDirectoryPath); // error catching to create backwards compability with CLI and old cli prompting
+    if (targetDirectoryPath) {
+      await copyFileWithStructure(filePath, verbose, targetDirectoryPath);
+
+      // Write a separate Psych-DS CSV for each array-of-objects column detected during generate()
+      const extractedArrays = metadata.getExtractedArrays();
+      for (const [colName, rows] of extractedArrays) {
+        const outFilename = deriveArrayFilename(file, colName);
+        const outPath = path.join(targetDirectoryPath, outFilename);
+        await fs.promises.writeFile(outPath, objectsToCSV(rows), 'utf8');
+        if (verbose) console.log(`  → wrote array data for "${colName}" to ${outPath}`);
+      }
+    }
   } catch (err) {
     console.error(`Error reading file ${file}: ${err} Please ensure this is data generated by JsPsych.`);
     return false;

packages/cli/src/utils.ts

@@ -7,4 +7,69 @@ export function expandHomeDir(directoryPath: string): string {
     return path.join(homeDir, directoryPath.slice(1));
   }
   return directoryPath;
+}
+
+/**
+ * Converts a column name or file stem to a Psych-DS safe value segment:
+ * lowercased; spaces, underscores, periods, and any other non-alphanumeric-hyphen
+ * characters replaced with hyphens; runs of hyphens collapsed to one.
+ */
+export function sanitizePsychDSSegment(name: string): string {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9-]+/g, '-')
+    .replace(/-{2,}/g, '-')
+    .replace(/^-|-$/g, '');
+}
+
+/**
+ * Derives the Psych-DS compliant output filename for a separate array CSV.
+ * Pattern: strip extension, strip trailing _data, sanitize, append _measure-{col}_data.csv
+ *
+ * "measure" is used as the key (rather than a structural label like "column") to align
+ * with the Psych-DS / schema.org convention of describing data by what is measured.
+ *
+ * Examples:
+ *   participant-001_session-1_data.csv + mouse_tracking_data
+ *     → participant-001_session-1_measure-mouse-tracking-data_data.csv
+ *   Keyboard Response.csv + response
+ *     → keyboard-response_measure-response_data.csv
+ *   stem + validation_data.pointData
+ *     → stem_measure-validation-data-pointdata_data.csv
+ */
+export function deriveArrayFilename(sourceFile: string, columnName: string): string {
+  const stem = path.basename(sourceFile, path.extname(sourceFile));
+  const withoutData = stem.replace(/_data$/i, '');
+  const safeStem = sanitizePsychDSSegment(withoutData);
+  const safeCol = sanitizePsychDSSegment(columnName);
+  return `${safeStem}_measure-${safeCol}_data.csv`;
+}
+
+/**
+ * Serialises an array of flat objects to RFC 4180 CSV.
+ * trial_index and element_index columns are placed first; remaining columns
+ * follow in the order they first appear across all rows.
+ */
+export function objectsToCSV(rows: Array<Record<string, any>>): string {
+  if (rows.length === 0) return '';
+
+  const priorityCols = ['trial_index', 'element_index'];
+  const allKeys = new Set<string>();
+  for (const row of rows) {
+    for (const key of Object.keys(row)) allKeys.add(key);
+  }
+  const otherCols = [...allKeys].filter(k => !priorityCols.includes(k));
+  const headers = [...priorityCols.filter(c => allKeys.has(c)), ...otherCols];
+
+  const escape = (val: any): string => {
+    const str = val === null || val === undefined ? '' : String(val);
+    return str.includes(',') || str.includes('"') || str.includes('\n')
+      ? `"${str.replace(/"/g, '""')}"` : str;
+  };
+
+  const lines = [headers.join(',')];
+  for (const row of rows) {
+    lines.push(headers.map(h => escape(row[h])).join(','));
+  }
+  return lines.join('\r\n');
 }

packages/metadata/src/index.ts

@@ -1,6 +1,6 @@
 import { AuthorFields, AuthorsMap } from "./AuthorsMap";
 import { PluginCache } from "./PluginCache";
-import { saveTextToFile, parseCSV } from "./utils";
+import { saveTextToFile, parseCSV, tryParseJSON } from "./utils";
 import { VariableFields, VariablesMap } from "./VariablesMap";
 
 /**
@@ -68,6 +68,8 @@ export default class JsPsychMetadata {
    */
   private verbose: boolean = false;
 
+  private extractedArrays: Map<string, Array<Record<string, any>>> = new Map();
+
   /**
    * Creates an instance of JsPsychMetadata while passing in JsPsych object to have access to context
    *  allowing it to access the screen printing information.
@@ -309,6 +311,15 @@ export default class JsPsychMetadata {
     return this.variables.getVariableNames();
   }
 
+  /**
+   * Returns accumulated array-column data keyed by column name.
+   * Each entry is a list of rows with trial_index, element_index, and the element's own fields.
+   * Used by the CLI to write Psych-DS compliant separate CSV files.
+   */
+  getExtractedArrays(): Map<string, Array<Record<string, any>>> {
+    return this.extractedArrays;
+  }
+
   /**
    * Method that allows you to display metadata at the end of an experiment.
    *
@@ -423,7 +434,7 @@ export default class JsPsychMetadata {
 
     for (const variable in observation) {
       var value = observation[variable];
-      var type = typeof value;
+      var type: string = typeof value;
 
       if (value === null || value === undefined || value === '' || value === "null"){ 
         continue; // Error checking
@@ -437,11 +448,41 @@ export default class JsPsychMetadata {
         } else if (value.toLowerCase() === "true" || value.toLowerCase() === "false") {
           type = "boolean";
           value = (value.toLowerCase() === "true");
+        } else if (value.startsWith("{") || value.startsWith("[")) {
+          const parsed = tryParseJSON(value);
+          if (parsed !== null) {
+            value = parsed;
+            type = Array.isArray(parsed) ? "array" : "object";
+          }
         }
       }
 
       if (this.ignored_variables.has(variable)) this.updateFields(variable, value, type);
-      else {
+      else if (type === "object" && value !== null && !Array.isArray(value)) {
+        await this.expandObjectFields(variable, value, pluginType, version);
+      } else if (type === "array" || (type === "object" && Array.isArray(value))) {
+        // Register parent via generateMetadata to get the plugin description, then
+        // override the stored type to "array" (generateMetadata would infer "object"
+        // because typeof [] === "object" in JS).
+        await this.generateMetadata(variable, value, pluginType, version);
+        this.updateVariable(variable, "value", "array");
+
+        // Accumulate array-of-objects rows for separate CSV output.
+        // Only object elements are expanded; null / primitive elements are skipped.
+        const objectElements = (value as any[]).filter(
+          (el) => el !== null && typeof el === "object" && !Array.isArray(el)
+        );
+        if (objectElements.length > 0) {
+          const trialIndex = observation["trial_index"];
+          const existing = this.extractedArrays.get(variable) ?? [];
+          (value as any[]).forEach((element, elementIndex) => {
+            if (element !== null && typeof element === "object" && !Array.isArray(element)) {
+              existing.push({ trial_index: trialIndex, element_index: elementIndex, ...element });
+            }
+          });
+          this.extractedArrays.set(variable, existing);
+        }
+      } else {
         await this.generateMetadata(variable, value, pluginType, version);
 
         if (extensionType) {
@@ -596,6 +637,21 @@ export default class JsPsychMetadata {
     } else this.setMetadataField(key, value);
   }
 
+  /**
+   * Registers the top-level keys of a plain JSON object as dotted sub-variables
+   * (e.g. response.Q0, response.Q1) and registers the parent with value: "object".
+   * One level deep only.
+   */
+  private async expandObjectFields(parentName: string, obj: Record<string, any>, pluginType: string, version: string) {
+    // Register the parent through the normal path so the plugin description is fetched.
+    // typeof obj === "object", so generateMetadata stores value: "object" and updateFields
+    // skips levels automatically — no override needed.
+    await this.generateMetadata(parentName, obj, pluginType, version);
+    for (const key of Object.keys(obj)) {
+      await this.generateMetadata(`${parentName}.${key}`, obj[key], pluginType, version);
+    }
+  }
+
   /**
    * Gets the description of a variable in a plugin by fetching the source code of the plugin
    * from a remote source (usually unpkg.com) as a string, passing the script to getJsdocsDescription

packages/metadata/src/utils.ts

@@ -62,6 +62,14 @@ export function JSON2CSV(objArray) {
   return result;
 }
 
+export function tryParseJSON(value: string): any | null {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return null;
+  }
+}
+
 export async function parseCSV(input) {''
   if (!parse) {
     throw new Error('Parser module not loaded');