From 8b77cd66c77595be5c99827f1210a14ce7957f4c Mon Sep 17 00:00:00 2001 From: Mandyx22 <1915537307@qq.com> Date: Fri, 29 May 2026 14:34:58 -0400 Subject: [PATCH 1/3] Detect and expand JSON-serialized nested columns in generate() Flat JSON objects (e.g. response: {Q0:4, Q1:3}) are now expanded into dotted sub-variables (response.Q0, response.Q1) in variableMeasured with correct types and min/max tracking. JSON arrays of objects are extracted into separate Psych-DS compliant CSV files ({stem}_measure-{col}_data.csv) with trial_index and element_index as join keys. Co-Authored-By: Claude Sonnet 4.6 --- packages/cli/src/data.ts | 15 +- packages/cli/src/utils.ts | 65 +++++++ packages/metadata/src/index.ts | 60 +++++- packages/metadata/src/utils.ts | 8 + .../tests/metadata-nested-columns.test.ts | 183 ++++++++++++++++++ 5 files changed, 327 insertions(+), 4 deletions(-) create mode 100644 packages/metadata/tests/metadata-nested-columns.test.ts diff --git a/packages/cli/src/data.ts b/packages/cli/src/data.ts index b0993be..87ef170 100644 --- a/packages/cli/src/data.ts +++ b/packages/cli/src/data.ts @@ -1,7 +1,7 @@ import fs from "fs"; import path from "path"; import JsPsychMetadata from "@jspsych/metadata"; -import { expandHomeDir } from "./utils"; +import { expandHomeDir, deriveArrayFilename, objectsToCSV } from "./utils"; // creating path -> handles the absolute vs non-absolute paths export const generatePath = (inputPath: string): string => { @@ -54,7 +54,18 @@ const processFile = async (metadata: JsPsychMetadata, directoryPath: string, fil return false; } - if (targetDirectoryPath) await copyFileWithStructure(filePath, verbose, targetDirectoryPath); // error catching to create backwards compability with CLI and old cli prompting + if (targetDirectoryPath) { + await copyFileWithStructure(filePath, verbose, targetDirectoryPath); + + // Write a separate Psych-DS CSV for each array-of-objects column detected during generate() + const extractedArrays = metadata.getExtractedArrays(); + for (const [colName, rows] of extractedArrays) { + const outFilename = deriveArrayFilename(file, colName); + const outPath = path.join(targetDirectoryPath, outFilename); + await fs.promises.writeFile(outPath, objectsToCSV(rows), 'utf8'); + if (verbose) console.log(` → wrote array data for "${colName}" to ${outPath}`); + } + } } catch (err) { console.error(`Error reading file ${file}: ${err} Please ensure this is data generated by JsPsych.`); return false; diff --git a/packages/cli/src/utils.ts b/packages/cli/src/utils.ts index d4e80ac..0853a7e 100644 --- a/packages/cli/src/utils.ts +++ b/packages/cli/src/utils.ts @@ -7,4 +7,69 @@ export function expandHomeDir(directoryPath: string): string { return path.join(homeDir, directoryPath.slice(1)); } return directoryPath; +} + +/** + * Converts a column name or file stem to a Psych-DS safe value segment: + * lowercased; spaces, underscores, periods, and any other non-alphanumeric-hyphen + * characters replaced with hyphens; runs of hyphens collapsed to one. + */ +export function sanitizePsychDSSegment(name: string): string { + return name + .toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-{2,}/g, '-') + .replace(/^-|-$/g, ''); +} + +/** + * Derives the Psych-DS compliant output filename for a separate array CSV. + * Pattern: strip extension, strip trailing _data, sanitize, append _measure-{col}_data.csv + * + * "measure" is used as the key (rather than a structural label like "column") to align + * with the Psych-DS / schema.org convention of describing data by what is measured. + * + * Examples: + * participant-001_session-1_data.csv + mouse_tracking_data + * → participant-001_session-1_measure-mouse-tracking-data_data.csv + * Keyboard Response.csv + response + * → keyboard-response_measure-response_data.csv + * stem + validation_data.pointData + * → stem_measure-validation-data-pointdata_data.csv + */ +export function deriveArrayFilename(sourceFile: string, columnName: string): string { + const stem = path.basename(sourceFile, path.extname(sourceFile)); + const withoutData = stem.replace(/_data$/i, ''); + const safeStem = sanitizePsychDSSegment(withoutData); + const safeCol = sanitizePsychDSSegment(columnName); + return `${safeStem}_measure-${safeCol}_data.csv`; +} + +/** + * Serialises an array of flat objects to RFC 4180 CSV. + * trial_index and element_index columns are placed first; remaining columns + * follow in the order they first appear across all rows. + */ +export function objectsToCSV(rows: Array>): string { + if (rows.length === 0) return ''; + + const priorityCols = ['trial_index', 'element_index']; + const allKeys = new Set(); + for (const row of rows) { + for (const key of Object.keys(row)) allKeys.add(key); + } + const otherCols = [...allKeys].filter(k => !priorityCols.includes(k)); + const headers = [...priorityCols.filter(c => allKeys.has(c)), ...otherCols]; + + const escape = (val: any): string => { + const str = val === null || val === undefined ? '' : String(val); + return str.includes(',') || str.includes('"') || str.includes('\n') + ? `"${str.replace(/"/g, '""')}"` : str; + }; + + const lines = [headers.join(',')]; + for (const row of rows) { + lines.push(headers.map(h => escape(row[h])).join(',')); + } + return lines.join('\r\n'); } \ No newline at end of file diff --git a/packages/metadata/src/index.ts b/packages/metadata/src/index.ts index a81a30e..510d3b7 100644 --- a/packages/metadata/src/index.ts +++ b/packages/metadata/src/index.ts @@ -1,6 +1,6 @@ import { AuthorFields, AuthorsMap } from "./AuthorsMap"; import { PluginCache } from "./PluginCache"; -import { saveTextToFile, parseCSV } from "./utils"; +import { saveTextToFile, parseCSV, tryParseJSON } from "./utils"; import { VariableFields, VariablesMap } from "./VariablesMap"; /** @@ -68,6 +68,8 @@ export default class JsPsychMetadata { */ private verbose: boolean = false; + private extractedArrays: Map>> = new Map(); + /** * Creates an instance of JsPsychMetadata while passing in JsPsych object to have access to context * allowing it to access the screen printing information. @@ -309,6 +311,15 @@ export default class JsPsychMetadata { return this.variables.getVariableNames(); } + /** + * Returns accumulated array-column data keyed by column name. + * Each entry is a list of rows with trial_index, element_index, and the element's own fields. + * Used by the CLI to write Psych-DS compliant separate CSV files. + */ + getExtractedArrays(): Map>> { + return this.extractedArrays; + } + /** * Method that allows you to display metadata at the end of an experiment. * @@ -437,11 +448,41 @@ export default class JsPsychMetadata { } else if (value.toLowerCase() === "true" || value.toLowerCase() === "false") { type = "boolean"; value = (value.toLowerCase() === "true"); + } else if (value.startsWith("{") || value.startsWith("[")) { + const parsed = tryParseJSON(value); + if (parsed !== null) { + value = parsed; + type = Array.isArray(parsed) ? "array" : "object"; + } } } if (this.ignored_variables.has(variable)) this.updateFields(variable, value, type); - else { + else if (type === "object" && value !== null && !Array.isArray(value)) { + await this.expandObjectFields(variable, value, pluginType, version); + } else if (type === "array" || (type === "object" && Array.isArray(value))) { + // Register parent via generateMetadata to get the plugin description, then + // override the stored type to "array" (generateMetadata would infer "object" + // because typeof [] === "object" in JS). + await this.generateMetadata(variable, value, pluginType, version); + this.updateVariable(variable, "value", "array"); + + // Accumulate array-of-objects rows for separate CSV output. + // Only object elements are expanded; null / primitive elements are skipped. + const objectElements = (value as any[]).filter( + (el) => el !== null && typeof el === "object" && !Array.isArray(el) + ); + if (objectElements.length > 0) { + const trialIndex = observation["trial_index"]; + const existing = this.extractedArrays.get(variable) ?? []; + (value as any[]).forEach((element, elementIndex) => { + if (element !== null && typeof element === "object" && !Array.isArray(element)) { + existing.push({ trial_index: trialIndex, element_index: elementIndex, ...element }); + } + }); + this.extractedArrays.set(variable, existing); + } + } else { await this.generateMetadata(variable, value, pluginType, version); if (extensionType) { @@ -596,6 +637,21 @@ export default class JsPsychMetadata { } else this.setMetadataField(key, value); } + /** + * Registers the top-level keys of a plain JSON object as dotted sub-variables + * (e.g. response.Q0, response.Q1) and registers the parent with value: "object". + * One level deep only. + */ + private async expandObjectFields(parentName: string, obj: Record, pluginType: string, version: string) { + // Register the parent through the normal path so the plugin description is fetched. + // typeof obj === "object", so generateMetadata stores value: "object" and updateFields + // skips levels automatically — no override needed. + await this.generateMetadata(parentName, obj, pluginType, version); + for (const key of Object.keys(obj)) { + await this.generateMetadata(`${parentName}.${key}`, obj[key], pluginType, version); + } + } + /** * Gets the description of a variable in a plugin by fetching the source code of the plugin * from a remote source (usually unpkg.com) as a string, passing the script to getJsdocsDescription diff --git a/packages/metadata/src/utils.ts b/packages/metadata/src/utils.ts index 63a2400..2cccaec 100644 --- a/packages/metadata/src/utils.ts +++ b/packages/metadata/src/utils.ts @@ -62,6 +62,14 @@ export function JSON2CSV(objArray) { return result; } +export function tryParseJSON(value: string): any | null { + try { + return JSON.parse(value); + } catch { + return null; + } +} + export async function parseCSV(input) {'' if (!parse) { throw new Error('Parser module not loaded'); diff --git a/packages/metadata/tests/metadata-nested-columns.test.ts b/packages/metadata/tests/metadata-nested-columns.test.ts new file mode 100644 index 0000000..192004f --- /dev/null +++ b/packages/metadata/tests/metadata-nested-columns.test.ts @@ -0,0 +1,183 @@ +import JsPsychMetadata from "../src/index"; + +const MOCK_PLUGIN_SOURCE = ` + const info = { + data: { + /** The participant's response */ + response: { + type: ParameterType.OBJECT, + }, + /** Reaction time in milliseconds */ + rt: { + type: ParameterType.INT, + }, + /** Mouse tracking data */ + mouse_tracking_data: { + type: ParameterType.OBJECT, + }, + }; + } +`; + +const mockFetch = jest.fn().mockResolvedValue({ + text: () => Promise.resolve(MOCK_PLUGIN_SOURCE), +}); + +const BASE = { trial_type: "mock-plugin", trial_index: 0, time_elapsed: 100 }; + +describe("Nested JSON column handling", () => { + beforeEach(() => { + (global as any).fetch = mockFetch; + mockFetch.mockClear(); + }); + + // ─── Case 1: flat JSON objects ─────────────────────────────────────────────── + + describe("flat JSON object columns", () => { + test("JSON input: sub-variables are registered with dotted names", async () => { + const data = JSON.stringify([ + { ...BASE, response: { Q0: 4, Q1: 3 } }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + expect(meta.getVariableNames()).toContain("response.Q0"); + expect(meta.getVariableNames()).toContain("response.Q1"); + }); + + test("CSV input: JSON object string is detected and expanded", async () => { + const csv = [ + "trial_type,trial_index,time_elapsed,response", + 'mock-plugin,0,100,"{""Q0"":4,""Q1"":3}"', + ].join("\n"); + const meta = new JsPsychMetadata(); + await meta.generate(csv, {}, "csv"); + + expect(meta.getVariableNames()).toContain("response.Q0"); + expect(meta.getVariableNames()).toContain("response.Q1"); + }); + + test("parent variable is registered with value: 'object' and no levels", async () => { + const data = JSON.stringify([ + { ...BASE, response: { Q0: 4, Q1: 3 } }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + expect(meta.getVariableNames()).toContain("response"); + const parent = meta.getVariable("response") as any; + expect(parent.value).toBe("object"); + expect(parent.levels).toBeUndefined(); + }); + + test("numeric sub-fields get minValue and maxValue tracked", async () => { + const data = JSON.stringify([ + { ...BASE, response: { rt: 300 } }, + { ...BASE, trial_index: 1, time_elapsed: 200, response: { rt: 700 } }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + const v = meta.getVariable("response.rt") as any; + expect(v.value).toBe("number"); + expect(v.minValue).toBe(300); + expect(v.maxValue).toBe(700); + }); + + test("null rows do not prevent sub-variables from being registered from other rows", async () => { + const data = JSON.stringify([ + { ...BASE, response: null }, + { ...BASE, trial_index: 1, time_elapsed: 200, response: { Q0: 1 } }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + expect(meta.getVariableNames()).toContain("response.Q0"); + }); + + test("plain string column is unchanged — still accumulates levels", async () => { + const data = JSON.stringify([ + { ...BASE, response: "yes" }, + { ...BASE, trial_index: 1, time_elapsed: 200, response: "no" }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + const v = meta.getVariable("response") as any; + expect(v.value).toBe("string"); + expect(v.levels).toContain("yes"); + expect(v.levels).toContain("no"); + }); + }); + + // ─── Case 2: JSON arrays of objects ───────────────────────────────────────── + + describe("JSON array-of-objects columns", () => { + test("parent variable is registered with value: 'array' and no levels", async () => { + const data = JSON.stringify([ + { ...BASE, mouse_tracking_data: [{ x: 120, y: 340, t: 0 }, { x: 121, y: 338, t: 16 }] }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + expect(meta.getVariableNames()).toContain("mouse_tracking_data"); + const v = meta.getVariable("mouse_tracking_data") as any; + expect(v.value).toBe("array"); + expect(v.levels).toBeUndefined(); + }); + + test("getExtractedArrays returns rows with trial_index and element_index", async () => { + const data = JSON.stringify([ + { ...BASE, trial_index: 3, mouse_tracking_data: [{ x: 120, y: 340, t: 0 }, { x: 121, y: 338, t: 16 }] }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + const arrays = meta.getExtractedArrays(); + expect(arrays.has("mouse_tracking_data")).toBe(true); + + const rows = arrays.get("mouse_tracking_data")!; + expect(rows).toHaveLength(2); + expect(rows[0]).toMatchObject({ trial_index: 3, element_index: 0, x: 120, y: 340, t: 0 }); + expect(rows[1]).toMatchObject({ trial_index: 3, element_index: 1, x: 121, y: 338, t: 16 }); + }); + + test("null elements within an array are skipped without error", async () => { + const data = JSON.stringify([ + { ...BASE, mouse_tracking_data: [{ x: 1 }, null, { x: 2 }] }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + const rows = meta.getExtractedArrays().get("mouse_tracking_data")!; + expect(rows).toHaveLength(2); + expect(rows[0].x).toBe(1); + expect(rows[1].x).toBe(2); + }); + + test("primitive array [1,2,3] falls through to normal handling — no extracted arrays entry", async () => { + const data = JSON.stringify([ + { ...BASE, scores: [1, 2, 3] }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + expect(meta.getExtractedArrays().has("scores")).toBe(false); + }); + + test("rows from multiple trials are all accumulated under the same column key", async () => { + const data = JSON.stringify([ + { ...BASE, trial_index: 0, mouse_tracking_data: [{ x: 1 }] }, + { ...BASE, trial_index: 1, time_elapsed: 200, mouse_tracking_data: [{ x: 2 }, { x: 3 }] }, + ]); + const meta = new JsPsychMetadata(); + await meta.generate(data); + + const rows = meta.getExtractedArrays().get("mouse_tracking_data")!; + expect(rows).toHaveLength(3); + expect(rows[0]).toMatchObject({ trial_index: 0, element_index: 0, x: 1 }); + expect(rows[1]).toMatchObject({ trial_index: 1, element_index: 0, x: 2 }); + expect(rows[2]).toMatchObject({ trial_index: 1, element_index: 1, x: 3 }); + }); + }); +}); From a5af08c075d00018e01010412b626821b0f47765 Mon Sep 17 00:00:00 2001 From: Mandyx22 <1915537307@qq.com> Date: Fri, 29 May 2026 14:45:20 -0400 Subject: [PATCH 2/3] Add changeset for nested JSON column expansion Co-Authored-By: Claude Sonnet 4.6 --- .changeset/expand-nested-json-columns.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/expand-nested-json-columns.md diff --git a/.changeset/expand-nested-json-columns.md b/.changeset/expand-nested-json-columns.md new file mode 100644 index 0000000..07a676b --- /dev/null +++ b/.changeset/expand-nested-json-columns.md @@ -0,0 +1,6 @@ +--- +"@jspsych/metadata": minor +"@jspsych/metadata-cli": minor +--- + +Detect and expand JSON-serialized nested columns in `generate()`. Flat JSON objects (e.g. `response: {"Q0":4,"Q1":3}`) are expanded into dotted sub-variables (`response.Q0`, `response.Q1`) in `variableMeasured` with correct types and min/max tracking. JSON arrays of objects are extracted into separate Psych-DS compliant CSV files (`{stem}_measure-{col}_data.csv`) with `trial_index` and `element_index` as join keys. From 75e37f9d7fa90d11a6c05cbf67c27ac1bb247db1 Mon Sep 17 00:00:00 2001 From: Mandyx22 <1915537307@qq.com> Date: Fri, 29 May 2026 15:01:55 -0400 Subject: [PATCH 3/3] Fix TypeScript error: widen type variable to string in generateObservation typeof never returns "array" so TypeScript rejected the assignment and comparison. Widening to string allows the custom "array" tag used to distinguish array-of-objects columns from plain objects. Co-Authored-By: Claude Sonnet 4.6 --- packages/metadata/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/metadata/src/index.ts b/packages/metadata/src/index.ts index 510d3b7..a91a03f 100644 --- a/packages/metadata/src/index.ts +++ b/packages/metadata/src/index.ts @@ -434,7 +434,7 @@ export default class JsPsychMetadata { for (const variable in observation) { var value = observation[variable]; - var type = typeof value; + var type: string = typeof value; if (value === null || value === undefined || value === '' || value === "null"){ continue; // Error checking