Fix always-empty columns being dropped from variableMeasured

.changeset/fix-always-empty-columns.md

@@ -0,0 +1,5 @@
+---
+"@jspsych/metadata": patch
+---
+
+Fix always-empty columns being silently dropped from variableMeasured. Columns whose values are null or empty across all rows in a dataset now appear in variableMeasured with a minimal `"value": "unknown"` entry, satisfying the Psych-DS requirement that every CSV column header has a corresponding entry.

packages/metadata/src/index.ts

@@ -425,7 +425,19 @@ export default class JsPsychMetadata {
       var value = observation[variable];
       var type = typeof value;
 
-      if (value === null || value === undefined || value === '' || value === "null"){ 
+      // Ensure every column header appears in variableMeasured even if all its values are null/empty.
+      // Columns that never get a real value keep value:"unknown" and no levels, which satisfies the
+      // Psych-DS requirement that every CSV column header has a corresponding variableMeasured entry.
+      if (!this.containsVariable(variable) && !this.ignored_variables.has(variable)) {
+        this.setVariable({
+          "@type": "PropertyValue",
+          name: variable,
+          description: { default: "unknown" },
+          value: "unknown",
+        });
+      }
+
+      if (value === null || value === undefined || value === '' || value === "null"){
         continue; // Error checking
       }
 
@@ -484,14 +496,18 @@ export default class JsPsychMetadata {
     var type = typeof value;
 
     if (!this.containsVariable(variable)) {
-      // probs should have update description called here
       const new_var = {
         "@type": "PropertyValue",
         name: variable,
         description: { default: "unknown" },
         value: type,
       };
       this.setVariable(new_var);
+    } else {
+      // Column was pre-registered with value:"unknown" in generateObservation; upgrade to the real type
+      // now that we have a concrete value.
+      const existing = this.getVariable(variable) as VariableFields;
+      if (existing.value === "unknown") this.updateVariable(variable, "value", type);
     }
 
     // hit the update variable decription fields

packages/metadata/tests/metadata-module.test.ts

@@ -1,5 +1,71 @@
 import JsPsychMetadata from "../src/index";
 
+describe("always-empty columns in variableMeasured", () => {
+  let jsPsychMetadata: JsPsychMetadata;
+
+  beforeEach(() => {
+    jsPsychMetadata = new JsPsychMetadata();
+  });
+
+  test("column with all empty values appears in variableMeasured with value:unknown", async () => {
+    const csv = [
+      "trial_type,rt,eye_tracking_status",
+      "jsPsych-html-keyboard-response,450,",
+      "jsPsych-html-keyboard-response,512,",
+      "jsPsych-html-keyboard-response,389,",
+    ].join("\n");
+
+    await jsPsychMetadata.generate(csv, {}, "csv");
+
+    const variableMeasured = jsPsychMetadata.getMetadata()["variableMeasured"] as any[];
+    const names = variableMeasured.map((v) => v.name);
+
+    expect(names).toContain("eye_tracking_status");
+
+    const emptyCol = variableMeasured.find((v) => v.name === "eye_tracking_status");
+    expect(emptyCol.value).toBe("unknown");
+    expect(emptyCol.levels).toBeUndefined();
+    expect(emptyCol.minValue).toBeUndefined();
+    expect(emptyCol.maxValue).toBeUndefined();
+  });
+
+  test("column with only null string values appears in variableMeasured with value:unknown", async () => {
+    const csv = [
+      "trial_type,rt,score",
+      "jsPsych-html-keyboard-response,450,null",
+      "jsPsych-html-keyboard-response,512,null",
+    ].join("\n");
+
+    await jsPsychMetadata.generate(csv, {}, "csv");
+
+    const variableMeasured = jsPsychMetadata.getMetadata()["variableMeasured"] as any[];
+    const emptyCol = variableMeasured.find((v) => v.name === "score");
+
+    expect(emptyCol).toBeDefined();
+    expect(emptyCol.value).toBe("unknown");
+    expect(emptyCol.levels).toBeUndefined();
+  });
+
+  test("column with some empty and some real values gets the correct type, not unknown", async () => {
+    const csv = [
+      "trial_type,rt,score",
+      "jsPsych-html-keyboard-response,450,",
+      "jsPsych-html-keyboard-response,512,8",
+      "jsPsych-html-keyboard-response,389,",
+    ].join("\n");
+
+    await jsPsychMetadata.generate(csv, {}, "csv");
+
+    const variableMeasured = jsPsychMetadata.getMetadata()["variableMeasured"] as any[];
+    const partialCol = variableMeasured.find((v) => v.name === "score");
+
+    expect(partialCol).toBeDefined();
+    expect(partialCol.value).toBe("number");
+    expect(partialCol.value).not.toBe("unknown");
+  });
+
+});
+
 // missing displaying data modules tests
 describe("JsPsychMetadata", () => {
   let jsPsychMetadata: JsPsychMetadata;