Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package db.migration;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.flywaydb.core.api.migration.BaseJavaMigration;
import org.flywaydb.core.api.migration.Context;

import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Map;

/**
* Rewrites stored CDEs to the single 'type' contract of the variable-type refactor:
* drops 'sql_type' and 'isCategorical', derives the new 'type' from them
* (categorical -> nominal, real -> continuous, int -> discrete, text -> nominal)
* and drops empty 'enumerations' lists.
*/
public class V1_1__MigrateCdeVariableTypes extends BaseJavaMigration {

private static final ObjectMapper objectMapper = new ObjectMapper();
private static final Map<String, String> TYPE_PER_SQL_TYPE =
Map.of("real", "continuous", "int", "discrete", "text", "nominal");
private static final Map<String, String> TYPE_PER_LEGACY_TYPE =
Map.of("real", "continuous", "integer", "discrete", "text", "nominal");

@Override
public void migrate(Context context) throws Exception {
try (Statement select = context.getConnection().createStatement();
ResultSet rows = select.executeQuery("SELECT uuid, variables, groups FROM data_model");
PreparedStatement update = context.getConnection().prepareStatement(
"UPDATE data_model SET variables = ?, groups = ? WHERE uuid = ?")) {
while (rows.next()) {
update.setString(1, migrateJson(rows.getString("variables")));
update.setString(2, migrateJson(rows.getString("groups")));
update.setObject(3, rows.getObject("uuid"));
update.addBatch();
}
update.executeBatch();
}
}

static String migrateJson(String json) throws JsonProcessingException {
if (json == null) {
return null;
}
JsonNode node = objectMapper.readTree(json);
migrateNode(node);
return objectMapper.writeValueAsString(node);
}

private static void migrateNode(JsonNode node) {
if (node.isArray()) {
node.forEach(V1_1__MigrateCdeVariableTypes::migrateNode);
return;
}
if (!node.isObject()) {
return;
}
ObjectNode object = (ObjectNode) node;
if (object.has("variables") || object.has("groups")) {
migrateNode(object.path("variables"));
migrateNode(object.path("groups"));
} else {
migrateVariable(object);
}
}

private static void migrateVariable(ObjectNode variable) {
JsonNode isCategorical = variable.remove("isCategorical");
JsonNode sqlType = variable.remove("sql_type");
String type = variable.path("type").asText();
if (isCategorical != null && isCategorical.asBoolean()) {
type = "nominal";
} else if (sqlType != null && TYPE_PER_SQL_TYPE.containsKey(sqlType.asText())) {
type = TYPE_PER_SQL_TYPE.get(sqlType.asText());
} else {
type = TYPE_PER_LEGACY_TYPE.getOrDefault(type, type);
}
variable.put("type", type);

JsonNode enumerations = variable.get("enumerations");
if (enumerations != null && enumerations.isArray() && enumerations.isEmpty()) {
variable.remove("enumerations");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,38 +8,51 @@ public record CommonDataElementDTO(
String code,
String label,
String description,
String sql_type,
Boolean isCategorical,
List<EnumerationDTO> enumerations,
Integer minValue,
Integer maxValue,
Double minValue,
Double maxValue,
String type,
String methodology,
String units
) {
public CommonDataElementDTO {
validateField(code, "code", code);
validateField(label, "label", code);
validateField(sql_type, "sql_type", code);
validateField(type, "type", code);

// Validate 'type' field against valid types
if (!NUMERICAL_TYPES.contains(type) && !CATEGORICAL_TYPES.contains(type)) {
throw new InvalidDataModelError("The CDE " + code + " has an invalid 'type'. Valid types are: "
+ List.of("continuous", "discrete", "nominal", "ordinal", "binary"));
}

if (Boolean.TRUE.equals(isCategorical) && (enumerations == null || enumerations.isEmpty())) {
throw new InvalidDataModelError("The CDE " + code + " has 'is_categorical' set to True but there are no enumerations.");
boolean hasEnumerations = enumerations != null && !enumerations.isEmpty();
if (NUMERICAL_TYPES.contains(type) && hasEnumerations) {
throw new InvalidDataModelError("The CDE " + code + " is " + type + " and cannot have enumerations.");
}
if (CATEGORICAL_TYPES.contains(type) && (minValue != null || maxValue != null)) {
throw new InvalidDataModelError("The CDE " + code + " is " + type + " and cannot have minValue/maxValue.");
}
if ("binary".equals(type) && (!hasEnumerations || enumerations.size() != 2)) {
throw new InvalidDataModelError("The CDE " + code + " is binary and must have exactly two enumerations.");
}

// Check minValue and maxValue values
if (minValue != null && maxValue != null && minValue >= maxValue) {
throw new InvalidDataModelError("The CDE " + code + " has minValue greater than or equal to maxValue.");
}

// Validate 'type' field against valid types
Set<String> validTypes = Set.of("nominal", "real", "integer", "text");
if (!validTypes.contains(type)) {
throw new InvalidDataModelError("The CDE " + code + " has an invalid 'type'. Valid types are: " + validTypes);
if ("discrete".equals(type) && (!isWholeNumber(minValue) || !isWholeNumber(maxValue))) {
throw new InvalidDataModelError("The CDE " + code + " is discrete and its minValue/maxValue must be whole numbers.");
}
}

private static final Set<String> NUMERICAL_TYPES = Set.of("continuous", "discrete");
private static final Set<String> CATEGORICAL_TYPES = Set.of("nominal", "ordinal", "binary");

private static boolean isWholeNumber(Double value) {
return value == null || value % 1 == 0;
}

private static void validateField(Object field, String fieldName, String code) {
if (field == null || (field instanceof String && ((String) field).isEmpty())) {
throw new InvalidDataModelError(String.format("Element: '%s' is missing from the CDE %s", fieldName, code));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package db.migration;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;

public class V1_1__MigrateCdeVariableTypesTest {

private static final ObjectMapper objectMapper = new ObjectMapper();

private static void assertMigrated(String expected, String stored) throws Exception {
assertEquals(objectMapper.readTree(expected),
objectMapper.readTree(V1_1__MigrateCdeVariableTypes.migrateJson(stored)));
}

@Test
void variablesTakeTheirTypeFromTheOldStorageFields() throws Exception {
assertMigrated("""
[{"code": "dataset", "type": "nominal", "enumerations": [{"code": "d", "label": "D"}]},
{"code": "age", "type": "discrete", "minValue": 0, "maxValue": 130},
{"code": "score", "type": "continuous"},
{"code": "subjectid", "type": "nominal"},
{"code": "mismatched", "type": "continuous"}]
""", """
[{"code": "dataset", "type": "nominal", "sql_type": "text", "isCategorical": true,
"enumerations": [{"code": "d", "label": "D"}]},
{"code": "age", "type": "integer", "sql_type": "int", "isCategorical": false,
"minValue": 0, "maxValue": 130},
{"code": "score", "type": "real", "sql_type": "real", "isCategorical": false,
"enumerations": []},
{"code": "subjectid", "type": "text", "sql_type": "text", "isCategorical": false},
{"code": "mismatched", "type": "nominal", "sql_type": "real", "isCategorical": false}]
""");
}

@Test
void nestedGroupVariablesAreMigrated() throws Exception {
assertMigrated("""
[{"code": "g", "label": "G", "variables": [{"code": "v", "type": "continuous"}],
"groups": [{"code": "h", "label": "H", "variables": [{"code": "w", "type": "nominal"}]}]}]
""", """
[{"code": "g", "label": "G", "variables": [{"code": "v", "type": "real", "sql_type": "real", "isCategorical": false}],
"groups": [{"code": "h", "label": "H", "variables": [{"code": "w", "type": "text"}]}]}]
""");
}

@Test
void alreadyMigratedAndNullJsonAreUnchanged() throws Exception {
assertMigrated("[{\"code\": \"v\", \"type\": \"ordinal\"}]", "[{\"code\": \"v\", \"type\": \"ordinal\"}]");
assertNull(V1_1__MigrateCdeVariableTypes.migrateJson(null));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package ebrainsv2.mip.datacatalog.datamodel;

import org.junit.jupiter.api.Test;

import java.util.List;

import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;

public class CommonDataElementDTOTest {

private static final List<CommonDataElementDTO.EnumerationDTO> TWO = List.of(
new CommonDataElementDTO.EnumerationDTO("0", "No"),
new CommonDataElementDTO.EnumerationDTO("1", "Yes"));

private static CommonDataElementDTO cde(String type, List<CommonDataElementDTO.EnumerationDTO> enumerations,
Double minValue, Double maxValue) {
return new CommonDataElementDTO("var", "Var", null, enumerations, minValue, maxValue, type, null, null);
}

@Test
void acceptsTheFiveVariableTypes() {
assertDoesNotThrow(() -> cde("continuous", null, 0.0, 10.0));
assertDoesNotThrow(() -> cde("discrete", null, null, null));
assertDoesNotThrow(() -> cde("nominal", null, null, null));
assertDoesNotThrow(() -> cde("ordinal", TWO.subList(0, 1), null, null));
assertDoesNotThrow(() -> cde("binary", TWO, null, null));
}

@Test
void rejectsLegacyTypes() {
for (String legacyType : List.of("real", "integer", "text")) {
assertThrows(InvalidDataModelError.class, () -> cde(legacyType, null, null, null));
}
}

@Test
void rejectsValuesThatDoNotMatchTheType() {
assertThrows(InvalidDataModelError.class, () -> cde("continuous", TWO, null, null));
assertThrows(InvalidDataModelError.class, () -> cde("nominal", TWO, 0.0, 1.0));
assertThrows(InvalidDataModelError.class, () -> cde("binary", TWO.subList(0, 1), null, null));
assertThrows(InvalidDataModelError.class, () -> cde("binary", null, null, null));
}

@Test
void acceptsFractionalBoundsOnlyForContinuous() {
assertDoesNotThrow(() -> cde("continuous", null, 0.01, 99.99));
assertDoesNotThrow(() -> cde("discrete", null, 0.0, 130.0));
assertThrows(InvalidDataModelError.class, () -> cde("discrete", null, 0.5, 10.0));
assertThrows(InvalidDataModelError.class, () -> cde("discrete", null, 0.0, 9.5));
}
}
8 changes: 1 addition & 7 deletions backend/src/test/resources/MinimalDataModelExample.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@
"code": "dataset",
"label": "Dataset Variable",
"description": "An example variable description",
"sql_type": "text",
"isCategorical": true,
"enumerations": [
{
"code": "enum1",
Expand All @@ -31,11 +29,9 @@
"code": "group_variable",
"label": "Group Variable",
"description": "A variable within a group",
"sql_type": "int",
"isCategorical": false,
"minValue": 0,
"maxValue": 100,
"type": "integer",
"type": "discrete",
"methodology": "group methodology",
"units": "years"
}
Expand All @@ -49,8 +45,6 @@
"code": "nested_group_variable",
"label": "Nested Group Variable",
"description": "A nested group variable",
"sql_type": "text",
"isCategorical": true,
"enumerations": [
{
"code": "nested_enum1",
Expand Down
14 changes: 9 additions & 5 deletions data_quality_tool/EXCEL_TO_CDES_JSON.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ and how each Excel column maps to the output JSON.
| --- | --- | --- |
| `name` | yes | non-empty |
| `code` | yes | non-empty |
| `type` | yes | one of `nominal`, `real`, `integer`, `text` |
| `type` | yes | one of `continuous`, `discrete` (numerical) or `nominal`, `ordinal`, `binary` (categorical) |
| `conceptPath` | yes | non-empty, formatted `level1/level2/...` |
| `values` | when `type=nominal` | `"{"code", "label"}"` pairs; codes must be unique |
| `values` | when `type=real`/`integer` | `<min>-<max>` range; `min` < `max`; whole numbers for `integer` |
| `values` | categorical types | `"{"code", "label"}"` pairs; codes must be unique; required with exactly two pairs for `binary`, optional for `nominal`/`ordinal` (free text) |
| `values` | numerical types | optional `<min>-<max>` range; `min` < `max`; whole numbers for `discrete` |
| `unit`, `description`, `canBeNull`, `comments`, `methodology` | no | any non-empty value is kept; empty values are dropped |

## Column → JSON mapping
Expand All @@ -34,7 +34,7 @@ and how each Excel column maps to the output JSON.
| `name` | `label` | |
| `code` | `code` | |
| `type` | `type` | |
| `values` | `enumerations` (nominal) / `minValue` + `maxValue` (real, integer) | |
| `values` | `enumerations` (nominal, ordinal, binary) / `minValue` + `maxValue` (continuous, discrete) | |
| `unit` | `units` | |
| `description` | `description` | |
| `canBeNull` | `canBeNull` | |
Expand All @@ -46,6 +46,10 @@ and how each Excel column maps to the output JSON.

- The `groups` tree is built from `conceptPath`: each `/`-separated level becomes
a nested group.
- `sql_type` and `isCategorical` are derived from `type`.
- `type` carries the full classification; `sql_type` and `isCategorical` are not
emitted.
- Migrating old files: `real` → `continuous`, `integer` → `discrete`, `nominal`
stays `nominal`, `text` → `nominal` (or `ordinal` when the categories are
ordered).
- Empty fields and empty `groups`/`variables`/`enumerations` lists are removed
from the output.
32 changes: 17 additions & 15 deletions data_quality_tool/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,10 @@ Hard requirements for `POST /excel-to-json` / `POST /validate-excel`, enforced b
- **Columns**: header must be exactly these 11 — `csvFile`, `name`, `code`, `type`, `values`,
`unit`, `description`, `canBeNull`, `comments`, `conceptPath`, `methodology`
- **Required per row**: `name`, `code`, `type`, `conceptPath`
- **`type`**: `nominal`, `real`, `integer`, or `text`
- **`values`** (only when filled): `"{"code", "label"}"` pairs for `nominal` (no duplicate codes);
`<min>-<max>` for `real`/`integer` (min < max; whole numbers for `integer`)
- **`type`**: numerical `continuous` or `discrete`; categorical `nominal`, `ordinal`, or `binary`
- **`values`** (only when filled): `"{"code", "label"}"` pairs for `nominal`/`ordinal`/`binary`
(no duplicate codes; `binary` requires exactly two); `<min>-<max>` for `continuous`/`discrete`
(min < max; whole numbers for `discrete`)
- **`conceptPath`**: `level1/level2/...` — builds the group hierarchy, then dropped

**Column → CDEs JSON mapping**
Expand All @@ -46,7 +47,7 @@ Hard requirements for `POST /excel-to-json` / `POST /validate-excel`, enforced b
| `name` | `label` | |
| `code` | `code` | |
| `type` | `type` | |
| `values` | `enumerations` / `minValue`+`maxValue` | nominal → enumerations; real/integer → range |
| `values` | `enumerations` / `minValue`+`maxValue` | categorical → enumerations; numerical → range |
| `unit` | `units` | |
| `description` | `description` | |
| `canBeNull` | `canBeNull` | |
Expand All @@ -55,26 +56,27 @@ Hard requirements for `POST /excel-to-json` / `POST /validate-excel`, enforced b
| `conceptPath` | — | consumed to build the tree |
| `csvFile` | — | **not used** in this direction |

`sql_type` and `isCategorical` are derived from `type`; empty fields and empty
`type` is the only typing field (no `sql_type` or `isCategorical`); empty fields and empty
`groups`/`variables`/`enumerations` lists are stripped. Failures return `400` with a single
bounded error line naming the variable.

## Validation Rules

Excel workbooks must contain exactly these columns: `csvFile`, `name`, `code`, `type`, `values`,
`unit`, `description`, `canBeNull`, `comments`, `conceptPath`, `methodology`. `name`, `code`, `type`,
and `conceptPath` are required on every row, `type` must be `nominal`, `real`, `integer`, or `text`,
`values` is checked as `{"code", "label"}, ...` for `nominal` and as `<min>-<max>` for `real` and
`integer` whenever it is filled in, duplicate enumeration codes are rejected, and `conceptPath` must
look like `characters/characters/...`.
and `conceptPath` are required on every row, `type` must be `continuous`, `discrete`, `nominal`,
`ordinal`, or `binary`, `values` is checked as `{"code", "label"}, ...` for the categorical types and
as `<min>-<max>` for the numerical types whenever it is filled in, `binary` needs exactly two values,
duplicate enumeration codes are rejected, and `conceptPath` must look like `characters/characters/...`.

JSON data models must carry non-empty `code`, `version`, `label`, and a `groups` list. Every group
needs a `code` that is unique within its path, and every Common Data Element needs `code`, `type`,
`sql_type`, and `isCategorical` that agree with the mapping `nominal` → `text`/`true`,
`real` → `real`/`false`, `integer` → `int`/`false`, `text` → `text`/`false`. Categorical elements
require `enumerations`, `minValue` must be below `maxValue`, element codes must be unique inside a
group, at least one dataset element (`code: dataset`, `sql_type: text`, categorical) must exist, and
longitudinal models must define `subjectid` and `visitid`.
needs a `code` that is unique within its path, and every Common Data Element needs `code` and a
`type` of `continuous`, `discrete`, `nominal`, `ordinal`, or `binary`. Numerical elements cannot carry
`enumerations` and categorical elements cannot carry `minValue`/`maxValue`; `binary` elements need
exactly two `enumerations`, `discrete` bounds must be whole numbers, and `minValue` must be below
`maxValue`. `nominal` and `ordinal` elements without `enumerations` are free-text variables. Element
codes must be unique inside a group, at least one dataset element (`code: dataset`, `type: nominal`,
with `enumerations`) must exist, and longitudinal models must define `subjectid` and `visitid`.

## Local Development

Expand Down
Loading
Loading