diff --git a/backend/src/main/java/db/migration/V1_1__MigrateCdeVariableTypes.java b/backend/src/main/java/db/migration/V1_1__MigrateCdeVariableTypes.java new file mode 100644 index 0000000..378f8b4 --- /dev/null +++ b/backend/src/main/java/db/migration/V1_1__MigrateCdeVariableTypes.java @@ -0,0 +1,89 @@ +package db.migration; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; + +/** + * Rewrites stored CDEs to the single 'type' contract of the variable-type refactor: + * drops 'sql_type' and 'isCategorical', derives the new 'type' from them + * (categorical -> nominal, real -> continuous, int -> discrete, text -> nominal) + * and drops empty 'enumerations' lists. + */ +public class V1_1__MigrateCdeVariableTypes extends BaseJavaMigration { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final Map TYPE_PER_SQL_TYPE = + Map.of("real", "continuous", "int", "discrete", "text", "nominal"); + private static final Map TYPE_PER_LEGACY_TYPE = + Map.of("real", "continuous", "integer", "discrete", "text", "nominal"); + + @Override + public void migrate(Context context) throws Exception { + try (Statement select = context.getConnection().createStatement(); + ResultSet rows = select.executeQuery("SELECT uuid, variables, groups FROM data_model"); + PreparedStatement update = context.getConnection().prepareStatement( + "UPDATE data_model SET variables = ?, groups = ? WHERE uuid = ?")) { + while (rows.next()) { + update.setString(1, migrateJson(rows.getString("variables"))); + update.setString(2, migrateJson(rows.getString("groups"))); + update.setObject(3, rows.getObject("uuid")); + update.addBatch(); + } + update.executeBatch(); + } + } + + static String migrateJson(String json) throws JsonProcessingException { + if (json == null) { + return null; + } + JsonNode node = objectMapper.readTree(json); + migrateNode(node); + return objectMapper.writeValueAsString(node); + } + + private static void migrateNode(JsonNode node) { + if (node.isArray()) { + node.forEach(V1_1__MigrateCdeVariableTypes::migrateNode); + return; + } + if (!node.isObject()) { + return; + } + ObjectNode object = (ObjectNode) node; + if (object.has("variables") || object.has("groups")) { + migrateNode(object.path("variables")); + migrateNode(object.path("groups")); + } else { + migrateVariable(object); + } + } + + private static void migrateVariable(ObjectNode variable) { + JsonNode isCategorical = variable.remove("isCategorical"); + JsonNode sqlType = variable.remove("sql_type"); + String type = variable.path("type").asText(); + if (isCategorical != null && isCategorical.asBoolean()) { + type = "nominal"; + } else if (sqlType != null && TYPE_PER_SQL_TYPE.containsKey(sqlType.asText())) { + type = TYPE_PER_SQL_TYPE.get(sqlType.asText()); + } else { + type = TYPE_PER_LEGACY_TYPE.getOrDefault(type, type); + } + variable.put("type", type); + + JsonNode enumerations = variable.get("enumerations"); + if (enumerations != null && enumerations.isArray() && enumerations.isEmpty()) { + variable.remove("enumerations"); + } + } +} diff --git a/backend/src/main/java/ebrainsv2/mip/datacatalog/datamodel/CommonDataElementDTO.java b/backend/src/main/java/ebrainsv2/mip/datacatalog/datamodel/CommonDataElementDTO.java index 2dc6529..1b4bb21 100644 --- a/backend/src/main/java/ebrainsv2/mip/datacatalog/datamodel/CommonDataElementDTO.java +++ b/backend/src/main/java/ebrainsv2/mip/datacatalog/datamodel/CommonDataElementDTO.java @@ -8,11 +8,9 @@ public record CommonDataElementDTO( String code, String label, String description, - String sql_type, - Boolean isCategorical, List enumerations, - Integer minValue, - Integer maxValue, + Double minValue, + Double maxValue, String type, String methodology, String units @@ -20,26 +18,41 @@ public record CommonDataElementDTO( public CommonDataElementDTO { validateField(code, "code", code); validateField(label, "label", code); - validateField(sql_type, "sql_type", code); validateField(type, "type", code); + // Validate 'type' field against valid types + if (!NUMERICAL_TYPES.contains(type) && !CATEGORICAL_TYPES.contains(type)) { + throw new InvalidDataModelError("The CDE " + code + " has an invalid 'type'. Valid types are: " + + List.of("continuous", "discrete", "nominal", "ordinal", "binary")); + } - if (Boolean.TRUE.equals(isCategorical) && (enumerations == null || enumerations.isEmpty())) { - throw new InvalidDataModelError("The CDE " + code + " has 'is_categorical' set to True but there are no enumerations."); + boolean hasEnumerations = enumerations != null && !enumerations.isEmpty(); + if (NUMERICAL_TYPES.contains(type) && hasEnumerations) { + throw new InvalidDataModelError("The CDE " + code + " is " + type + " and cannot have enumerations."); + } + if (CATEGORICAL_TYPES.contains(type) && (minValue != null || maxValue != null)) { + throw new InvalidDataModelError("The CDE " + code + " is " + type + " and cannot have minValue/maxValue."); + } + if ("binary".equals(type) && (!hasEnumerations || enumerations.size() != 2)) { + throw new InvalidDataModelError("The CDE " + code + " is binary and must have exactly two enumerations."); } // Check minValue and maxValue values if (minValue != null && maxValue != null && minValue >= maxValue) { throw new InvalidDataModelError("The CDE " + code + " has minValue greater than or equal to maxValue."); } - - // Validate 'type' field against valid types - Set validTypes = Set.of("nominal", "real", "integer", "text"); - if (!validTypes.contains(type)) { - throw new InvalidDataModelError("The CDE " + code + " has an invalid 'type'. Valid types are: " + validTypes); + if ("discrete".equals(type) && (!isWholeNumber(minValue) || !isWholeNumber(maxValue))) { + throw new InvalidDataModelError("The CDE " + code + " is discrete and its minValue/maxValue must be whole numbers."); } } + private static final Set NUMERICAL_TYPES = Set.of("continuous", "discrete"); + private static final Set CATEGORICAL_TYPES = Set.of("nominal", "ordinal", "binary"); + + private static boolean isWholeNumber(Double value) { + return value == null || value % 1 == 0; + } + private static void validateField(Object field, String fieldName, String code) { if (field == null || (field instanceof String && ((String) field).isEmpty())) { throw new InvalidDataModelError(String.format("Element: '%s' is missing from the CDE %s", fieldName, code)); diff --git a/backend/src/test/java/db/migration/V1_1__MigrateCdeVariableTypesTest.java b/backend/src/test/java/db/migration/V1_1__MigrateCdeVariableTypesTest.java new file mode 100644 index 0000000..3eb6d20 --- /dev/null +++ b/backend/src/test/java/db/migration/V1_1__MigrateCdeVariableTypesTest.java @@ -0,0 +1,54 @@ +package db.migration; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class V1_1__MigrateCdeVariableTypesTest { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private static void assertMigrated(String expected, String stored) throws Exception { + assertEquals(objectMapper.readTree(expected), + objectMapper.readTree(V1_1__MigrateCdeVariableTypes.migrateJson(stored))); + } + + @Test + void variablesTakeTheirTypeFromTheOldStorageFields() throws Exception { + assertMigrated(""" + [{"code": "dataset", "type": "nominal", "enumerations": [{"code": "d", "label": "D"}]}, + {"code": "age", "type": "discrete", "minValue": 0, "maxValue": 130}, + {"code": "score", "type": "continuous"}, + {"code": "subjectid", "type": "nominal"}, + {"code": "mismatched", "type": "continuous"}] + """, """ + [{"code": "dataset", "type": "nominal", "sql_type": "text", "isCategorical": true, + "enumerations": [{"code": "d", "label": "D"}]}, + {"code": "age", "type": "integer", "sql_type": "int", "isCategorical": false, + "minValue": 0, "maxValue": 130}, + {"code": "score", "type": "real", "sql_type": "real", "isCategorical": false, + "enumerations": []}, + {"code": "subjectid", "type": "text", "sql_type": "text", "isCategorical": false}, + {"code": "mismatched", "type": "nominal", "sql_type": "real", "isCategorical": false}] + """); + } + + @Test + void nestedGroupVariablesAreMigrated() throws Exception { + assertMigrated(""" + [{"code": "g", "label": "G", "variables": [{"code": "v", "type": "continuous"}], + "groups": [{"code": "h", "label": "H", "variables": [{"code": "w", "type": "nominal"}]}]}] + """, """ + [{"code": "g", "label": "G", "variables": [{"code": "v", "type": "real", "sql_type": "real", "isCategorical": false}], + "groups": [{"code": "h", "label": "H", "variables": [{"code": "w", "type": "text"}]}]}] + """); + } + + @Test + void alreadyMigratedAndNullJsonAreUnchanged() throws Exception { + assertMigrated("[{\"code\": \"v\", \"type\": \"ordinal\"}]", "[{\"code\": \"v\", \"type\": \"ordinal\"}]"); + assertNull(V1_1__MigrateCdeVariableTypes.migrateJson(null)); + } +} diff --git a/backend/src/test/java/ebrainsv2/mip/datacatalog/datamodel/CommonDataElementDTOTest.java b/backend/src/test/java/ebrainsv2/mip/datacatalog/datamodel/CommonDataElementDTOTest.java new file mode 100644 index 0000000..0dc7abf --- /dev/null +++ b/backend/src/test/java/ebrainsv2/mip/datacatalog/datamodel/CommonDataElementDTOTest.java @@ -0,0 +1,52 @@ +package ebrainsv2.mip.datacatalog.datamodel; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class CommonDataElementDTOTest { + + private static final List TWO = List.of( + new CommonDataElementDTO.EnumerationDTO("0", "No"), + new CommonDataElementDTO.EnumerationDTO("1", "Yes")); + + private static CommonDataElementDTO cde(String type, List enumerations, + Double minValue, Double maxValue) { + return new CommonDataElementDTO("var", "Var", null, enumerations, minValue, maxValue, type, null, null); + } + + @Test + void acceptsTheFiveVariableTypes() { + assertDoesNotThrow(() -> cde("continuous", null, 0.0, 10.0)); + assertDoesNotThrow(() -> cde("discrete", null, null, null)); + assertDoesNotThrow(() -> cde("nominal", null, null, null)); + assertDoesNotThrow(() -> cde("ordinal", TWO.subList(0, 1), null, null)); + assertDoesNotThrow(() -> cde("binary", TWO, null, null)); + } + + @Test + void rejectsLegacyTypes() { + for (String legacyType : List.of("real", "integer", "text")) { + assertThrows(InvalidDataModelError.class, () -> cde(legacyType, null, null, null)); + } + } + + @Test + void rejectsValuesThatDoNotMatchTheType() { + assertThrows(InvalidDataModelError.class, () -> cde("continuous", TWO, null, null)); + assertThrows(InvalidDataModelError.class, () -> cde("nominal", TWO, 0.0, 1.0)); + assertThrows(InvalidDataModelError.class, () -> cde("binary", TWO.subList(0, 1), null, null)); + assertThrows(InvalidDataModelError.class, () -> cde("binary", null, null, null)); + } + + @Test + void acceptsFractionalBoundsOnlyForContinuous() { + assertDoesNotThrow(() -> cde("continuous", null, 0.01, 99.99)); + assertDoesNotThrow(() -> cde("discrete", null, 0.0, 130.0)); + assertThrows(InvalidDataModelError.class, () -> cde("discrete", null, 0.5, 10.0)); + assertThrows(InvalidDataModelError.class, () -> cde("discrete", null, 0.0, 9.5)); + } +} diff --git a/backend/src/test/resources/MinimalDataModelExample.json b/backend/src/test/resources/MinimalDataModelExample.json index b8c5077..87bd885 100644 --- a/backend/src/test/resources/MinimalDataModelExample.json +++ b/backend/src/test/resources/MinimalDataModelExample.json @@ -9,8 +9,6 @@ "code": "dataset", "label": "Dataset Variable", "description": "An example variable description", - "sql_type": "text", - "isCategorical": true, "enumerations": [ { "code": "enum1", @@ -31,11 +29,9 @@ "code": "group_variable", "label": "Group Variable", "description": "A variable within a group", - "sql_type": "int", - "isCategorical": false, "minValue": 0, "maxValue": 100, - "type": "integer", + "type": "discrete", "methodology": "group methodology", "units": "years" } @@ -49,8 +45,6 @@ "code": "nested_group_variable", "label": "Nested Group Variable", "description": "A nested group variable", - "sql_type": "text", - "isCategorical": true, "enumerations": [ { "code": "nested_enum1", diff --git a/data_quality_tool/EXCEL_TO_CDES_JSON.md b/data_quality_tool/EXCEL_TO_CDES_JSON.md index ea69287..25e5cd7 100644 --- a/data_quality_tool/EXCEL_TO_CDES_JSON.md +++ b/data_quality_tool/EXCEL_TO_CDES_JSON.md @@ -21,10 +21,10 @@ and how each Excel column maps to the output JSON. | --- | --- | --- | | `name` | yes | non-empty | | `code` | yes | non-empty | -| `type` | yes | one of `nominal`, `real`, `integer`, `text` | +| `type` | yes | one of `continuous`, `discrete` (numerical) or `nominal`, `ordinal`, `binary` (categorical) | | `conceptPath` | yes | non-empty, formatted `level1/level2/...` | -| `values` | when `type=nominal` | `"{"code", "label"}"` pairs; codes must be unique | -| `values` | when `type=real`/`integer` | `-` range; `min` < `max`; whole numbers for `integer` | +| `values` | categorical types | `"{"code", "label"}"` pairs; codes must be unique; required with exactly two pairs for `binary`, optional for `nominal`/`ordinal` (free text) | +| `values` | numerical types | optional `-` range; `min` < `max`; whole numbers for `discrete` | | `unit`, `description`, `canBeNull`, `comments`, `methodology` | no | any non-empty value is kept; empty values are dropped | ## Column → JSON mapping @@ -34,7 +34,7 @@ and how each Excel column maps to the output JSON. | `name` | `label` | | | `code` | `code` | | | `type` | `type` | | -| `values` | `enumerations` (nominal) / `minValue` + `maxValue` (real, integer) | | +| `values` | `enumerations` (nominal, ordinal, binary) / `minValue` + `maxValue` (continuous, discrete) | | | `unit` | `units` | | | `description` | `description` | | | `canBeNull` | `canBeNull` | | @@ -46,6 +46,10 @@ and how each Excel column maps to the output JSON. - The `groups` tree is built from `conceptPath`: each `/`-separated level becomes a nested group. -- `sql_type` and `isCategorical` are derived from `type`. +- `type` carries the full classification; `sql_type` and `isCategorical` are not + emitted. +- Migrating old files: `real` → `continuous`, `integer` → `discrete`, `nominal` + stays `nominal`, `text` → `nominal` (or `ordinal` when the categories are + ordered). - Empty fields and empty `groups`/`variables`/`enumerations` lists are removed from the output. diff --git a/data_quality_tool/README.md b/data_quality_tool/README.md index 9d9d9c0..ef8d5f8 100644 --- a/data_quality_tool/README.md +++ b/data_quality_tool/README.md @@ -34,9 +34,10 @@ Hard requirements for `POST /excel-to-json` / `POST /validate-excel`, enforced b - **Columns**: header must be exactly these 11 — `csvFile`, `name`, `code`, `type`, `values`, `unit`, `description`, `canBeNull`, `comments`, `conceptPath`, `methodology` - **Required per row**: `name`, `code`, `type`, `conceptPath` -- **`type`**: `nominal`, `real`, `integer`, or `text` -- **`values`** (only when filled): `"{"code", "label"}"` pairs for `nominal` (no duplicate codes); - `-` for `real`/`integer` (min < max; whole numbers for `integer`) +- **`type`**: numerical `continuous` or `discrete`; categorical `nominal`, `ordinal`, or `binary` +- **`values`** (only when filled): `"{"code", "label"}"` pairs for `nominal`/`ordinal`/`binary` + (no duplicate codes; `binary` requires exactly two); `-` for `continuous`/`discrete` + (min < max; whole numbers for `discrete`) - **`conceptPath`**: `level1/level2/...` — builds the group hierarchy, then dropped **Column → CDEs JSON mapping** @@ -46,7 +47,7 @@ Hard requirements for `POST /excel-to-json` / `POST /validate-excel`, enforced b | `name` | `label` | | | `code` | `code` | | | `type` | `type` | | -| `values` | `enumerations` / `minValue`+`maxValue` | nominal → enumerations; real/integer → range | +| `values` | `enumerations` / `minValue`+`maxValue` | categorical → enumerations; numerical → range | | `unit` | `units` | | | `description` | `description` | | | `canBeNull` | `canBeNull` | | @@ -55,7 +56,7 @@ Hard requirements for `POST /excel-to-json` / `POST /validate-excel`, enforced b | `conceptPath` | — | consumed to build the tree | | `csvFile` | — | **not used** in this direction | -`sql_type` and `isCategorical` are derived from `type`; empty fields and empty +`type` is the only typing field (no `sql_type` or `isCategorical`); empty fields and empty `groups`/`variables`/`enumerations` lists are stripped. Failures return `400` with a single bounded error line naming the variable. @@ -63,18 +64,19 @@ bounded error line naming the variable. Excel workbooks must contain exactly these columns: `csvFile`, `name`, `code`, `type`, `values`, `unit`, `description`, `canBeNull`, `comments`, `conceptPath`, `methodology`. `name`, `code`, `type`, -and `conceptPath` are required on every row, `type` must be `nominal`, `real`, `integer`, or `text`, -`values` is checked as `{"code", "label"}, ...` for `nominal` and as `-` for `real` and -`integer` whenever it is filled in, duplicate enumeration codes are rejected, and `conceptPath` must -look like `characters/characters/...`. +and `conceptPath` are required on every row, `type` must be `continuous`, `discrete`, `nominal`, +`ordinal`, or `binary`, `values` is checked as `{"code", "label"}, ...` for the categorical types and +as `-` for the numerical types whenever it is filled in, `binary` needs exactly two values, +duplicate enumeration codes are rejected, and `conceptPath` must look like `characters/characters/...`. JSON data models must carry non-empty `code`, `version`, `label`, and a `groups` list. Every group -needs a `code` that is unique within its path, and every Common Data Element needs `code`, `type`, -`sql_type`, and `isCategorical` that agree with the mapping `nominal` → `text`/`true`, -`real` → `real`/`false`, `integer` → `int`/`false`, `text` → `text`/`false`. Categorical elements -require `enumerations`, `minValue` must be below `maxValue`, element codes must be unique inside a -group, at least one dataset element (`code: dataset`, `sql_type: text`, categorical) must exist, and -longitudinal models must define `subjectid` and `visitid`. +needs a `code` that is unique within its path, and every Common Data Element needs `code` and a +`type` of `continuous`, `discrete`, `nominal`, `ordinal`, or `binary`. Numerical elements cannot carry +`enumerations` and categorical elements cannot carry `minValue`/`maxValue`; `binary` elements need +exactly two `enumerations`, `discrete` bounds must be whole numbers, and `minValue` must be below +`maxValue`. `nominal` and `ordinal` elements without `enumerations` are free-text variables. Element +codes must be unique inside a group, at least one dataset element (`code: dataset`, `type: nominal`, +with `enumerations`) must exist, and longitudinal models must define `subjectid` and `visitid`. ## Local Development diff --git a/data_quality_tool/common_entities.py b/data_quality_tool/common_entities.py index 1199614..d0c4103 100644 --- a/data_quality_tool/common_entities.py +++ b/data_quality_tool/common_entities.py @@ -15,12 +15,9 @@ } -EXCEL_TYPE_2_SQL_TYPE_ISCATEGORICAL_MAP = { - "nominal": ("text", True), - "real": ("real", False), - "integer": ("int", False), - "text": ("text", False), -} +NUMERICAL_TYPES = ("continuous", "discrete") +CATEGORICAL_TYPES = ("nominal", "ordinal", "binary") +VARIABLE_TYPES = NUMERICAL_TYPES + CATEGORICAL_TYPES EXCEL_COLUMNS = [ "csvFile", diff --git a/data_quality_tool/converter/excel_to_json.py b/data_quality_tool/converter/excel_to_json.py index 4961659..0d202bf 100644 --- a/data_quality_tool/converter/excel_to_json.py +++ b/data_quality_tool/converter/excel_to_json.py @@ -3,7 +3,9 @@ import re from common_entities import ( - EXCEL_TYPE_2_SQL_TYPE_ISCATEGORICAL_MAP, + CATEGORICAL_TYPES, + NUMERICAL_TYPES, + VARIABLE_TYPES, InvalidDataModelError, ) @@ -86,16 +88,20 @@ def insert_variable_into_structure(root, variable, path): def process_values_based_on_type(row, variable): """ Processes the 'values' field based on the variable's 'type': - - For 'integer' or 'real', extracts 'minValue' and 'maxValue' from a range specified in 'values' - and ensures these values are of the appropriate type. - If 'type' is 'integer' but the parsed bounds aren't whole numbers, raises DatasetError. - - For 'nominal', retrieves a list of 'enumerations' from 'values'. + - For 'continuous' or 'discrete', extracts 'minValue' and 'maxValue' from a range specified + in 'values' and ensures these values are of the appropriate type. + If 'type' is 'discrete' but the parsed bounds aren't whole numbers, raises InvalidDataModelError. + - For 'nominal', 'ordinal' or 'binary', retrieves a list of 'enumerations' from 'values'. + 'binary' requires exactly two enumerations; 'nominal' and 'ordinal' may omit values + (free-text variables). """ code = row.get("code") values = row.get("values") + if values is not None and pd.isna(values): + values = None variable_type = row.get("type") - if variable_type in ["real", "integer"] and values: + if variable_type in NUMERICAL_TYPES and values: values = str(values) # Split on the last hyphen only parts = values.rsplit("-", 1) @@ -115,31 +121,32 @@ def process_values_based_on_type(row, variable): f"Range values for variable {code} must be valid numbers but got '{values}'." ) - if variable_type == "integer": - # If either bound has a fractional part, that's invalid for integer variables + if variable_type == "discrete": + # If either bound has a fractional part, that's invalid for discrete variables if not min_val.is_integer() or not max_val.is_integer(): raise InvalidDataModelError( - f"Variable {code} declared as integer but range bounds '{values}' are not whole numbers." + f"Variable {code} declared as discrete but range bounds '{values}' are not whole numbers." ) # Safe to cast to int variable["minValue"] = int(min_val) variable["maxValue"] = int(max_val) - else: # real + else: # continuous variable["minValue"] = _parse_number_preserving_integer(min_str) variable["maxValue"] = _parse_number_preserving_integer(max_str) - elif variable_type == "nominal": - if not values: + elif variable_type in CATEGORICAL_TYPES: + enumerations = process_enumerations(values) if values else [] + if variable_type == "binary" and len(enumerations) != 2: raise InvalidDataModelError( - f"The 'values' should not be empty for variable {code} when type is 'nominal'" + f"Variable {code} declared as binary must have exactly two values but got '{values}'." ) - variable["enumerations"] = process_enumerations(values) + if enumerations: + variable["enumerations"] = enumerations def validate_variable_type(row): - valid_types = set(EXCEL_TYPE_2_SQL_TYPE_ISCATEGORICAL_MAP.keys()) - if "type" not in row or row["type"] not in valid_types: - valid_types_str = ", ".join(valid_types) + if "type" not in row or row["type"] not in VARIABLE_TYPES: + valid_types_str = ", ".join(VARIABLE_TYPES) raise InvalidDataModelError( f"The row must have a 'type' field with a valid value." f" Valid values are: {valid_types_str}, got '{row.get('type')}' instead." @@ -164,11 +171,6 @@ def process_variable(row): # Process 'values' based on variable type, which might modify 'variable' in-place process_values_based_on_type(row, variable) - ( - variable["sql_type"], - variable["isCategorical"], - ) = EXCEL_TYPE_2_SQL_TYPE_ISCATEGORICAL_MAP[variable["type"]] - return variable @@ -229,8 +231,8 @@ def clean_empty_fields(data): def convert_excel_to_json(df): """ - Converts a DataFrame from Excel into a JSON structure, handling enumerations specifically, - and adds 'isCategorical' and 'sql_type' based on the 'type'. + Converts a DataFrame from Excel into a JSON structure, handling enumerations specifically. + The 'type' carries the full variable classification. """ df = df.where(pd.notna(df), None) diff --git a/data_quality_tool/converter/json_to_excel.py b/data_quality_tool/converter/json_to_excel.py index 56513ef..362d06b 100644 --- a/data_quality_tool/converter/json_to_excel.py +++ b/data_quality_tool/converter/json_to_excel.py @@ -22,8 +22,11 @@ def extract_values(variable): ] return ",".join(dict_strings) - min_value = variable.get("minValue", "") - max_value = variable.get("maxValue", "") + # The catalog backend sends bounds as doubles, so print 130.0 back as 130. + min_value, max_value = ( + int(v) if isinstance(v, float) and v.is_integer() else v + for v in (variable.get("minValue", ""), variable.get("maxValue", "")) + ) if min_value or max_value: return f"{min_value}-{max_value}" return "" diff --git a/data_quality_tool/tests/MinimalDataModelError.xlsx b/data_quality_tool/tests/MinimalDataModelError.xlsx index dcde6f6..b7dc1b6 100644 Binary files a/data_quality_tool/tests/MinimalDataModelError.xlsx and b/data_quality_tool/tests/MinimalDataModelError.xlsx differ diff --git a/data_quality_tool/tests/MinimalDataModelExample.json b/data_quality_tool/tests/MinimalDataModelExample.json index 4b3b3ce..ac5bec3 100644 --- a/data_quality_tool/tests/MinimalDataModelExample.json +++ b/data_quality_tool/tests/MinimalDataModelExample.json @@ -8,8 +8,6 @@ "code": "dataset", "label": "Dataset Variable", "description": "An example variable description", - "sql_type": "text", - "isCategorical": true, "enumerations": [ { "code": "enum1", @@ -30,11 +28,9 @@ "code": "group_variable", "label": "Group Variable", "description": "A variable within a group", - "sql_type": "int", - "isCategorical": false, "minValue": -10, "maxValue": 100, - "type": "integer", + "type": "discrete", "methodology": "group methodology", "units": "years" } @@ -48,8 +44,6 @@ "code": "nested_group_variable", "label": "Nested Group Variable", "description": "A nested group variable", - "sql_type": "text", - "isCategorical": true, "enumerations": [ { "code": "nested_enum1", diff --git a/data_quality_tool/tests/MinimalDataModelExample.xlsx b/data_quality_tool/tests/MinimalDataModelExample.xlsx index dc21ba1..d9acc1a 100644 Binary files a/data_quality_tool/tests/MinimalDataModelExample.xlsx and b/data_quality_tool/tests/MinimalDataModelExample.xlsx differ diff --git a/data_quality_tool/tests/excel_to_json/test_clean_empty_fields.py b/data_quality_tool/tests/excel_to_json/test_clean_empty_fields.py index 559bbc9..7a8aad3 100644 --- a/data_quality_tool/tests/excel_to_json/test_clean_empty_fields.py +++ b/data_quality_tool/tests/excel_to_json/test_clean_empty_fields.py @@ -19,7 +19,7 @@ def test_remove_empty_lists_and_strings(self): "label": "Group Variable", "description": "", "enumerations": [], - "type": "integer", + "type": "discrete", } ], "groups": [], @@ -36,7 +36,7 @@ def test_remove_empty_lists_and_strings(self): { "code": "group_variable", "label": "Group Variable", - "type": "integer", + "type": "discrete", } ], } diff --git a/data_quality_tool/tests/excel_to_json/test_convert_excel_to_json.py b/data_quality_tool/tests/excel_to_json/test_convert_excel_to_json.py index abc950a..6de6c61 100644 --- a/data_quality_tool/tests/excel_to_json/test_convert_excel_to_json.py +++ b/data_quality_tool/tests/excel_to_json/test_convert_excel_to_json.py @@ -10,7 +10,7 @@ class TestConvertExcelToJson(unittest.TestCase): def setUp(self): # Basic setup for DataFrame used in multiple tests self.data = { - "type": ["nominal", "integer"], + "type": ["nominal", "discrete"], "values": ['{"code1", "label1"}, {"code2", "label2"}', "1-100"], "name": ["Nominal Variable", "Integer Variable"], "code": ["NomVar", "IntVar"], @@ -55,7 +55,7 @@ def test_empty_dataframe(self): def test_partially_missing_data(self): # Scenario where some rows have missing 'conceptPath' or other critical fields data = { - "type": ["nominal", "integer", "text"], + "type": ["nominal", "discrete", "nominal"], "values": [ '{"code1", "label1"}', "1-100", @@ -98,7 +98,7 @@ def test_dataframe_with_invalid_type_raises_error(self): def test_variables_without_group_nesting(self): # Variables that should be placed directly under the root due to lack of conceptPath or other logic data = { - "type": ["text", "real"], + "type": ["nominal", "continuous"], "values": [None, "0.1-100.0"], "name": ["Ungrouped Text Variable", "Ungrouped Real Variable"], "code": ["UngroupedTextVar", "UngroupedRealVar"], diff --git a/data_quality_tool/tests/excel_to_json/test_process_values_based_on_type.py b/data_quality_tool/tests/excel_to_json/test_process_values_based_on_type.py index 0729180..9f763a8 100644 --- a/data_quality_tool/tests/excel_to_json/test_process_values_based_on_type.py +++ b/data_quality_tool/tests/excel_to_json/test_process_values_based_on_type.py @@ -6,19 +6,19 @@ class TestProcessValuesBasedOnType(unittest.TestCase): def test_process_integer_range(self): - row = {"type": "integer", "values": "1-100"} + row = {"type": "discrete", "values": "1-100"} variable = {} process_values_based_on_type(row, variable) self.assertEqual(variable["minValue"], 1) self.assertEqual(variable["maxValue"], 100) def test_process_min_max_invalid_type(self): - row = {"type": "integer", "values": "1.1-100.1"} + row = {"type": "discrete", "values": "1.1-100.1"} variable = {} with self.assertRaises(InvalidDataModelError) as context: process_values_based_on_type(row, variable) self.assertIn( - "Variable None declared as integer but range bounds '1.1-100.1' are not whole numbers.", + "Variable None declared as discrete but range bounds '1.1-100.1' are not whole numbers.", str(context.exception), ) @@ -32,25 +32,45 @@ def test_process_nominal_enumerations(self): ] self.assertEqual(variable["enumerations"], expected_enumerations) - def test_nominal_without_values_raises_error(self): - row = {"type": "nominal", "code": "code1"} - variable = {"code": "V1"} - with self.assertRaises(InvalidDataModelError) as context: + def test_nominal_and_ordinal_without_values_have_no_enumerations(self): + for variable_type in ["nominal", "ordinal"]: + variable = {"code": "V1"} + process_values_based_on_type({"type": variable_type}, variable) + self.assertEqual(variable, {"code": "V1"}) + + def test_process_ordinal_and_binary_enumerations(self): + for variable_type in ["ordinal", "binary"]: + row = {"type": variable_type, "values": '{"0", "No"}, {"1", "Yes"}'} + variable = {} process_values_based_on_type(row, variable) - self.assertIn( - "The 'values' should not be empty for variable code1 when type is 'nominal'", - str(context.exception), - ) + self.assertEqual( + variable["enumerations"], + [{"code": "0", "label": "No"}, {"code": "1", "label": "Yes"}], + ) + + def test_binary_without_two_values_raises_error(self): + for values in [ + None, + '{"0", "No"}', + '{"0", "No"}, {"1", "Yes"}, {"2", "Maybe"}', + ]: + row = {"type": "binary", "code": "code1", "values": values} + with self.assertRaises(InvalidDataModelError) as context: + process_values_based_on_type(row, {}) + self.assertIn( + "Variable code1 declared as binary must have exactly two values", + str(context.exception), + ) def test_missing_values_for_range_raises_error(self): - row = {"type": "real", "code": "V2"} + row = {"type": "continuous", "code": "V2"} variable = {} process_values_based_on_type(row, variable) self.assertNotIn("minValue", variable) self.assertNotIn("maxValue", variable) def test_integer_range_with_correct_conversion(self): - row = {"type": "integer", "values": "1-100"} + row = {"type": "discrete", "values": "1-100"} variable = {} process_values_based_on_type(row, variable) self.assertEqual(variable["minValue"], 1) @@ -59,7 +79,7 @@ def test_integer_range_with_correct_conversion(self): self.assertIsInstance(variable["maxValue"], int) def test_real_range_with_correct_conversion(self): - row = {"type": "real", "values": "0.1-99.9"} + row = {"type": "continuous", "values": "0.1-99.9"} variable = {} process_values_based_on_type(row, variable) self.assertEqual(variable["minValue"], 0.1) @@ -68,7 +88,7 @@ def test_real_range_with_correct_conversion(self): self.assertIsInstance(variable["maxValue"], float) def test_invalid_integer_range_raises_error(self): - row = {"type": "integer", "values": "not-an-int - 100", "code": "VInvalidInt"} + row = {"type": "discrete", "values": "not-an-int - 100", "code": "VInvalidInt"} variable = {} with self.assertRaises(InvalidDataModelError) as context: process_values_based_on_type(row, variable) @@ -78,7 +98,11 @@ def test_invalid_integer_range_raises_error(self): ) def test_invalid_real_range_raises_error(self): - row = {"type": "real", "values": "0.1 - not-a-real", "code": "VInvalidReal"} + row = { + "type": "continuous", + "values": "0.1 - not-a-real", + "code": "VInvalidReal", + } variable = {} with self.assertRaises(InvalidDataModelError) as context: process_values_based_on_type(row, variable) @@ -88,7 +112,7 @@ def test_invalid_real_range_raises_error(self): ) def test_invalid_type_in_values_range_raises_error(self): - row = {"type": "real", "values": "a-b", "code": "VInvalidReal"} + row = {"type": "continuous", "values": "a-b", "code": "VInvalidReal"} variable = {} with self.assertRaises(InvalidDataModelError) as context: process_values_based_on_type(row, variable) diff --git a/data_quality_tool/tests/excel_to_json/test_process_variable.py b/data_quality_tool/tests/excel_to_json/test_process_variable.py index 1a26c3c..5fc1e9c 100644 --- a/data_quality_tool/tests/excel_to_json/test_process_variable.py +++ b/data_quality_tool/tests/excel_to_json/test_process_variable.py @@ -20,15 +20,13 @@ def test_process_nominal_variable(self): {"code": "code1", "label": "label1"}, {"code": "code2", "label": "label2"}, ], - "isCategorical": True, - "sql_type": "text", } result_variable = process_variable(row) self.assertEqual(result_variable, expected_variable) def test_process_integer_variable(self): row = { - "type": "integer", + "type": "discrete", "values": "1-100", "name": "Integer Variable", "code": "IntVar", @@ -36,11 +34,9 @@ def test_process_integer_variable(self): expected_variable = { "label": "Integer Variable", "code": "IntVar", - "type": "integer", + "type": "discrete", "minValue": 1, "maxValue": 100, - "isCategorical": False, - "sql_type": "int", } result_variable = process_variable(row) self.assertEqual(result_variable, expected_variable) @@ -56,7 +52,7 @@ def test_invalid_type_raises_error(self): def test_process_real_variable_with_range(self): row = { - "type": "real", + "type": "continuous", "values": "0.01-99.99", "name": "Real Variable", "code": "RealVar", @@ -65,19 +61,17 @@ def test_process_real_variable_with_range(self): expected_variable = { "label": "Real Variable", "code": "RealVar", - "type": "real", + "type": "continuous", "minValue": 0.01, "maxValue": 99.99, "description": "A real number variable", - "isCategorical": False, - "sql_type": "real", } result_variable = process_variable(row) self.assertEqual(result_variable, expected_variable) def test_process_text_variable(self): row = { - "type": "text", + "type": "nominal", "name": "Text Variable", "code": "TextVar", "description": "A text variable", @@ -85,32 +79,28 @@ def test_process_text_variable(self): expected_variable = { "label": "Text Variable", "code": "TextVar", - "type": "text", + "type": "nominal", "description": "A text variable", - "isCategorical": False, - "sql_type": "text", } result_variable = process_variable(row) self.assertEqual(result_variable, expected_variable) def test_variable_with_missing_values_field(self): - row = {"type": "nominal", "name": "Nominal Variable", "code": "NominalVar"} + row = {"type": "binary", "name": "Binary Variable", "code": "BinaryVar"} with self.assertRaises(InvalidDataModelError) as context: process_variable(row) self.assertIn( - "The 'values' should not be empty for variable NominalVar when type is 'nominal'", + "Variable BinaryVar declared as binary must have exactly two values", str(context.exception), ) def test_variable_with_optional_fields_missing(self): - row = {"type": "integer", "values": "1-100", "code": "IntVar"} + row = {"type": "discrete", "values": "1-100", "code": "IntVar"} expected_variable = { "code": "IntVar", - "type": "integer", + "type": "discrete", "minValue": 1, "maxValue": 100, - "isCategorical": False, - "sql_type": "int", } result_variable = process_variable(row) self.assertEqual(result_variable, expected_variable) @@ -118,7 +108,7 @@ def test_variable_with_optional_fields_missing(self): def test_process_variable_with_additional_fields(self): row = { - "type": "real", + "type": "continuous", "values": "10.5-20.5", "name": "Additional Field Variable", "code": "AddFieldVar", @@ -127,11 +117,9 @@ def test_process_variable_with_additional_fields(self): expected_variable = { "label": "Additional Field Variable", "code": "AddFieldVar", - "type": "real", + "type": "continuous", "minValue": 10.5, "maxValue": 20.5, - "isCategorical": False, - "sql_type": "real", } result_variable = process_variable(row) self.assertEqual(result_variable, expected_variable) diff --git a/data_quality_tool/tests/excel_to_json/test_remove_single_variable_groups.py b/data_quality_tool/tests/excel_to_json/test_remove_single_variable_groups.py index 3f3a752..efb8e85 100644 --- a/data_quality_tool/tests/excel_to_json/test_remove_single_variable_groups.py +++ b/data_quality_tool/tests/excel_to_json/test_remove_single_variable_groups.py @@ -16,8 +16,6 @@ def test_remove_empty_lists_and_strings(self): "code": "dataset", "label": "Dataset Variable", "description": "An example variable description", - "sql_type": "text", - "isCategorical": True, "enumerations": [{"code": "enum1", "label": "Enumeration 1"}], "type": "nominal", "methodology": "example methodology", @@ -44,8 +42,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "LAA", @@ -57,8 +53,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Mimic", @@ -70,8 +64,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "More than one", @@ -83,8 +75,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "PFO", @@ -96,8 +86,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "SVD", @@ -109,8 +97,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Unknown etiology despite complete evaluation", @@ -122,8 +108,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Unknown etiology with incomplete evaluation", @@ -135,8 +119,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "CE without PFO", @@ -148,8 +130,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Other", @@ -161,8 +141,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, ], }, @@ -185,8 +163,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "MCA", @@ -198,8 +174,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "PCA", @@ -211,8 +185,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Side", @@ -225,8 +197,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "3", "label": "bilateral"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Side", @@ -239,8 +209,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "3", "label": "bilateral"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Side", @@ -253,8 +221,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "3", "label": "bilateral"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, ], } @@ -270,8 +236,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Infratentorial side", @@ -284,8 +248,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "3", "label": "bilateral"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, ], }, @@ -304,8 +266,6 @@ def test_remove_empty_lists_and_strings(self): "code": "dataset", "label": "Dataset Variable", "description": "An example variable description", - "sql_type": "text", - "isCategorical": True, "enumerations": [{"code": "enum1", "label": "Enumeration 1"}], "type": "nominal", "methodology": "example methodology", @@ -332,8 +292,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "LAA", @@ -345,8 +303,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Mimic", @@ -358,8 +314,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "More than one", @@ -371,8 +325,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "PFO", @@ -384,8 +336,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "SVD", @@ -397,8 +347,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Unknown etiology despite complete evaluation", @@ -410,8 +358,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Unknown etiology with incomplete evaluation", @@ -423,8 +369,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "CE without PFO", @@ -436,8 +380,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Other", @@ -449,8 +391,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, ], }, @@ -473,8 +413,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "MCA", @@ -486,8 +424,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "PCA", @@ -499,8 +435,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Side", @@ -513,8 +447,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "3", "label": "bilateral"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Side", @@ -527,8 +459,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "3", "label": "bilateral"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Side", @@ -541,8 +471,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "3", "label": "bilateral"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, ], } @@ -558,8 +486,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "1", "label": "yes"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, { "label": "Infratentorial side", @@ -572,8 +498,6 @@ def test_remove_empty_lists_and_strings(self): {"code": "3", "label": "bilateral"}, {"code": "9", "label": "unknown"}, ], - "sql_type": "text", - "isCategorical": True, }, ], }, diff --git a/data_quality_tool/tests/excel_to_json/test_validate_variable_type.py b/data_quality_tool/tests/excel_to_json/test_validate_variable_type.py index aafa87b..5ceec41 100644 --- a/data_quality_tool/tests/excel_to_json/test_validate_variable_type.py +++ b/data_quality_tool/tests/excel_to_json/test_validate_variable_type.py @@ -1,15 +1,12 @@ import unittest -from common_entities import InvalidDataModelError -from converter.excel_to_json import ( - EXCEL_TYPE_2_SQL_TYPE_ISCATEGORICAL_MAP, - validate_variable_type, -) +from common_entities import VARIABLE_TYPES, InvalidDataModelError +from converter.excel_to_json import validate_variable_type class TestValidateVariableType(unittest.TestCase): def test_with_valid_type(self): - for valid_type in EXCEL_TYPE_2_SQL_TYPE_ISCATEGORICAL_MAP.keys(): + for valid_type in VARIABLE_TYPES: row = {"type": valid_type} validate_variable_type(row) # Should not raise an error @@ -22,6 +19,11 @@ def test_with_invalid_type(self): in str(context.exception) ) + def test_with_legacy_type(self): + for legacy_type in ["real", "integer", "text"]: + with self.assertRaises(InvalidDataModelError): + validate_variable_type({"type": legacy_type}) + def test_missing_type_field(self): row = {"name": "Variable without type"} with self.assertRaises(InvalidDataModelError) as context: diff --git a/data_quality_tool/tests/json_to_excel/test_extract_values.py b/data_quality_tool/tests/json_to_excel/test_extract_values.py index 82ea6dd..2d6035c 100644 --- a/data_quality_tool/tests/json_to_excel/test_extract_values.py +++ b/data_quality_tool/tests/json_to_excel/test_extract_values.py @@ -20,6 +20,12 @@ def test_with_min_max_values(self): expected = "1-10" self.assertEqual(extract_values(variable), expected) + def test_with_float_min_max_values(self): + self.assertEqual(extract_values({"minValue": 0.0, "maxValue": 130.0}), "0-130") + self.assertEqual( + extract_values({"minValue": 0.01, "maxValue": 99.99}), "0.01-99.99" + ) + def test_with_min_value_only(self): variable = {"minValue": "1"} expected = "1-" diff --git a/data_quality_tool/tests/json_to_excel/test_parse_variables.py b/data_quality_tool/tests/json_to_excel/test_parse_variables.py index 422fa2b..28aa031 100644 --- a/data_quality_tool/tests/json_to_excel/test_parse_variables.py +++ b/data_quality_tool/tests/json_to_excel/test_parse_variables.py @@ -8,7 +8,7 @@ def setUp(self): self.basic_variable = { "code": "V1", "label": "Variable 1", - "type": "integer", + "type": "discrete", "enumerations": [{"code": "1", "label": "One"}], "units": "units", "description": "A basic variable.", @@ -20,7 +20,7 @@ def test_complete_variable(self): "", # csvFile is not present in the basic_variable "Variable 1", "V1", - "integer", + "discrete", '{"1","One"}', "units", "A basic variable.", diff --git a/data_quality_tool/tests/test_endpoint.py b/data_quality_tool/tests/test_endpoint.py index 01c0198..acb3808 100644 --- a/data_quality_tool/tests/test_endpoint.py +++ b/data_quality_tool/tests/test_endpoint.py @@ -55,8 +55,6 @@ def test_excel_to_json_conversion(self): "label": "Nested Enumeration 1", } ], - "sql_type": "text", - "isCategorical": True, } ], } @@ -65,14 +63,12 @@ def test_excel_to_json_conversion(self): { "label": "Group Variable", "code": "group_variable", - "type": "integer", + "type": "discrete", "units": "years", "description": "A variable within a group", "methodology": "group methodology", "minValue": -10, "maxValue": 100, - "sql_type": "int", - "isCategorical": False, } ], } @@ -86,8 +82,6 @@ def test_excel_to_json_conversion(self): "description": "An example variable description", "methodology": "example methodology", "enumerations": [{"code": "enum1", "label": "Enumeration 1"}], - "sql_type": "text", - "isCategorical": True, } ], "version": "to be defined", @@ -121,7 +115,7 @@ def test_json_to_excel_conversion(self): self.assertEqual( df["code"].tolist(), ["dataset", "group_variable", "nested_group_variable"] ) - self.assertEqual(df["type"].tolist(), ["nominal", "integer", "nominal"]) + self.assertEqual(df["type"].tolist(), ["nominal", "discrete", "nominal"]) self.assertEqual( df["values"].tolist(), [ @@ -157,8 +151,6 @@ def test_json_to_excel_conversion_with_null_top_level_variables(self): { "label": "Dataset Variable", "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": [ {"code": "dataset1", "label": "Dataset 1"} @@ -200,8 +192,6 @@ def test_json_to_excel_conversion_with_null_first_level_group_variables(self): { "label": "Dataset Variable", "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": [ {"code": "dataset1", "label": "Dataset 1"} diff --git a/data_quality_tool/tests/validator/test_contains_required_dataset.py b/data_quality_tool/tests/validator/test_contains_required_dataset.py index a78922a..11b6e90 100644 --- a/data_quality_tool/tests/validator/test_contains_required_dataset.py +++ b/data_quality_tool/tests/validator/test_contains_required_dataset.py @@ -6,7 +6,13 @@ class TestContainsRequiredDataset(unittest.TestCase): def test_dataset_present_in_variables(self): # Dataset CDE is directly in the variables list - variables = [{"code": "dataset", "sql_type": "text", "isCategorical": True}] + variables = [ + { + "code": "dataset", + "type": "nominal", + "enumerations": [{"code": "d1", "label": "D1"}], + } + ] groups = [] self.assertTrue(contains_required_dataset(variables, groups)) @@ -16,16 +22,24 @@ def test_dataset_present_in_nested_group(self): groups = [ { "variables": [ - {"code": "dataset", "sql_type": "text", "isCategorical": True} + { + "code": "dataset", + "type": "nominal", + "enumerations": [{"code": "d1", "label": "D1"}], + } ], "groups": [], } ] self.assertTrue(contains_required_dataset(variables, groups)) + def test_dataset_without_enumerations_is_not_accepted(self): + variables = [{"code": "dataset", "type": "nominal"}] + self.assertFalse(contains_required_dataset(variables, [])) + def test_dataset_absent(self): # Dataset CDE is absent - variables = [{"code": "other_var", "sql_type": "real", "isCategorical": False}] + variables = [{"code": "other_var", "type": "continuous"}] groups = [] self.assertFalse(contains_required_dataset(variables, groups)) @@ -43,8 +57,8 @@ def test_dataset_deeply_nested(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, + "type": "nominal", + "enumerations": [{"code": "d1", "label": "D1"}], } ], "groups": [], @@ -66,8 +80,8 @@ def test_null_group_variables_are_treated_as_empty(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, + "type": "nominal", + "enumerations": [{"code": "d1", "label": "D1"}], } ], "groups": [], @@ -100,11 +114,21 @@ def test_deeply_nested_without_dataset(self): def test_multiple_datasets_in_different_locations(self): # Multiple dataset CDEs in different locations - variables = [{"code": "dataset", "sql_type": "text", "isCategorical": True}] + variables = [ + { + "code": "dataset", + "type": "nominal", + "enumerations": [{"code": "d1", "label": "D1"}], + } + ] groups = [ { "variables": [ - {"code": "dataset", "sql_type": "text", "isCategorical": True} + { + "code": "dataset", + "type": "nominal", + "enumerations": [{"code": "d1", "label": "D1"}], + } ], "groups": [], } @@ -119,6 +143,6 @@ def test_edge_cases_empty_lists(self): def test_misspelled_dataset_code(self): # Dataset code is misspelled - variables = [{"code": "datast", "sql_type": "text", "isCategorical": True}] + variables = [{"code": "datast", "type": "nominal"}] groups = [] self.assertFalse(contains_required_dataset(variables, groups)) diff --git a/data_quality_tool/tests/validator/test_excel_validation.py b/data_quality_tool/tests/validator/test_excel_validation.py index 7ad2a4a..11cd349 100644 --- a/data_quality_tool/tests/validator/test_excel_validation.py +++ b/data_quality_tool/tests/validator/test_excel_validation.py @@ -29,7 +29,7 @@ def test_validate_enumeration_invalid(self): ) def test_validate_min_max_valid(self): - self.assertIsNone(validate_min_max("1-2")) + self.assertEqual((1.0, 2.0), validate_min_max("1-2")) def test_validate_min_max_invalid_format(self): with self.assertRaises(InvalidDataModelError) as context: @@ -68,12 +68,19 @@ def test_validate_variable_with_min_max(self): row = { "name": "test", "code": "T01", - "type": "integer", + "type": "discrete", "conceptPath": "valid/format", "values": "10-100", } validate_variable(row) + def test_validate_variable_discrete_requires_whole_number_bounds(self): + base = {"name": "test", "code": "T01", "conceptPath": "valid/format"} + validate_variable({**base, "type": "continuous", "values": "1.5-2.5"}) + with self.assertRaises(InvalidDataModelError) as context: + validate_variable({**base, "type": "discrete", "values": "1.5-2.5"}) + self.assertIn("requires whole-number bounds", str(context.exception)) + def test_validate_variable_invalid_type(self): row = { "name": "test", @@ -85,6 +92,25 @@ def test_validate_variable_invalid_type(self): validate_variable(row) self.assertTrue("Invalid 'type'" in str(context.exception)) + def test_validate_variable_categorical_types(self): + base = {"name": "test", "code": "T01", "conceptPath": "valid/format"} + validate_variable({**base, "type": "nominal", "values": None}) + validate_variable({**base, "type": "ordinal", "values": '{"1","Low"}'}) + validate_variable( + {**base, "type": "binary", "values": '{"0","No"}, {"1","Yes"}'} + ) + for values in [None, '{"0","No"}']: + with self.assertRaises(InvalidDataModelError) as context: + validate_variable({**base, "type": "binary", "values": values}) + self.assertIn( + "Type 'binary' requires exactly two values", str(context.exception) + ) + + def test_validate_variable_legacy_type(self): + row = {"name": "test", "code": "T01", "type": "real", "conceptPath": "a/b"} + with self.assertRaises(InvalidDataModelError): + validate_variable(row) + def test_validate_excel_valid(self): df = pd.DataFrame( [ diff --git a/data_quality_tool/tests/validator/test_validate_common_data_element.py b/data_quality_tool/tests/validator/test_validate_common_data_element.py index 3d17e19..55a5628 100644 --- a/data_quality_tool/tests/validator/test_validate_common_data_element.py +++ b/data_quality_tool/tests/validator/test_validate_common_data_element.py @@ -8,8 +8,6 @@ def test_valid_common_data_element_nominal(self): # Test a valid nominal common data element with enumerations cde = { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["value1", "value2"], } @@ -19,7 +17,7 @@ def test_valid_common_data_element_nominal(self): def test_missing_required_fields(self): # Test CDE missing one or more required fields - cde_incomplete = {"code": "002", "type": "real"} + cde_incomplete = {"code": "002"} path = "/test/missing_fields" with self.assertRaises(InvalidDataModelError): validate_common_data_element(cde_incomplete, path) @@ -28,45 +26,61 @@ def test_invalid_type(self): # Test CDE with an invalid type cde_invalid_type = { "code": "003", - "sql_type": "text", - "isCategorical": False, "type": "undefined_type", } path = "/test/invalid_type" with self.assertRaises(InvalidDataModelError): validate_common_data_element(cde_invalid_type, path) - def test_mismatch_sql_type_and_isCategorical(self): - # Test CDE where sql_type or isCategorical does not match the expected values for the type - cde_mismatch = { - "code": "004", - "sql_type": "int", # Expected to be 'text' for 'nominal' - "isCategorical": True, - "type": "nominal", - } - path = "/test/mismatch" - with self.assertRaises(InvalidDataModelError): - validate_common_data_element(cde_mismatch, path) + def test_legacy_types_are_rejected(self): + for legacy_type in ["real", "integer", "text"]: + with self.assertRaises(InvalidDataModelError): + validate_common_data_element( + {"code": "004", "type": legacy_type}, "/test/legacy" + ) - def test_categorical_without_enumerations(self): - # Test CDE that is categorical but missing enumerations - cde_no_enumerations = { - "code": "005", - "sql_type": "text", - "isCategorical": True, - "type": "nominal", - } - path = "/test/no_enumerations" - with self.assertRaises(InvalidDataModelError): - validate_common_data_element(cde_no_enumerations, path) + def test_numerical_with_enumerations(self): + cde = {"code": "004", "type": "continuous", "enumerations": ["a", "b"]} + with self.assertRaisesRegex( + InvalidDataModelError, "cannot have 'enumerations'" + ): + validate_common_data_element(cde, "/test/numerical_enumerations") + + def test_categorical_with_range(self): + cde = {"code": "004", "type": "ordinal", "enumerations": ["a"], "minValue": 0} + with self.assertRaisesRegex(InvalidDataModelError, "cannot have 'minValue'"): + validate_common_data_element(cde, "/test/categorical_range") + + def test_nominal_and_ordinal_without_enumerations_are_free_text(self): + for variable_type in ["nominal", "ordinal"]: + validate_common_data_element( + {"code": "005", "type": variable_type}, "/test/free_text" + ) + + def test_binary_requires_two_enumerations(self): + for enumerations in [None, ["a"], ["a", "b", "c"]]: + cde = {"code": "005", "type": "binary", "enumerations": enumerations} + with self.assertRaisesRegex(InvalidDataModelError, "exactly two"): + validate_common_data_element(cde, "/test/binary") + validate_common_data_element( + {"code": "005", "type": "binary", "enumerations": ["a", "b"]}, + "/test/binary", + ) + + def test_discrete_requires_whole_number_range(self): + cde = {"code": "005", "type": "discrete", "minValue": 0.5, "maxValue": 3} + with self.assertRaisesRegex(InvalidDataModelError, "whole-number"): + validate_common_data_element(cde, "/test/discrete") + validate_common_data_element( + {"code": "005", "type": "discrete", "minValue": 0, "maxValue": 3}, + "/test/discrete", + ) def test_minValue_greater_than_maxValue(self): # Test CDE with minValue greater than maxValue cde_invalid_range = { "code": "006", - "sql_type": "real", - "isCategorical": False, - "type": "real", + "type": "continuous", "minValue": 10, "maxValue": 5, } diff --git a/data_quality_tool/tests/validator/test_validate_data_model.py b/data_quality_tool/tests/validator/test_validate_data_model.py index 2642c04..5c40a72 100644 --- a/data_quality_tool/tests/validator/test_validate_data_model.py +++ b/data_quality_tool/tests/validator/test_validate_data_model.py @@ -13,8 +13,6 @@ def test_valid_data_model(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["dataset1", "dataset2"], } @@ -25,8 +23,6 @@ def test_valid_data_model(self): "variables": [ { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], } @@ -37,8 +33,6 @@ def test_valid_data_model(self): "variables": [ { "code": "002", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], } @@ -84,8 +78,6 @@ def test_data_model_without_required_dataset(self): "variables": [ { "code": "000", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], } @@ -96,8 +88,6 @@ def test_data_model_without_required_dataset(self): "variables": [ { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], } @@ -106,7 +96,7 @@ def test_data_model_without_required_dataset(self): } ], } - expected_message = "The DataModel must include at least one dataset CommonDataElement with code 'dataset', 'sql_type' as 'text', and 'isCategorical' set to true." + expected_message = "The DataModel must include at least one dataset CommonDataElement with code 'dataset', 'type' as 'nominal', and 'enumerations'." with self.assertRaisesRegex(InvalidDataModelError, expected_message): validate_json(data_model_no_dataset) @@ -119,22 +109,16 @@ def test_valid_longitudinal_data_model(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["dataset1", "dataset2"], }, { "code": "subjectid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", }, { "code": "visitid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", }, ], "groups": [ @@ -143,8 +127,6 @@ def test_valid_longitudinal_data_model(self): "variables": [ { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], } @@ -165,8 +147,7 @@ def test_data_model_with_invalid_variables(self): "variables": [ { "code": "invalid_var", - "sql_type": "text", - } # Missing 'isCategorical' and 'type' + } # Missing 'type' ], "groups": [ { @@ -174,8 +155,6 @@ def test_data_model_with_invalid_variables(self): "variables": [ { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], } @@ -184,7 +163,7 @@ def test_data_model_with_invalid_variables(self): } ], } - expected_message = "Missing required field 'isCategorical' in CommonDataElement at path: '/DM006/invalid_var'. Please ensure all required fields are provided." + expected_message = "Missing required field 'type' in CommonDataElement at path: '/DM006/invalid_var'. Please ensure all required fields are provided." with self.assertRaisesRegex(InvalidDataModelError, expected_message): validate_json(data_model) @@ -197,16 +176,12 @@ def test_longitudinal_data_model_missing_required_variables(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["dataset1"], }, { "code": "visitid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", }, ], "longitudinal": True, @@ -216,8 +191,6 @@ def test_longitudinal_data_model_missing_required_variables(self): "variables": [ { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], } @@ -239,8 +212,6 @@ def test_data_model_with_deeply_nested_group_errors(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["dataset1"], } @@ -252,12 +223,12 @@ def test_data_model_with_deeply_nested_group_errors(self): } # Invalid variable configuration ], } - expected_message = "Missing required field 'sql_type' in CommonDataElement at path: '/DM008/group1/group2/deep_var'. Please ensure all required fields are provided." + expected_message = "Missing required field 'type' in CommonDataElement at path: '/DM008/group1/group2/deep_var'. Please ensure all required fields are provided." with self.assertRaisesRegex(InvalidDataModelError, expected_message): validate_json(data_model) - def test_data_model_categorical_variable_without_enumerations(self): - # Test data models where a categorical variable lacks enumerations + def test_data_model_binary_variable_without_enumerations(self): + # Test data models where a binary variable lacks enumerations data_model = { "code": "DM009", "version": "1.0", @@ -265,9 +236,7 @@ def test_data_model_categorical_variable_without_enumerations(self): "variables": [ { "code": "categorical_var", - "sql_type": "text", - "isCategorical": True, - "type": "nominal", + "type": "binary", } # Missing 'enumerations' ], @@ -277,8 +246,6 @@ def test_data_model_categorical_variable_without_enumerations(self): "variables": [ { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], } @@ -288,7 +255,7 @@ def test_data_model_categorical_variable_without_enumerations(self): ], } - expected_message = "'enumerations' is required for categorical CommonDataElement at path: '/DM009/categorical_var', but it is missing." + expected_message = "'binary' CommonDataElement at path: '/DM009/categorical_var' must have exactly two 'enumerations'." with self.assertRaisesRegex(InvalidDataModelError, expected_message): validate_json(data_model) @@ -301,8 +268,6 @@ def test_duplicate_group_codes(self): "variables": [ { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], } @@ -329,8 +294,6 @@ def test_data_model_with_deeply_nested_group_errors_updated_path(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["dataset1"], } @@ -341,13 +304,13 @@ def test_data_model_with_deeply_nested_group_errors_updated_path(self): "groups": [ { "code": "groupB", - "variables": [{"code": "deep_var", "sql_type": "int"}], + "variables": [{"code": "deep_var", "type": "integer"}], } ], } ], } - expected_message = "Missing required field 'isCategorical' in CommonDataElement at path: '/DM008/groupA/groupB/deep_var'. Please ensure all required fields are provided." + expected_message = "Invalid 'type' value 'integer' in CommonDataElement at path: '/DM008/groupA/groupB/deep_var'." with self.assertRaisesRegex(InvalidDataModelError, expected_message): validate_json(data_model) @@ -377,8 +340,6 @@ def test_variables_explicit_null_is_treated_as_empty_list(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["dataset1"], } @@ -402,8 +363,6 @@ def test_variables_empty_list(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["dataset1"], } @@ -426,8 +385,6 @@ def test_valid_data_model_without_top_level_variables(self): "variables": [ { "code": "dataset", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["dataset1"], } @@ -462,8 +419,6 @@ def test_groups_not_a_list(self): "variables": [ { "code": "V001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["example1", "example2"], # Added enumerations } @@ -485,15 +440,13 @@ def test_groups_empty_list(self): "variables": [ { "code": "V001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["example1", "example2"], # Added enumerations } ], "groups": [], # Empty list for groups } - expected_message = "The DataModel must include at least one dataset CommonDataElement with code 'dataset', 'sql_type' as 'text', and 'isCategorical' set to true." + expected_message = "The DataModel must include at least one dataset CommonDataElement with code 'dataset', 'type' as 'nominal', and 'enumerations'." with self.assertRaisesRegex( InvalidDataModelError, expected_message, @@ -508,8 +461,6 @@ def test_groups_contains_non_dictionary(self): "variables": [ { "code": "V001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", } ], diff --git a/data_quality_tool/tests/validator/test_validate_group.py b/data_quality_tool/tests/validator/test_validate_group.py index fa1f31b..5bc35fe 100644 --- a/data_quality_tool/tests/validator/test_validate_group.py +++ b/data_quality_tool/tests/validator/test_validate_group.py @@ -11,16 +11,12 @@ def test_valid_group_with_variables(self): "variables": [ { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["yes", "no"], }, { "code": "002", - "sql_type": "real", - "isCategorical": False, - "type": "real", + "type": "continuous", }, ], "groups": [], @@ -38,9 +34,7 @@ def test_valid_group_with_nested_groups(self): "variables": [ { "code": "003", - "sql_type": "int", - "isCategorical": False, - "type": "integer", + "type": "discrete", } ], "groups": [], @@ -91,8 +85,6 @@ def test_group_with_deeply_nested_groups(self): "variables": [ { "code": "101", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["A", "B", "C"], } @@ -114,9 +106,7 @@ def test_group_with_mixed_valid_and_invalid_variables(self): "variables": [ { "code": "102", - "sql_type": "int", - "isCategorical": False, - "type": "integer", + "type": "discrete", }, # Valid {"code": "103"}, # Invalid, missing required fields other than 'code' ], @@ -136,9 +126,7 @@ def test_valid_group_with_empty_variables_and_non_empty_groups(self): "variables": [ { "code": "104", - "sql_type": "real", - "isCategorical": False, - "type": "real", + "type": "continuous", } ], "groups": [], @@ -153,9 +141,7 @@ def test_group_with_all_invalid_variables(self): # Test a group where all variables are invalid group = { "variables": [ - { - "sql_type": "text" - }, # Invalid, missing 'code', 'isCategorical', and 'type' + {"type": "nominal"}, # Invalid, missing 'code' and 'type' {}, # Completely invalid ], "groups": [], @@ -172,8 +158,6 @@ def test_group_with_valid_and_invalid_nested_groups(self): "variables": [ { "code": "105", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["X", "Y"], } @@ -194,16 +178,12 @@ def test_duplicate_codes_within_same_group(self): "variables": [ { "code": "001", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["X", "Y"], }, { "code": "001", - "sql_type": "int", - "isCategorical": False, - "type": "integer", + "type": "discrete", }, ], "groups": [], @@ -221,8 +201,6 @@ def test_duplicate_codes_across_nested_groups(self): "variables": [ { "code": "002", - "sql_type": "text", - "isCategorical": True, "type": "nominal", "enumerations": ["X", "Y"], } @@ -233,9 +211,7 @@ def test_duplicate_codes_across_nested_groups(self): "variables": [ { "code": "002", - "sql_type": "real", - "isCategorical": False, - "type": "real", + "type": "continuous", } ], "groups": [], diff --git a/data_quality_tool/tests/validator/test_validate_longitudinal_elements.py b/data_quality_tool/tests/validator/test_validate_longitudinal_elements.py index f04794c..54eac60 100644 --- a/data_quality_tool/tests/validator/test_validate_longitudinal_elements.py +++ b/data_quality_tool/tests/validator/test_validate_longitudinal_elements.py @@ -12,15 +12,11 @@ def test_longitudinal_elements_at_top_level(self): variables = [ { "code": "subjectid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", }, { "code": "visitid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", }, ] groups = [] @@ -35,15 +31,11 @@ def test_longitudinal_elements_within_nested_group(self): "variables": [ { "code": "subjectid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", }, { "code": "visitid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", }, ], "groups": [], @@ -65,9 +57,7 @@ def test_longitudinal_elements_spread_across_levels(self): variables = [ { "code": "subjectid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", } ] groups = [ @@ -75,9 +65,7 @@ def test_longitudinal_elements_spread_across_levels(self): "variables": [ { "code": "visitid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", } ], "groups": [], @@ -91,9 +79,7 @@ def test_multiple_instances_of_longitudinal_elements(self): variables = [ { "code": "subjectid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", } ] groups = [ @@ -101,15 +87,11 @@ def test_multiple_instances_of_longitudinal_elements(self): "variables": [ { "code": "visitid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", }, { "code": "subjectid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", }, # Another instance ], "groups": [], @@ -122,9 +104,7 @@ def test_longitudinal_elements_missing_visit_id(self): variables = [ { "code": "subjectid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", } ] groups = [] @@ -137,9 +117,7 @@ def test_deeply_nested_longitudinal_elements(self): variables = [ { "code": "subjectid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", } ] groups = [ @@ -153,9 +131,7 @@ def test_deeply_nested_longitudinal_elements(self): "variables": [ { "code": "visitid", - "sql_type": "text", - "isCategorical": False, - "type": "text", + "type": "nominal", } ], "groups": [], diff --git a/data_quality_tool/validator/excel_validator.py b/data_quality_tool/validator/excel_validator.py index e1b3e36..1f09e2e 100644 --- a/data_quality_tool/validator/excel_validator.py +++ b/data_quality_tool/validator/excel_validator.py @@ -6,7 +6,9 @@ InvalidDataModelError, REQUIRED_COLUMNS, EXCEL_COLUMNS, - EXCEL_TYPE_2_SQL_TYPE_ISCATEGORICAL_MAP, + CATEGORICAL_TYPES, + NUMERICAL_TYPES, + VARIABLE_TYPES, ) # Regex for validation @@ -42,6 +44,7 @@ def validate_enumerations(values): raise InvalidDataModelError( f"Duplicate codes found in enumeration values {codes=}." ) + return codes def validate_min_max(values): @@ -67,6 +70,7 @@ def validate_min_max(values): raise InvalidDataModelError( f"Min value must be smaller than max value {min_str=} and {max_str=}." ) + return min_val, max_val def validate_concept_path(concept_path): @@ -80,16 +84,27 @@ def validate_concept_path(concept_path): def validate_variable_type(row): """Validate the type and values of a variable based on its type.""" type_val = row.get("type") - valid_excel_types = EXCEL_TYPE_2_SQL_TYPE_ISCATEGORICAL_MAP.keys() - if type_val not in valid_excel_types: - valid_types_str = ", ".join(valid_excel_types) + if type_val not in VARIABLE_TYPES: + valid_types_str = ", ".join(VARIABLE_TYPES) raise InvalidDataModelError( f"Invalid 'type': {type_val}. Valid types: {valid_types_str}." ) - if type_val == "nominal": - validate_enumerations(row.get("values", "")) - elif type_val in ["real", "integer"] and row.get("values"): - validate_min_max(row["values"]) + if type_val == "binary" and not row.get("values"): + raise InvalidDataModelError("Type 'binary' requires exactly two values.") + if type_val in CATEGORICAL_TYPES and row.get("values"): + codes = validate_enumerations(row["values"]) + if type_val == "binary" and len(codes) != 2: + raise InvalidDataModelError( + f"Type 'binary' requires exactly two values but got {codes=}." + ) + elif type_val in NUMERICAL_TYPES and row.get("values"): + min_val, max_val = validate_min_max(row["values"]) + if type_val == "discrete" and not ( + min_val.is_integer() and max_val.is_integer() + ): + raise InvalidDataModelError( + f"Type 'discrete' requires whole-number bounds but got '{row['values']}'." + ) def validate_variable(row): diff --git a/data_quality_tool/validator/json_validator.py b/data_quality_tool/validator/json_validator.py index 640abd6..cd3b9ef 100644 --- a/data_quality_tool/validator/json_validator.py +++ b/data_quality_tool/validator/json_validator.py @@ -1,15 +1,13 @@ -from common_entities import InvalidDataModelError - -TYPE_2_SQL = { - "nominal": ("text", True), - "real": ("real", False), - "integer": ("int", False), - "text": ("text", False), -} +from common_entities import ( + CATEGORICAL_TYPES, + NUMERICAL_TYPES, + VARIABLE_TYPES, + InvalidDataModelError, +) def validate_common_data_element(cde, path): - required_fields = ["code", "sql_type", "isCategorical", "type"] + required_fields = ["code", "type"] for field in required_fields: if field not in cde: @@ -17,23 +15,30 @@ def validate_common_data_element(cde, path): f"Missing required field '{field}' in CommonDataElement at path: '{path}'. Please ensure all required fields are provided." ) type_key = cde.get("type") - if type_key not in TYPE_2_SQL: + if type_key not in VARIABLE_TYPES: + raise InvalidDataModelError( + f"Invalid 'type' value '{type_key}' in CommonDataElement at path: '{path}'. Must be one of {list(VARIABLE_TYPES)}." + ) + + bounds = [cde[key] for key in ("minValue", "maxValue") if cde.get(key) is not None] + if type_key in CATEGORICAL_TYPES and bounds: + raise InvalidDataModelError( + f"Categorical type '{type_key}' cannot have 'minValue' or 'maxValue' in CommonDataElement at path: '{path}'." + ) + + if type_key in NUMERICAL_TYPES and cde.get("enumerations"): raise InvalidDataModelError( - f"Invalid 'type' value '{type_key}' in CommonDataElement at path: '{path}'. Must be one of {list(TYPE_2_SQL.keys())}." + f"Numerical type '{type_key}' cannot have 'enumerations' in CommonDataElement at path: '{path}'." ) - expected_sql_type, expected_is_categorical = TYPE_2_SQL[type_key] - if ( - cde.get("sql_type") != expected_sql_type - or cde.get("isCategorical") != expected_is_categorical - ): + if type_key == "binary" and len(cde.get("enumerations") or []) != 2: raise InvalidDataModelError( - f"Incorrect 'sql_type' or 'isCategorical' for type '{type_key}' in CommonDataElement at path: '{path}'. Expected ('{expected_sql_type}', {expected_is_categorical}), but got ('{cde.get('sql_type')}', {cde.get('isCategorical')})." + f"'binary' CommonDataElement at path: '{path}' must have exactly two 'enumerations'." ) - if cde.get("isCategorical") and not cde.get("enumerations"): + if type_key == "discrete" and any(float(bound) % 1 for bound in bounds): raise InvalidDataModelError( - f"'enumerations' is required for categorical CommonDataElement at path: '{path}', but it is missing." + f"'discrete' CommonDataElement at path: '{path}' must have whole-number 'minValue' and 'maxValue'." ) if cde.get("minValue") is not None and cde.get("maxValue") is not None: @@ -127,7 +132,7 @@ def validate_json(data_model): if not contains_required_dataset(data_model["variables"], data_model["groups"]): raise InvalidDataModelError( - "The DataModel must include at least one dataset CommonDataElement with code 'dataset', 'sql_type' as 'text', and 'isCategorical' set to true." + "The DataModel must include at least one dataset CommonDataElement with code 'dataset', 'type' as 'nominal', and 'enumerations'." ) if data_model.get("longitudinal"): @@ -144,8 +149,8 @@ def contains_required_dataset(variables, groups, path=""): dataset_present = any( v.get("code") == "dataset" - and v.get("sql_type") == "text" - and v.get("isCategorical") + and v.get("type") == "nominal" + and v.get("enumerations") for v in variables ) if dataset_present: diff --git a/frontend/src/app/interfaces/pathology.interface.ts b/frontend/src/app/interfaces/pathology.interface.ts index d2816c3..49db930 100644 --- a/frontend/src/app/interfaces/pathology.interface.ts +++ b/frontend/src/app/interfaces/pathology.interface.ts @@ -3,8 +3,6 @@ export interface Variable { label: string; code?: string; description?: string; - sql_type?: string; - isCategorical?: boolean; enumerations?: any[]; type?: string; methodology?: string; @@ -40,8 +38,6 @@ export interface D3HierarchyNode { value?: number; // Optional, as this may only be used for leaf nodes code?: string; description?: string; - sql_type?: string; - isCategorical?: boolean; enumerations?: any[]; type?: string; methodology?: string; diff --git a/frontend/src/app/pages/pathology-page/visualization/tidy-tree.ts b/frontend/src/app/pages/pathology-page/visualization/tidy-tree.ts index 2f27a45..0a92b7f 100644 --- a/frontend/src/app/pages/pathology-page/visualization/tidy-tree.ts +++ b/frontend/src/app/pages/pathology-page/visualization/tidy-tree.ts @@ -120,7 +120,7 @@ const buildTooltipHtml = (node: TreeDatum): string => { addRow('Maximum', node.max); const enumerationMarkup = - node.type === 'nominal' && node.enumerations?.length + node.enumerations?.length ? `
Enumerations