From 5a5f56fae394fa734bc9b999004c3789f54cc479 Mon Sep 17 00:00:00 2001 From: Reinhard Munz Date: Tue, 8 Mar 2022 23:42:24 +0100 Subject: [PATCH 1/8] fixed metadata to have spec in front --- ...lt_with_rand_names_datatopredict_full.yaml | 2 +- ..._with_rand_names_datatopredict_simple.yaml | 2 +- .../adult_with_rand_names_minimal_full.yaml | 4 ++-- .../adult_with_rand_names_minimal_simple.yaml | 4 ++-- .../adult_with_rand_names_regular_full.yaml | 11 +--------- .../adult_with_rand_names_regular_simple.yaml | 8 +------- .../adult_with_rand_names_w_header_full.yaml | 20 +------------------ ...adult_with_rand_names_w_header_simple.yaml | 7 +------ 8 files changed, 10 insertions(+), 48 deletions(-) diff --git a/dq0/examples/census/_data/adult_with_rand_names_datatopredict_full.yaml b/dq0/examples/census/_data/adult_with_rand_names_datatopredict_full.yaml index e304302d..230f521d 100644 --- a/dq0/examples/census/_data/adult_with_rand_names_datatopredict_full.yaml +++ b/dq0/examples/census/_data/adult_with_rand_names_datatopredict_full.yaml @@ -1,5 +1,6 @@ meta_dataset: format: 'full' + specification: 'dataset_v1' node: type_name: 'dataset' attributes: @@ -584,4 +585,3 @@ meta_dataset: - type_name: 'boolean' key: 'synthesizable' value: true - specification: 'dataset_v1' diff --git a/dq0/examples/census/_data/adult_with_rand_names_datatopredict_simple.yaml b/dq0/examples/census/_data/adult_with_rand_names_datatopredict_simple.yaml index f0007256..feef1356 100644 --- a/dq0/examples/census/_data/adult_with_rand_names_datatopredict_simple.yaml +++ b/dq0/examples/census/_data/adult_with_rand_names_datatopredict_simple.yaml @@ -1,5 +1,6 @@ meta_dataset: format: simple + specification: 'dataset_v1' node: dataset: attributes: @@ -230,4 +231,3 @@ meta_dataset: 'cardinality': 9 'private_synthesis': 'synthesizable': true - specification: 'dataset_v1' diff --git a/dq0/examples/census/_data/adult_with_rand_names_minimal_full.yaml b/dq0/examples/census/_data/adult_with_rand_names_minimal_full.yaml index 98a5187d..3043e63b 100644 --- a/dq0/examples/census/_data/adult_with_rand_names_minimal_full.yaml +++ b/dq0/examples/census/_data/adult_with_rand_names_minimal_full.yaml @@ -1,5 +1,6 @@ meta_dataset: format: 'full' + specification: 'dataset_v1' node: type_name: 'dataset' attributes: @@ -29,7 +30,7 @@ meta_dataset: value: 'csv' - type_name: 'string' key: 'uri' - value: "/Users/wolfganggross/code/dq0-sdk/dq0/examples/census/_data/adult_with_rand_names.csv" + value: "../dq0-sdk/dq0/examples/census/_data/adult_with_rand_names.csv" - type_name: 'list' key: 'data' value: @@ -54,4 +55,3 @@ meta_dataset: - type_name: 'string' key: 'name' value: 'ACI Table' - specification: 'dataset_v1' diff --git a/dq0/examples/census/_data/adult_with_rand_names_minimal_simple.yaml b/dq0/examples/census/_data/adult_with_rand_names_minimal_simple.yaml index e22e022a..9c4dd28d 100644 --- a/dq0/examples/census/_data/adult_with_rand_names_minimal_simple.yaml +++ b/dq0/examples/census/_data/adult_with_rand_names_minimal_simple.yaml @@ -1,5 +1,6 @@ meta_dataset: format: simple + specification: 'dataset_v1' node: dataset: attributes: @@ -13,7 +14,6 @@ meta_dataset: attributes: 'connector': 'type_name': 'csv' - 'uri': "/Users/wolfganggross/code/dq0-sdk/dq0/examples/census/_data/adult_with_rand_names.csv" + 'uri': "../dq0-sdk/dq0/examples/census/_data/adult_with_rand_names.csv" 'data': 'name': "adult database" - specification: 'dataset_v1' diff --git a/dq0/examples/census/_data/adult_with_rand_names_regular_full.yaml b/dq0/examples/census/_data/adult_with_rand_names_regular_full.yaml index 9214acb0..3827597d 100644 --- a/dq0/examples/census/_data/adult_with_rand_names_regular_full.yaml +++ b/dq0/examples/census/_data/adult_with_rand_names_regular_full.yaml @@ -1,5 +1,6 @@ meta_dataset: format: 'full' + specification: 'dataset_v1' node: type_name: 'dataset' attributes: @@ -328,15 +329,6 @@ meta_dataset: - type_name: 'boolean' key: 'is_feature' value: true - # - type_name: 'list' - # key: 'private_sql_and_synthesis' - # value: - # - type_name: 'int' - # key: 'lower' - # value: 39824 - # - type_name: 'int' - # key: 'upper' - # value: 334679 - type_name: 'list' key: 'private_synthesis' value: @@ -642,4 +634,3 @@ meta_dataset: - type_name: 'boolean' key: 'synthesizable' value: true - specification: 'dataset_v1' diff --git a/dq0/examples/census/_data/adult_with_rand_names_regular_simple.yaml b/dq0/examples/census/_data/adult_with_rand_names_regular_simple.yaml index 9271df22..199ffd18 100644 --- a/dq0/examples/census/_data/adult_with_rand_names_regular_simple.yaml +++ b/dq0/examples/census/_data/adult_with_rand_names_regular_simple.yaml @@ -1,5 +1,6 @@ meta_dataset: format: simple + specification: 'dataset_v1' node: dataset: attributes: @@ -111,9 +112,6 @@ meta_dataset: 'name': 'education-num' 'machine_learning': 'is_feature': true - # 'private_sql_and_synthesis': - # 'lower': 6 - # 'upper': 14 'private_sql_and_synthesis': 'cardinality': 16 'private_synthesis': @@ -136,9 +134,6 @@ meta_dataset: 'name': 'fnlwgt' 'machine_learning': 'is_feature': true - # 'private_sql_and_synthesis': - # 'lower': 39824 - # 'upper': 334679 'private_sql_and_synthesis': 'cardinality': 28521 'private_synthesis': @@ -254,4 +249,3 @@ meta_dataset: 'cardinality': 9 'private_synthesis': 'synthesizable': true - specification: 'dataset_v1' diff --git a/dq0/examples/census/_data/adult_with_rand_names_w_header_full.yaml b/dq0/examples/census/_data/adult_with_rand_names_w_header_full.yaml index 3d119939..33521e9e 100644 --- a/dq0/examples/census/_data/adult_with_rand_names_w_header_full.yaml +++ b/dq0/examples/census/_data/adult_with_rand_names_w_header_full.yaml @@ -1,5 +1,6 @@ meta_dataset: format: 'full' + specification: 'dataset_v1' node: type_name: 'dataset' attributes: @@ -209,15 +210,6 @@ meta_dataset: - type_name: 'boolean' key: 'is_feature' value: true - # - type_name: 'list' - # key: 'private_sql_and_synthesis' - # value: - # - type_name: 'int' - # key: 'lower' - # value: 6 - # - type_name: 'int' - # key: 'upper' - # value: 14 - type_name: 'list' key: 'private_synthesis' value: @@ -276,15 +268,6 @@ meta_dataset: - type_name: 'boolean' key: 'is_feature' value: true - # - type_name: 'list' - # key: 'private_sql_and_synthesis' - # value: - # - type_name: 'int' - # key: 'lower' - # value: 45868 - # - type_name: 'int' - # key: 'upper' - # value: 349148 - type_name: 'list' key: 'private_synthesis' value: @@ -590,4 +573,3 @@ meta_dataset: - type_name: 'boolean' key: 'synthesizable' value: true - specification: 'dataset_v1' diff --git a/dq0/examples/census/_data/adult_with_rand_names_w_header_simple.yaml b/dq0/examples/census/_data/adult_with_rand_names_w_header_simple.yaml index 7d313a70..a94fb811 100644 --- a/dq0/examples/census/_data/adult_with_rand_names_w_header_simple.yaml +++ b/dq0/examples/census/_data/adult_with_rand_names_w_header_simple.yaml @@ -1,5 +1,6 @@ meta_dataset: format: simple + specification: 'dataset_v1' node: dataset: attributes: @@ -85,9 +86,6 @@ meta_dataset: 'name': 'education-num' 'machine_learning': 'is_feature': true - # 'private_sql_and_synthesis': - # 'lower': 6 - # 'upper': 14 'private_synthesis': 'synthesizable': false 'private_sql_and_synthesis': @@ -111,8 +109,6 @@ meta_dataset: 'machine_learning': 'is_feature': true 'private_sql_and_synthesis': - # 'lower': 45868 - # 'upper': 349148 'cardinality': 28521 'private_synthesis': 'synthesizable': false @@ -227,4 +223,3 @@ meta_dataset: 'cardinality': 9 'private_synthesis': 'synthesizable': true - specification: 'dataset_v1' From d31f00fddda5d9d3b53f55cfa9627981e4ffecd7 Mon Sep 17 00:00:00 2001 From: Reinhard Munz Date: Tue, 15 Mar 2022 08:02:36 +0100 Subject: [PATCH 2/8] added run metadata and new connectors --- dq0/sdk/connector/__init__.py | 0 dq0/sdk/connector/connector.py | 17 ++ dq0/sdk/connector/connector_factory.py | 70 ++++++ dq0/sdk/connector/connector_kind.py | 18 ++ dq0/sdk/connector/kind/__init__.py | 0 dq0/sdk/connector/kind/csv/__init__.py | 0 dq0/sdk/connector/kind/csv/csv.py | 52 +++++ dq0/sdk/connector/kind/sql/__init__.py | 0 .../connector/kind/sql/db/mysql/__init__.py | 0 dq0/sdk/connector/kind/sql/db/mysql/mysql.py | 34 +++ .../kind/sql/db/mysql/mysql_query.py | 20 ++ .../kind/sql/db/postgresql/__init__.py | 0 .../kind/sql/db/postgresql/postgresql.py | 33 +++ .../sql/db/postgresql/postgresql_query.py | 20 ++ .../connector/kind/sql/db/sqlite/__init__.py | 0 .../connector/kind/sql/db/sqlite/sqlite.py | 34 +++ .../kind/sql/db/sqlite/sqlite_query.py | 20 ++ dq0/sdk/connector/kind/sql/query.py | 18 ++ dq0/sdk/connector/kind/sql/sql.py | 12 + .../{dataset => }/attribute_utils.py | 0 .../{dataset => }/attributes_group.py | 40 +++- .../v1/column/attributes_column_data.py | 2 +- .../attributes_column_machine_learning.py | 2 +- .../column/attributes_column_private_sql.py | 4 +- ...ibutes_column_private_sql_and_synthesis.py | 2 +- .../attributes_column_private_synthesis.py | 2 +- .../interface/dataset/v1/column/column.py | 2 +- .../database/attributes_database_connector.py | 209 ++++++++++-------- .../v1/database/attributes_database_data.py | 2 +- ...ttributes_database_differential_privacy.py | 2 +- .../interface/dataset/v1/database/database.py | 2 +- .../v1/dataset/attributes_dataset_data.py | 4 +- ...attributes_dataset_differential_privacy.py | 2 +- .../interface/dataset/v1/dataset/dataset.py | 2 +- .../v1/schema/attributes_schema_data.py | 2 +- .../attributes_schema_differential_privacy.py | 2 +- .../interface/dataset/v1/schema/schema.py | 2 +- .../dataset/v1/table/attributes_table_data.py | 2 +- .../attributes_table_differential_privacy.py | 2 +- .../v1/table/attributes_table_private_sql.py | 2 +- .../attributes_table_private_synthesis.py | 2 +- .../interface/dataset/v1/table/table.py | 2 +- .../interface/{dataset => }/entity.py | 2 +- .../{dataset => }/entity_iterator.py | 0 dq0/sdk/data/metadata/interface/interface.py | 66 ++++-- .../data/metadata/interface/run/__init__.py | 0 .../metadata/interface/run/v1/__init__.py | 0 .../metadata/interface/run/v1/run/__init__.py | 0 .../run/v1/run/attributes_run_data.py | 62 ++++++ .../run/v1/run/attributes_run_sql.py | 64 ++++++ .../run/attributes_run_sql_query_processor.py | 28 +++ .../attributes_run_sql_result_processor.py | 28 +++ .../data/metadata/interface/run/v1/run/run.py | 64 ++++++ .../dataset/v1/column_boolean.py | 4 +- .../dataset/v1/column_datetime.py | 4 +- .../specification/dataset/v1/column_float.py | 4 +- .../specification/dataset/v1/column_int.py | 4 +- .../specification/dataset/v1/column_string.py | 4 +- .../specification/dataset/v1/connector.py | 10 + .../specification/dataset/v1/connector_csv.py | 17 +- .../dataset/v1/connector_mysql.py | 107 +++++++++ .../dataset/v1/connector_postgresql.py | 17 +- .../dataset/v1/connector_sqlite.py | 77 +++++++ .../specification/dataset/v1/database.py | 22 +- .../specification/dataset/v1/dataset.py | 16 +- .../specification/dataset/v1/schema.py | 16 +- .../specification/dataset/v1/table.py | 19 +- .../specification/default_permissions.py | 13 ++ .../specification/json_schema/attribute.py | 38 ---- .../json_schema/attributes_group.py | 70 +----- .../json_schema/dataset/__init__.py | 0 .../json_schema/dataset/attribute.py | 42 ++++ .../json_schema/dataset/attributes_group.py | 66 ++++++ .../json_schema/{ => dataset}/column.py | 25 ++- .../specification/json_schema/metadata.py | 21 ++ .../specification/json_schema/run/__init__.py | 0 .../json_schema/run/attribute.py | 49 ++++ .../json_schema/run/attributes_group.py | 77 +++++++ .../metadata/specification/run/__init__.py | 0 .../run/specification_version.py | 12 + .../metadata/specification/run/v1/__init__.py | 0 .../data/metadata/specification/run/v1/run.py | 126 +++++++++++ .../specification/run/v1/specification_v1.py | 26 +++ .../specification/specification_factory.py | 17 +- dq0/sdk/data/metadata/structure/metadata.py | 2 +- .../data/metadata/structure/node/node_type.py | 6 +- 86 files changed, 1544 insertions(+), 322 deletions(-) create mode 100644 dq0/sdk/connector/__init__.py create mode 100644 dq0/sdk/connector/connector.py create mode 100644 dq0/sdk/connector/connector_factory.py create mode 100644 dq0/sdk/connector/connector_kind.py create mode 100644 dq0/sdk/connector/kind/__init__.py create mode 100644 dq0/sdk/connector/kind/csv/__init__.py create mode 100644 dq0/sdk/connector/kind/csv/csv.py create mode 100644 dq0/sdk/connector/kind/sql/__init__.py create mode 100644 dq0/sdk/connector/kind/sql/db/mysql/__init__.py create mode 100644 dq0/sdk/connector/kind/sql/db/mysql/mysql.py create mode 100644 dq0/sdk/connector/kind/sql/db/mysql/mysql_query.py create mode 100644 dq0/sdk/connector/kind/sql/db/postgresql/__init__.py create mode 100644 dq0/sdk/connector/kind/sql/db/postgresql/postgresql.py create mode 100644 dq0/sdk/connector/kind/sql/db/postgresql/postgresql_query.py create mode 100644 dq0/sdk/connector/kind/sql/db/sqlite/__init__.py create mode 100644 dq0/sdk/connector/kind/sql/db/sqlite/sqlite.py create mode 100644 dq0/sdk/connector/kind/sql/db/sqlite/sqlite_query.py create mode 100644 dq0/sdk/connector/kind/sql/query.py create mode 100644 dq0/sdk/connector/kind/sql/sql.py rename dq0/sdk/data/metadata/interface/{dataset => }/attribute_utils.py (100%) rename dq0/sdk/data/metadata/interface/{dataset => }/attributes_group.py (66%) rename dq0/sdk/data/metadata/interface/{dataset => }/entity.py (99%) rename dq0/sdk/data/metadata/interface/{dataset => }/entity_iterator.py (100%) create mode 100644 dq0/sdk/data/metadata/interface/run/__init__.py create mode 100644 dq0/sdk/data/metadata/interface/run/v1/__init__.py create mode 100644 dq0/sdk/data/metadata/interface/run/v1/run/__init__.py create mode 100644 dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_data.py create mode 100644 dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql.py create mode 100644 dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_query_processor.py create mode 100644 dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py create mode 100644 dq0/sdk/data/metadata/interface/run/v1/run/run.py create mode 100644 dq0/sdk/data/metadata/specification/dataset/v1/connector_mysql.py create mode 100644 dq0/sdk/data/metadata/specification/dataset/v1/connector_sqlite.py create mode 100644 dq0/sdk/data/metadata/specification/json_schema/dataset/__init__.py create mode 100644 dq0/sdk/data/metadata/specification/json_schema/dataset/attribute.py create mode 100644 dq0/sdk/data/metadata/specification/json_schema/dataset/attributes_group.py rename dq0/sdk/data/metadata/specification/json_schema/{ => dataset}/column.py (69%) create mode 100644 dq0/sdk/data/metadata/specification/json_schema/run/__init__.py create mode 100644 dq0/sdk/data/metadata/specification/json_schema/run/attribute.py create mode 100644 dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py create mode 100644 dq0/sdk/data/metadata/specification/run/__init__.py create mode 100644 dq0/sdk/data/metadata/specification/run/specification_version.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/__init__.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/run.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/specification_v1.py diff --git a/dq0/sdk/connector/__init__.py b/dq0/sdk/connector/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/connector/connector.py b/dq0/sdk/connector/connector.py new file mode 100644 index 00000000..520f7df6 --- /dev/null +++ b/dq0/sdk/connector/connector.py @@ -0,0 +1,17 @@ +from dq0.sdk.connector.connector_kind import ConnectorKind + + +class Connector: + def __init__(self, kind): + if not ConnectorKind.is_valid(kind=kind): + raise ValueError(f"kind {kind} is not valid") + self._kind = kind + + def get_kind(self): + return self._kind + + def to_pandas(self): + raise NotImplementedError + + def new_query(self, query_string): + raise NotImplementedError diff --git a/dq0/sdk/connector/connector_factory.py b/dq0/sdk/connector/connector_factory.py new file mode 100644 index 00000000..c437a068 --- /dev/null +++ b/dq0/sdk/connector/connector_factory.py @@ -0,0 +1,70 @@ +from dq0.sdk.connector.kind.csv.csv import CSV +from dq0.sdk.connector.kind.sql.db.mysql.mysql import MySQL +from dq0.sdk.connector.kind.sql.db.postgresql.postgresql import PostgreSQL +from dq0.sdk.connector.kind.sql.db.sqlite.sqlite import SQLite + + +class ConnectorFactory: + @staticmethod + def connector_from(meta_database): + meta_connector = meta_database.connector + type_name = meta_connector.type_name + if type_name == 'csv': + return ConnectorFactory.csv_from(meta_connector=meta_connector) + if type_name == 'mysql': + return ConnectorFactory.mysql_from(meta_database=meta_database) + if type_name == 'postgresql': + return ConnectorFactory.postgresql_from(meta_database=meta_database) + if type_name == 'sqlite': + return ConnectorFactory.sqlite_from(meta_connector=meta_connector) + raise ValueError(f"type_name={type_name} is unknown") + + @staticmethod + def csv_from(meta_connector): + if meta_connector.type_name != 'csv': + raise ValueError(f"type_name={meta_connector.type_name} != csv") + filepath = meta_connector.uri + if len(filepath) == 0: + raise ValueError("uri not specified") + sep = meta_connector.sep if meta_connector.sep is not None else ',' + header = meta_connector.header_row if meta_connector.header_row is not None else 'infer' + names = meta_connector.header_columns + index_col = meta_connector.index_col + skipinitialspace = meta_connector.skipinitialspace \ + if meta_connector.skipinitialspace is not None else False + na_values = meta_connector.na_values + decimal = meta_connector.decimal if meta_connector.decimal is not None else '.' + return CSV(filepath=filepath, sep=sep, header=header, names=names, index_col=index_col, + skipinitialspace=skipinitialspace, na_values=na_values, decimal=decimal) + + @staticmethod + def mysql_from(meta_database): + meta_connector = meta_database.connector + if meta_connector.type_name != 'mysql': + raise ValueError(f"type_name={meta_connector.type_name} != mysql") + username = meta_connector.username if meta_connector.username is not None else '' + password = meta_connector.password if meta_connector.password is not None else '' + host = meta_connector.host if meta_connector.host is not None else '' + port = meta_connector.port if meta_connector.port is not None else '' + database = meta_database.data.name if meta_database.data.name is not None else '' + charset = meta_connector.charset if meta_connector.charset is not None else '' + return MySQL(username=username, password=password, host=host, port=port, database=database, charset=charset) + + @staticmethod + def postgresql_from(meta_database): + meta_connector = meta_database.connector + if meta_connector.type_name != 'postgresql': + raise ValueError(f"type_name={meta_connector.type_name} != postgresql") + username = meta_connector.username if meta_connector.username is not None else '' + password = meta_connector.password if meta_connector.password is not None else '' + host = meta_connector.host if meta_connector.host is not None else '' + port = meta_connector.port if meta_connector.port is not None else '' + database = meta_database.data.name if meta_database.data.name is not None else '' + return PostgreSQL(username=username, password=password, host=host, port=port, database=database) + + @staticmethod + def sqlite_from(meta_connector): + if meta_connector.type_name != 'sqlite': + raise ValueError(f"type_name={meta_connector.type_name} != sqlite") + uri = meta_connector.uri if meta_connector.uri is not None else '' + return SQLite(filepath=uri) diff --git a/dq0/sdk/connector/connector_kind.py b/dq0/sdk/connector/connector_kind.py new file mode 100644 index 00000000..496b0ea1 --- /dev/null +++ b/dq0/sdk/connector/connector_kind.py @@ -0,0 +1,18 @@ +class ConnectorKind: + CSV = 'csv' + SQL_MYSQL = 'sql_mysql' + SQL_POSTGRESQL = 'sql_postgresql' + SQL_SQLITE = 'sql_sqlite' + + @staticmethod + def is_valid(kind): + return \ + kind == ConnectorKind.CSV or \ + ConnectorKind.is_valid_sql(kind=kind) + + @staticmethod + def is_valid_sql(kind): + return \ + kind == ConnectorKind.SQL_MYSQL or \ + kind == ConnectorKind.SQL_POSTGRESQL or \ + kind == ConnectorKind.SQL_SQLITE diff --git a/dq0/sdk/connector/kind/__init__.py b/dq0/sdk/connector/kind/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/connector/kind/csv/__init__.py b/dq0/sdk/connector/kind/csv/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/connector/kind/csv/csv.py b/dq0/sdk/connector/kind/csv/csv.py new file mode 100644 index 00000000..b93b9680 --- /dev/null +++ b/dq0/sdk/connector/kind/csv/csv.py @@ -0,0 +1,52 @@ +from pathlib import Path + +from dq0.sdk.connector.connector import Connector +from dq0.sdk.connector.connector_kind import ConnectorKind + +import pandas + + +class CSV(Connector): + def __init__(self, filepath, sep=',', header='infer', names=None, index_col=None, + skipinitialspace=False, na_values=None, decimal='.'): + super().__init__(ConnectorKind.CSV) + self._filepath = Path(filepath) + if not self._filepath.is_file(): + raise ValueError(f"filepath={filepath} is not a file") + self._sep = sep + self._header = header + self._names = names + self._index_col = index_col + self._skipinitialspace = skipinitialspace + self._na_values = na_values + self._decimal = decimal + + def get_filepath(self): + return self._filepath + + def get_sep(self): + return self._sep + + def get_header(self): + return self._header + + def get_names(self): + return self._names + + def get_index_col(self): + return self._index_col + + def get_skipinitialspace(self): + return self._skipinitialspace + + def get_na_values(self): + return self._na_values + + def get_decimal(self): + return self._decimal + + def to_pandas(self): + return pandas.read_csv(filepath=self._filepath, sep=self._sep, header=self._header, + names=self._names, index_col=self._index_col, + skipinitialspace=self._skipinitialspace, na_values=self._na_values, + decimal=self._decimal) diff --git a/dq0/sdk/connector/kind/sql/__init__.py b/dq0/sdk/connector/kind/sql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/connector/kind/sql/db/mysql/__init__.py b/dq0/sdk/connector/kind/sql/db/mysql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/connector/kind/sql/db/mysql/mysql.py b/dq0/sdk/connector/kind/sql/db/mysql/mysql.py new file mode 100644 index 00000000..d36a687c --- /dev/null +++ b/dq0/sdk/connector/kind/sql/db/mysql/mysql.py @@ -0,0 +1,34 @@ +from dq0.sdk.connector.connector_kind import ConnectorKind +from dq0.sdk.connector.kind.sql.db.mysql.mysql_query import MySQLQuery +from dq0.sdk.connector.kind.sql.sql import SQL + +import sqlalchemy + + +class MySQL(SQL): + @staticmethod + def connection_uri(username='', password='', host='', port='', database='', charset=''): + if len(username) == 0: + password = '' + if len(host) == 0: + raise Exception("host not provided") + password_sep = ':' if 0 < len(password) else '' + user_sep = '@' if 0 < len(username) else '' + port_sep = ':' if 0 < len(port) else '' + database_sep = '/' if 0 < len(database) else '' + charset_sep = '?charset=' if 0 < len(charset) else '' + return f"mysql+mysqlconnector://{username}{password_sep}{password}{user_sep}{host}{port_sep}{port}{database_sep}{database}{charset_sep}{charset}" + + def __init__(self, username='', password='', host='', port='', database='', charset=''): + super().__init__(ConnectorKind.SQL_MYSQL) + self._connection_uri = MySQL.connection_uri(username=username, password=password, host=host, port=port, database=database, charset=charset) + self._engine = sqlalchemy.create_engine(self.get_connection_uri()) + + def get_connection_uri(self): + return self._connection_uri + + def get_engine(self): + return self._engine + + def new_query(self, query_string): + return MySQLQuery(connector=self, query_string=query_string) diff --git a/dq0/sdk/connector/kind/sql/db/mysql/mysql_query.py b/dq0/sdk/connector/kind/sql/db/mysql/mysql_query.py new file mode 100644 index 00000000..868767a6 --- /dev/null +++ b/dq0/sdk/connector/kind/sql/db/mysql/mysql_query.py @@ -0,0 +1,20 @@ +from dq0.sdk.connector.connector_kind import ConnectorKind +from dq0.sdk.connector.kind.sql.query import Query + +import pandas + + +class MySQLQuery(Query): + def __init__(self, connector, query_string): + super().__init__(connector, query_string) + if not connector.get_kind() == ConnectorKind.SQL_MYSQL: + raise ValueError(f"connector with kind={connector.get_kind()} is not of valid {ConnectorKind.SQL_MYSQL} kind") + + def to_pandas(self): + engine = self.get_connector().get_engine() + if engine is None: + raise ValueError("engine is None") + connection = engine.raw_connection() + df = pandas.read_sql_query(sql=self.get_query_string(), con=connection) + connection.close() + return df diff --git a/dq0/sdk/connector/kind/sql/db/postgresql/__init__.py b/dq0/sdk/connector/kind/sql/db/postgresql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/connector/kind/sql/db/postgresql/postgresql.py b/dq0/sdk/connector/kind/sql/db/postgresql/postgresql.py new file mode 100644 index 00000000..cfbd8a69 --- /dev/null +++ b/dq0/sdk/connector/kind/sql/db/postgresql/postgresql.py @@ -0,0 +1,33 @@ +from dq0.sdk.connector.connector_kind import ConnectorKind +from dq0.sdk.connector.kind.sql.db.postgresql.postgresql_query import PostgreSQLQuery +from dq0.sdk.connector.kind.sql.sql import SQL + +import sqlalchemy + + +class PostgreSQL(SQL): + @staticmethod + def connection_uri(username='', password='', host='', port='', database=''): + if len(username) == 0: + password = '' + if len(host) == 0: + raise Exception("host not provided") + password_sep = ':' if 0 < len(password) else '' + user_sep = '@' if 0 < len(username) else '' + port_sep = ':' if 0 < len(port) else '' + database_sep = '/' if 0 < len(database) else '' + return f"postgresql+psycopg2://{username}{password_sep}{password}{user_sep}{host}{port_sep}{port}{database_sep}{database}" + + def __init__(self, username='', password='', host='', port='', database=''): + super().__init__(ConnectorKind.SQL_POSTGRESQL) + self._connection_uri = PostgreSQL.connection_uri(username=username, password=password, host=host, port=port, database=database) + self._engine = sqlalchemy.create_engine(self.get_connection_uri()) + + def get_connection_uri(self): + return self._connection_uri + + def get_engine(self): + return self._engine + + def new_query(self, query_string): + return PostgreSQLQuery(connector=self, query_string=query_string) diff --git a/dq0/sdk/connector/kind/sql/db/postgresql/postgresql_query.py b/dq0/sdk/connector/kind/sql/db/postgresql/postgresql_query.py new file mode 100644 index 00000000..3dd96365 --- /dev/null +++ b/dq0/sdk/connector/kind/sql/db/postgresql/postgresql_query.py @@ -0,0 +1,20 @@ +from dq0.sdk.connector.connector_kind import ConnectorKind +from dq0.sdk.connector.kind.sql.query import Query + +import pandas + + +class PostgreSQLQuery(Query): + def __init__(self, connector, query_string): + super().__init__(connector, query_string) + if not connector.get_kind() == ConnectorKind.SQL_POSTGRESQL: + raise ValueError(f"connector with kind={connector.get_kind()} is not of valid {ConnectorKind.SQL_POSTGRESQL} kind") + + def to_pandas(self): + engine = self.get_connector().get_engine() + if engine is None: + raise ValueError("engine is None") + connection = engine.raw_connection() + df = pandas.read_sql_query(sql=self.get_query_string(), con=connection) + connection.close() + return df diff --git a/dq0/sdk/connector/kind/sql/db/sqlite/__init__.py b/dq0/sdk/connector/kind/sql/db/sqlite/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/connector/kind/sql/db/sqlite/sqlite.py b/dq0/sdk/connector/kind/sql/db/sqlite/sqlite.py new file mode 100644 index 00000000..5f33b76d --- /dev/null +++ b/dq0/sdk/connector/kind/sql/db/sqlite/sqlite.py @@ -0,0 +1,34 @@ +from pathlib import Path + +from dq0.sdk.connector.connector_kind import ConnectorKind +from dq0.sdk.connector.kind.sql.db.sqlite.sqlite_query import SQLiteQuery +from dq0.sdk.connector.kind.sql.sql import SQL + +import sqlalchemy + + +class SQLite(SQL): + @staticmethod + def connection_uri(filepath): + absolute_path = filepath.resolve() + return f"sqlite:///{absolute_path}" + + def __init__(self, filepath): + super().__init__(ConnectorKind.SQL_SQLITE) + self._filepath = Path(filepath) + if not self._filepath.is_file(): + raise ValueError(f"filepath={filepath} is not a file") + self._connection_uri = SQLite.connection_uri(filepath=self._filepath) + self._engine = sqlalchemy.create_engine(self.get_connection_uri()) + + def get_filepath(self): + return self._filepath + + def get_connection_uri(self): + return self._connection_uri + + def get_engine(self): + return self._engine + + def new_query(self, query_string): + return SQLiteQuery(connector=self, query_string=query_string) diff --git a/dq0/sdk/connector/kind/sql/db/sqlite/sqlite_query.py b/dq0/sdk/connector/kind/sql/db/sqlite/sqlite_query.py new file mode 100644 index 00000000..9da332be --- /dev/null +++ b/dq0/sdk/connector/kind/sql/db/sqlite/sqlite_query.py @@ -0,0 +1,20 @@ +from dq0.sdk.connector.connector_kind import ConnectorKind +from dq0.sdk.connector.kind.sql.query import Query + +import pandas + + +class SQLiteQuery(Query): + def __init__(self, connector, query_string): + super().__init__(connector, query_string) + if not connector.get_kind() == ConnectorKind.SQL_SQLITE: + raise ValueError(f"connector with kind={connector.get_kind()} is not of valid {ConnectorKind.SQL_SQLITE} kind") + + def to_pandas(self): + engine = self.get_connector().get_engine() + if engine is None: + raise ValueError("engine is None") + connection = engine.raw_connection() + df = pandas.read_sql_query(sql=self.get_query_string(), con=connection) + connection.close() + return df diff --git a/dq0/sdk/connector/kind/sql/query.py b/dq0/sdk/connector/kind/sql/query.py new file mode 100644 index 00000000..0c15ecb9 --- /dev/null +++ b/dq0/sdk/connector/kind/sql/query.py @@ -0,0 +1,18 @@ +from dq0.sdk.connector.connector_kind import ConnectorKind + + +class Query: + def __init__(self, connector, query_string): + if not ConnectorKind.is_valid_sql(kind=connector.get_kind()): + raise ValueError(f"connector with kind={connector.get_kind()} is not of valid sql kind") + self._connector = connector + self._query_string = query_string + + def get_connector(self): + return self._connector + + def get_query_string(self): + return self._query_string + + def to_pandas(self): + raise NotImplementedError diff --git a/dq0/sdk/connector/kind/sql/sql.py b/dq0/sdk/connector/kind/sql/sql.py new file mode 100644 index 00000000..0a45bab9 --- /dev/null +++ b/dq0/sdk/connector/kind/sql/sql.py @@ -0,0 +1,12 @@ +from dq0.sdk.connector.connector import Connector +from dq0.sdk.connector.connector_kind import ConnectorKind + + +class SQL(Connector): + def __init__(self, kind): + super().__init__(kind=kind) + if not ConnectorKind.is_valid_sql(kind=kind): + raise ValueError(f"kind={kind} is not a valid sql kind") + + def new_query(self, query_string): + raise NotImplementedError diff --git a/dq0/sdk/data/metadata/interface/dataset/attribute_utils.py b/dq0/sdk/data/metadata/interface/attribute_utils.py similarity index 100% rename from dq0/sdk/data/metadata/interface/dataset/attribute_utils.py rename to dq0/sdk/data/metadata/interface/attribute_utils.py diff --git a/dq0/sdk/data/metadata/interface/dataset/attributes_group.py b/dq0/sdk/data/metadata/interface/attributes_group.py similarity index 66% rename from dq0/sdk/data/metadata/interface/dataset/attributes_group.py rename to dq0/sdk/data/metadata/interface/attributes_group.py index a21f3df6..36ef9732 100644 --- a/dq0/sdk/data/metadata/interface/dataset/attributes_group.py +++ b/dq0/sdk/data/metadata/interface/attributes_group.py @@ -1,16 +1,16 @@ -from dq0.sdk.data.metadata.interface.dataset.attribute_utils import AttributeUtils -from dq0.sdk.data.metadata.interface.dataset.entity import Entity +from dq0.sdk.data.metadata.interface.attribute_utils import AttributeUtils +from dq0.sdk.data.metadata.interface.entity import Entity from dq0.sdk.data.metadata.structure.attribute.attribute_list import AttributeList from dq0.sdk.data.metadata.structure.permissions.permissions import Permissions class AttributesGroup: - def __init__(self, key, permissions, entity, attribute_list=None): + def __init__(self, key, permissions, entity, attribute_list=None, create_attributes_group_func=None): if key is not None and not isinstance(key, str): raise Exception(f"key {key} is not of type str, is of type {type(key)} instead") Permissions.check(permissions=permissions) - if not isinstance(entity, Entity): - raise Exception(f"entity is not of type Entity, is of type {type(entity)} instead") + if not isinstance(entity, Entity) and not isinstance(entity, AttributesGroup): + raise Exception(f"entity is neither of type Entity nor of type AttributesGroup, is of type {type(entity)} instead") self._entity = entity self._key = key self._permissions = permissions @@ -21,6 +21,8 @@ def __init__(self, key, permissions, entity, attribute_list=None): f"is of type {type(attribute_list)} instead") if key != attribute_list.get_key(): raise Exception(f"keys do not match: {key} != {attribute_list.get_key()}") + self._create_attributes_group_func = create_attributes_group_func + self._attribute_groups = {} def to_dict(self, request_uuids=set(), full=True): return self._attribute_list.to_dict(request_uuids=request_uuids, full=full) if self._attribute_list is not None else None @@ -49,15 +51,18 @@ def delete(self): keys.add(attribute.get_key()) for key in keys: self.delete_attribute(key=key) + self.wipe() def wipe(self): self._attribute_list = None + for attribute_group in self._attribute_groups.values(): + attribute_group.wipe() def get_attribute(self, key): return self._attribute_list.get_attribute(key=key) if self._attribute_list is not None else None def get_attribute_value(self, key): - attribute = self._attribute_list.get_attribute(key=key) if self._attribute_list is not None else None + attribute = self.get_attribute(key=key) return attribute.get_value() if attribute is not None else None def set_attribute_value(self, type_name, key, value, permissions, allow_modification=True): @@ -83,3 +88,26 @@ def delete_attribute(self, key): if len(self._attribute_list.get_value()) == 0: self._entity.remove_attribute(key=self.get_key()) self._attribute_list = None + + def add_attribute(self, attribute): + attribute = self.get_attribute(key=attribute.get_key()) + if attribute is not None: + raise Exception(f"attribute {attribute.get_key()} is already set and cannot be modified") + if self._attribute_list is None: + attribute_list = AttributeList(key=self.get_key(), value=[], permissions=self._permissions) + self._entity.add_attribute(attribute=attribute_list) + self._attribute_list = attribute_list + self._attribute_list.add_attribute(attribute=attribute) + + def remove_attribute(self, key): + self.delete_attribute(key=key) + if key in self._attribute_groups: + self._attribute_groups[key].wipe() + + def get_attribute_group(self, key): + if key not in self._attribute_groups: + attribute_list = self.get_attribute(key=key) + if self._create_attributes_group_func is None: + raise Exception("create_attributes_group_func is None") + self._attribute_groups[key] = self._create_attributes_group_func(key=key, attribute_list=attribute_list) + return self._attribute_groups[key] diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_data.py b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_data.py index 5be5b2f3..769ccca8 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_data.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_data.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_machine_learning.py b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_machine_learning.py index 51666146..2b6b8654 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_machine_learning.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_machine_learning.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql.py b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql.py index 62486b41..937840f1 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql.py @@ -1,5 +1,5 @@ -from dq0.sdk.data.metadata.interface.dataset.attribute_utils import AttributeUtils -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attribute_utils import AttributeUtils +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql_and_synthesis.py b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql_and_synthesis.py index bf6fabc0..b2d5d8f5 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql_and_synthesis.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql_and_synthesis.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_synthesis.py b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_synthesis.py index ee3a9402..364ed841 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_synthesis.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_synthesis.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/column/column.py b/dq0/sdk/data/metadata/interface/dataset/v1/column/column.py index 5e1fc6a8..6a65bd9a 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/column/column.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/column/column.py @@ -1,9 +1,9 @@ -from dq0.sdk.data.metadata.interface.dataset.entity import Entity from dq0.sdk.data.metadata.interface.dataset.v1.column.attributes_column_data import AttributesColumnData from dq0.sdk.data.metadata.interface.dataset.v1.column.attributes_column_machine_learning import AttributesColumnMachineLearning from dq0.sdk.data.metadata.interface.dataset.v1.column.attributes_column_private_sql import AttributesColumnPrivateSql from dq0.sdk.data.metadata.interface.dataset.v1.column.attributes_column_private_sql_and_synthesis import AttributesColumnPrivateSqlAndSynthesis from dq0.sdk.data.metadata.interface.dataset.v1.column.attributes_column_private_synthesis import AttributesColumnPrivateSynthesis +from dq0.sdk.data.metadata.interface.entity import Entity from dq0.sdk.data.metadata.structure.attribute.attribute_list import AttributeList from dq0.sdk.data.metadata.structure.node.node import Node from dq0.sdk.data.metadata.structure.node.node_type import NodeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_connector.py b/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_connector.py index 4c36ecd6..3422c307 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_connector.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_connector.py @@ -1,5 +1,5 @@ -from dq0.sdk.data.metadata.interface.dataset.attribute_utils import AttributeUtils -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attribute_utils import AttributeUtils +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType @@ -11,9 +11,9 @@ def __init__(self, database, attribute_list=None): entity=database, attribute_list=attribute_list) - def type_name_check(self, type_name): - if self.type_name != type_name: - raise Exception(f"access forbidden, typename mismatch: {self.type_name} != {type_name}") + def type_names_check(self, type_names): + if self.type_name not in type_names: + raise Exception(f"access forbidden, typename mismatch: {self.type_name} not in {type_names}") # type_name @property @@ -34,18 +34,37 @@ def type_name(self): raise Exception(f"type_name {self.type_name} may not be deleted") # ========================================================================================================================================================== - # csv + # modular properties # ========================================================================================================================================================== + # charset + @property + def charset(self): + self.type_names_check(type_names={'mysql'}) + return self.get_attribute_value(key='charset') + + @charset.setter + def charset(self, new_charset): + self.type_names_check(type_names={'mysql'}) + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='charset', + value=new_charset, + permissions=DefaultPermissions.owner_attribute(role_uuids=self.get_role_uuids())) + + @charset.deleter + def charset(self): + self.type_names_check(type_names={'mysql'}) + self.delete_attribute(key='charset') + # decimal @property def decimal(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) return self.get_attribute_value(key='decimal') @decimal.setter def decimal(self, new_decimal): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, key='decimal', value=new_decimal, @@ -53,18 +72,18 @@ def decimal(self, new_decimal): @decimal.deleter def decimal(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.delete_attribute(key='decimal') # header_columns @property def header_columns(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) return AttributeUtils.value_to_list(input_value=self.get_attribute_value(key='header_columns')) @header_columns.setter def header_columns(self, new_header_columns): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) shared_permissions = DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()) self.set_attribute_value(type_name=AttributeType.TYPE_NAME_LIST, key='header_columns', @@ -75,19 +94,19 @@ def header_columns(self, new_header_columns): @header_columns.deleter def header_columns(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.delete_attribute(key='header_columns') # header_row @property def header_row(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) value = self.get_attribute_value(key='header_row') return AttributeUtils.value_to_list(input_value=value) if isinstance(value, list) else value @header_row.setter def header_row(self, new_header_row): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) shared_permissions = DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()) value = AttributeUtils.list_to_value(input_list=new_header_row, type_name=AttributeType.TYPE_NAME_INT, @@ -100,19 +119,38 @@ def header_row(self, new_header_row): @header_row.deleter def header_row(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.delete_attribute(key='header_row') + # host + @property + def host(self): + self.type_names_check(type_names={'mysql', 'postgresql'}) + return self.get_attribute_value(key='host') + + @host.setter + def host(self, new_host): + self.type_names_check(type_names={'mysql', 'postgresql'}) + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='host', + value=new_host, + permissions=DefaultPermissions.owner_attribute(role_uuids=self.get_role_uuids())) + + @host.deleter + def host(self): + self.type_names_check(type_names={'mysql', 'postgresql'}) + self.delete_attribute(key='host') + # index_col @property def index_col(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) value = self.get_attribute_value(key='index_col') return AttributeUtils.value_to_list(input_value=value) if isinstance(value, list) else value @index_col.setter def index_col(self, new_index_col): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) shared_permissions = DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()) type_name = None value = None @@ -143,18 +181,18 @@ def index_col(self, new_index_col): @index_col.deleter def index_col(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.delete_attribute(key='index_col') # na_values @property def na_values(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) return AttributeUtils.value_to_dict(input_value=self.get_attribute_value(key='na_values')) @na_values.setter def na_values(self, new_na_values): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) shared_permissions = DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()) self.set_attribute_value(type_name=AttributeType.TYPE_NAME_LIST, key='na_values', @@ -165,18 +203,56 @@ def na_values(self, new_na_values): @na_values.deleter def na_values(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.delete_attribute(key='na_values') + # password + @property + def password(self): + self.type_names_check(type_names={'mysql', 'postgresql'}) + return self.get_attribute_value(key='password') + + @password.setter + def password(self, new_password): + self.type_names_check(type_names={'mysql', 'postgresql'}) + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='password', + value=new_password, + permissions=DefaultPermissions.owner_attribute(role_uuids=self.get_role_uuids())) + + @password.deleter + def password(self): + self.type_names_check(type_names={'mysql', 'postgresql'}) + self.delete_attribute(key='password') + + # port + @property + def port(self): + self.type_names_check(type_names={'mysql', 'postgresql'}) + return self.get_attribute_value(key='port') + + @port.setter + def port(self, new_port): + self.type_names_check(type_names={'mysql', 'postgresql'}) + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_INT, + key='port', + value=new_port, + permissions=DefaultPermissions.owner_attribute(role_uuids=self.get_role_uuids())) + + @port.deleter + def port(self): + self.type_names_check(type_names={'mysql', 'postgresql'}) + self.delete_attribute(key='port') + # sep @property def sep(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) return self.get_attribute_value(key='sep') @sep.setter def sep(self, new_sep): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, key='sep', value=new_sep, @@ -184,18 +260,18 @@ def sep(self, new_sep): @sep.deleter def sep(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.delete_attribute(key='sep') # skipinitialspace @property def skipinitialspace(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) return self.get_attribute_value(key='skipinitialspace') @skipinitialspace.setter def skipinitialspace(self, new_skipinitialspace): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.set_attribute_value(type_name=AttributeType.TYPE_NAME_BOOLEAN, key='skipinitialspace', value=new_skipinitialspace, @@ -203,18 +279,18 @@ def skipinitialspace(self, new_skipinitialspace): @skipinitialspace.deleter def skipinitialspace(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.delete_attribute(key='skipinitialspace') # uri @property def uri(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv', 'sqlite'}) return self.get_attribute_value(key='uri') @uri.setter def uri(self, new_uri): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv', 'sqlite'}) self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, key='uri', value=new_uri, @@ -222,18 +298,18 @@ def uri(self, new_uri): @uri.deleter def uri(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv', 'sqlite'}) self.delete_attribute(key='uri') # use_original_header @property def use_original_header(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) return self.get_attribute_value(key='use_original_header') @use_original_header.setter def use_original_header(self, new_use_original_header): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.set_attribute_value(type_name=AttributeType.TYPE_NAME_BOOLEAN, key='use_original_header', value=new_use_original_header, @@ -241,79 +317,18 @@ def use_original_header(self, new_use_original_header): @use_original_header.deleter def use_original_header(self): - self.type_name_check(type_name='csv') + self.type_names_check(type_names={'csv'}) self.delete_attribute(key='use_original_header') - # ========================================================================================================================================================== - # postgresql - # ========================================================================================================================================================== - - # host - @property - def host(self): - self.type_name_check(type_name='postgresql') - return self.get_attribute_value(key='host') - - @host.setter - def host(self, new_host): - self.type_name_check(type_name='postgresql') - self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, - key='host', - value=new_host, - permissions=DefaultPermissions.owner_attribute(role_uuids=self.get_role_uuids())) - - @host.deleter - def host(self): - self.type_name_check(type_name='postgresql') - self.delete_attribute(key='host') - - # password - @property - def password(self): - self.type_name_check(type_name='postgresql') - return self.get_attribute_value(key='password') - - @password.setter - def password(self, new_password): - self.type_name_check(type_name='postgresql') - self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, - key='password', - value=new_password, - permissions=DefaultPermissions.owner_attribute(role_uuids=self.get_role_uuids())) - - @password.deleter - def password(self): - self.type_name_check(type_name='postgresql') - self.delete_attribute(key='password') - - # port - @property - def port(self): - self.type_name_check(type_name='postgresql') - return self.get_attribute_value(key='port') - - @port.setter - def port(self, new_port): - self.type_name_check(type_name='postgresql') - self.set_attribute_value(type_name=AttributeType.TYPE_NAME_INT, - key='port', - value=new_port, - permissions=DefaultPermissions.owner_attribute(role_uuids=self.get_role_uuids())) - - @port.deleter - def port(self): - self.type_name_check(type_name='postgresql') - self.delete_attribute(key='port') - # username @property def username(self): - self.type_name_check(type_name='postgresql') + self.type_names_check(type_names={'mysql', 'postgresql'}) return self.get_attribute_value(key='username') @username.setter def username(self, new_username): - self.type_name_check(type_name='postgresql') + self.type_names_check(type_names={'mysql', 'postgresql'}) self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, key='username', value=new_username, @@ -321,5 +336,5 @@ def username(self, new_username): @username.deleter def username(self): - self.type_name_check(type_name='postgresql') + self.type_names_check(type_names={'mysql', 'postgresql'}) self.delete_attribute(key='username') diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_data.py b/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_data.py index 26ffead8..64d6961a 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_data.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_data.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_differential_privacy.py b/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_differential_privacy.py index fc853122..cbab60b9 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_differential_privacy.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/database/attributes_database_differential_privacy.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/database/database.py b/dq0/sdk/data/metadata/interface/dataset/v1/database/database.py index 173f6a56..49fe62f5 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/database/database.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/database/database.py @@ -1,8 +1,8 @@ -from dq0.sdk.data.metadata.interface.dataset.entity import Entity from dq0.sdk.data.metadata.interface.dataset.v1.database.attributes_database_connector import AttributesDatabaseConnector from dq0.sdk.data.metadata.interface.dataset.v1.database.attributes_database_data import AttributesDatabaseData from dq0.sdk.data.metadata.interface.dataset.v1.database.attributes_database_differential_privacy import AttributesDatabaseDifferentialPrivacy from dq0.sdk.data.metadata.interface.dataset.v1.schema.schema import Schema +from dq0.sdk.data.metadata.interface.entity import Entity from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.node.node_type import NodeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/dataset/attributes_dataset_data.py b/dq0/sdk/data/metadata/interface/dataset/v1/dataset/attributes_dataset_data.py index 2ee0f906..e2ae4965 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/dataset/attributes_dataset_data.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/dataset/attributes_dataset_data.py @@ -1,5 +1,5 @@ -from dq0.sdk.data.metadata.interface.dataset.attribute_utils import AttributeUtils -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attribute_utils import AttributeUtils +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/dataset/attributes_dataset_differential_privacy.py b/dq0/sdk/data/metadata/interface/dataset/v1/dataset/attributes_dataset_differential_privacy.py index 9aab0b87..45a46179 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/dataset/attributes_dataset_differential_privacy.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/dataset/attributes_dataset_differential_privacy.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/dataset/dataset.py b/dq0/sdk/data/metadata/interface/dataset/v1/dataset/dataset.py index dfad8212..0c95ed54 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/dataset/dataset.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/dataset/dataset.py @@ -1,7 +1,7 @@ -from dq0.sdk.data.metadata.interface.dataset.entity import Entity from dq0.sdk.data.metadata.interface.dataset.v1.database.database import Database from dq0.sdk.data.metadata.interface.dataset.v1.dataset.attributes_dataset_data import AttributesDatasetData from dq0.sdk.data.metadata.interface.dataset.v1.dataset.attributes_dataset_differential_privacy import AttributesDatasetDifferentialPrivacy +from dq0.sdk.data.metadata.interface.entity import Entity from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.node.node_type import NodeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/schema/attributes_schema_data.py b/dq0/sdk/data/metadata/interface/dataset/v1/schema/attributes_schema_data.py index 2ff7c6b3..28590aee 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/schema/attributes_schema_data.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/schema/attributes_schema_data.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/schema/attributes_schema_differential_privacy.py b/dq0/sdk/data/metadata/interface/dataset/v1/schema/attributes_schema_differential_privacy.py index dfb35e24..a3be601c 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/schema/attributes_schema_differential_privacy.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/schema/attributes_schema_differential_privacy.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/schema/schema.py b/dq0/sdk/data/metadata/interface/dataset/v1/schema/schema.py index 74490119..27a05ede 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/schema/schema.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/schema/schema.py @@ -1,7 +1,7 @@ -from dq0.sdk.data.metadata.interface.dataset.entity import Entity from dq0.sdk.data.metadata.interface.dataset.v1.schema.attributes_schema_data import AttributesSchemaData from dq0.sdk.data.metadata.interface.dataset.v1.schema.attributes_schema_differential_privacy import AttributesSchemaDifferentialPrivacy from dq0.sdk.data.metadata.interface.dataset.v1.table.table import Table +from dq0.sdk.data.metadata.interface.entity import Entity from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.node.node_type import NodeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_data.py b/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_data.py index 9fbfd3f6..0c2d5d5a 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_data.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_data.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_differential_privacy.py b/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_differential_privacy.py index 92df7b00..c07a612c 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_differential_privacy.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_differential_privacy.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_private_sql.py b/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_private_sql.py index 18218427..b5149841 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_private_sql.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_private_sql.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_private_synthesis.py b/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_private_synthesis.py index 2869617b..6cda68f3 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_private_synthesis.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/table/attributes_table_private_synthesis.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/table/table.py b/dq0/sdk/data/metadata/interface/dataset/v1/table/table.py index d109947a..7fc632b7 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/table/table.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/table/table.py @@ -1,9 +1,9 @@ -from dq0.sdk.data.metadata.interface.dataset.entity import Entity from dq0.sdk.data.metadata.interface.dataset.v1.column.column import Column from dq0.sdk.data.metadata.interface.dataset.v1.table.attributes_table_data import AttributesTableData from dq0.sdk.data.metadata.interface.dataset.v1.table.attributes_table_differential_privacy import AttributesTableDifferentialPrivacy from dq0.sdk.data.metadata.interface.dataset.v1.table.attributes_table_private_sql import AttributesTablePrivateSql from dq0.sdk.data.metadata.interface.dataset.v1.table.attributes_table_private_synthesis import AttributesTablePrivateSynthesis +from dq0.sdk.data.metadata.interface.entity import Entity from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.node.node_type import NodeType diff --git a/dq0/sdk/data/metadata/interface/dataset/entity.py b/dq0/sdk/data/metadata/interface/entity.py similarity index 99% rename from dq0/sdk/data/metadata/interface/dataset/entity.py rename to dq0/sdk/data/metadata/interface/entity.py index 4536f3ed..8c7f38d3 100644 --- a/dq0/sdk/data/metadata/interface/dataset/entity.py +++ b/dq0/sdk/data/metadata/interface/entity.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.interface.dataset.entity_iterator import EntityIterator +from dq0.sdk.data.metadata.interface.entity_iterator import EntityIterator from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute_list import AttributeList from dq0.sdk.data.metadata.structure.attribute.attribute_string import AttributeString diff --git a/dq0/sdk/data/metadata/interface/dataset/entity_iterator.py b/dq0/sdk/data/metadata/interface/entity_iterator.py similarity index 100% rename from dq0/sdk/data/metadata/interface/dataset/entity_iterator.py rename to dq0/sdk/data/metadata/interface/entity_iterator.py diff --git a/dq0/sdk/data/metadata/interface/interface.py b/dq0/sdk/data/metadata/interface/interface.py index 36f24a79..f67b22f0 100644 --- a/dq0/sdk/data/metadata/interface/interface.py +++ b/dq0/sdk/data/metadata/interface/interface.py @@ -1,8 +1,10 @@ -from dq0.sdk.data.metadata.interface.dataset.entity import Entity from dq0.sdk.data.metadata.interface.dataset.v1.dataset.dataset import Dataset +from dq0.sdk.data.metadata.interface.entity import Entity from dq0.sdk.data.metadata.interface.interface_iterator import InterfaceIterator -from dq0.sdk.data.metadata.specification.dataset.v1.specification_v1 import SpecificationV1 +from dq0.sdk.data.metadata.interface.run.v1.run.run import Run +from dq0.sdk.data.metadata.specification.dataset.v1.specification_v1 import SpecificationV1 as DatasetSpecificationV1 from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.run.v1.specification_v1 import SpecificationV1 as RunSpecificationV1 from dq0.sdk.data.metadata.structure.metadata import Metadata from dq0.sdk.data.metadata.structure.node.node import Node from dq0.sdk.data.metadata.structure.node.node_type import NodeType @@ -42,26 +44,6 @@ def to_dict(self, request_uuids=set(), full=True): def get_metadata(self): return self._metadata - def dataset(self, name=None): - node = self._metadata.get_node(root_key='dataset') - if name is None: - if node is None: - raise Exception("get without name may only work for existing dataset node") - name = Entity.name_of(node=node) - if 'dataset' not in self._entities: - specification = self._metadata.get_specification(root_key='dataset') - if isinstance(specification, SpecificationV1): - self._entities['dataset'] = Dataset(name=name, parent=self, - permissions=DefaultPermissions.shared_node(role_uuids=self.get_role_uuids()), - data_permissions=DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()), - name_permissions=DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()), - role_uuids=self.get_role_uuids(), node=node) - else: - raise Exception("no interface for specified version available") - if name != self._entities['dataset'].get_name(): - raise Exception(f"name mismatch: {name} != {self._entities['dataset'].get_name()}") - return self._entities['dataset'] - def create(self): pass @@ -92,3 +74,43 @@ def set_child_name(self, old_name, new_name): def apply_defaults_and_verify(self, specifications=None): return Interface(metadata=self._metadata.apply_defaults_and_verify(specifications=specifications), role_uuids=self.get_role_uuids()) + + def dataset(self, name=None): + node = self._metadata.get_node(root_key='dataset') + if name is None: + if node is None: + raise Exception("get without name may only work for existing dataset node") + name = Entity.name_of(node=node) + if 'dataset' not in self._entities: + specification = self._metadata.get_specification(root_key='dataset') + if isinstance(specification, DatasetSpecificationV1): + self._entities['dataset'] = Dataset(name=name, parent=self, + permissions=DefaultPermissions.shared_node(role_uuids=self.get_role_uuids()), + data_permissions=DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()), + name_permissions=DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()), + role_uuids=self.get_role_uuids(), node=node) + else: + raise Exception("no interface for specified version available") + if name != self._entities['dataset'].get_name(): + raise Exception(f"name mismatch: {name} != {self._entities['dataset'].get_name()}") + return self._entities['dataset'] + + def run(self, name=None): + node = self._metadata.get_node(root_key='run') + if name is None: + if node is None: + raise Exception("get without name may only work for existing run node") + name = Entity.name_of(node=node) + if 'run' not in self._entities: + specification = self._metadata.get_specification(root_key='run') + if isinstance(specification, RunSpecificationV1): + self._entities['run'] = Run(name=name, parent=self, + permissions=DefaultPermissions.analyst_node(role_uuids=self.get_role_uuids()), + data_permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids()), + name_permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids()), + role_uuids=self.get_role_uuids(), node=node) + else: + raise Exception("no interface for specified version available") + if name != self._entities['run'].get_name(): + raise Exception(f"name mismatch: {name} != {self._entities['run'].get_name()}") + return self._entities['run'] diff --git a/dq0/sdk/data/metadata/interface/run/__init__.py b/dq0/sdk/data/metadata/interface/run/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/data/metadata/interface/run/v1/__init__.py b/dq0/sdk/data/metadata/interface/run/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/__init__.py b/dq0/sdk/data/metadata/interface/run/v1/run/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_data.py b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_data.py new file mode 100644 index 00000000..c96ca3ca --- /dev/null +++ b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_data.py @@ -0,0 +1,62 @@ +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class AttributesRunData(AttributesGroup): + def __init__(self, run, attribute_list=None): + super().__init__(key='data', + permissions=DefaultPermissions.analyst_attribute(role_uuids=run.get_role_uuids()), + entity=run, + attribute_list=attribute_list) + + # description + @property + def description(self): + return self.get_attribute_value(key='description') + + @description.setter + def description(self, new_description): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='description', + value=new_description, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + + @description.deleter + def description(self): + self.delete_attribute(key='description') + + # name + @property + def name(self): + return self.get_attribute_value(key='name') + + @name.setter + def name(self, new_name): + old_name = self.get_attribute_value(key='name') + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='name', + value=new_name, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + self.get_entity().set_name(old_name=old_name, new_name=new_name) + + @name.deleter + def name(self): + raise Exception("name may not be deleted") + + # type_name + @property + def type_name(self): + return self.get_attribute_value(key='type_name') + + @type_name.setter + def type_name(self, new_type_name): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='type_name', + value=new_type_name, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids()), + allow_modification=False) + + @type_name.deleter + def type_name(self): + raise Exception(f"type_name {self.type_name} may not be deleted") diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql.py b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql.py new file mode 100644 index 00000000..0accfce3 --- /dev/null +++ b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql.py @@ -0,0 +1,64 @@ +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.interface.run.v1.run.attributes_run_sql_query_processor import AttributesRunSQLQueryProcessor +from dq0.sdk.data.metadata.interface.run.v1.run.attributes_run_sql_result_processor import AttributesRunSQLResultProcessor +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class AttributesRunSQL(AttributesGroup): + def __init__(self, run, attribute_list=None): + super().__init__(key='sql', + permissions=DefaultPermissions.analyst_attribute(role_uuids=run.get_role_uuids()), + entity=run, + attribute_list=attribute_list, + create_attributes_group_func=self.create_attributes_group) + + def create_attributes_group(self, key, attribute_list=None): + if key == 'query_processor': + return AttributesRunSQLQueryProcessor(attributes_run_sql=self, attribute_list=attribute_list) + elif key == 'result_processor': + return AttributesRunSQLResultProcessor(attributes_run_sql=self, attribute_list=attribute_list) + else: + raise Exception(f"key {key} is invalid") + + # query_string + @property + def query_string(self): + return self.get_attribute_value(key='query_string') + + @query_string.setter + def query_string(self, new_query_string): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='query_string', + value=new_query_string, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + + @query_string.deleter + def query_string(self): + raise Exception("query_string may not be deleted") + + # query_processor + @property + def query_processor(self): + return super().get_attribute_group(key='query_processor') + + @query_processor.setter + def query_processor(self, _): + raise Exception("query_processor attribute group may not be set") + + @query_processor.deleter + def query_processor(self): + super().get_attribute_group(key='query_processor').delete() + + # result_processor + @property + def result_processor(self): + return super().get_attribute_group(key='result_processor') + + @result_processor.setter + def result_processor(self, _): + raise Exception("result_processor attribute group may not be set") + + @result_processor.deleter + def result_processor(self): + super().get_attribute_group(key='result_processor').delete() diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_query_processor.py b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_query_processor.py new file mode 100644 index 00000000..901c9f46 --- /dev/null +++ b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_query_processor.py @@ -0,0 +1,28 @@ +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class AttributesRunSQLQueryProcessor(AttributesGroup): + def __init__(self, attributes_run_sql, attribute_list=None): + super().__init__(key='query_processor', + permissions=DefaultPermissions.analyst_attribute(role_uuids=attributes_run_sql.get_role_uuids()), + entity=attributes_run_sql, + attribute_list=attribute_list) + + # type_name + @property + def type_name(self): + return self.get_attribute_value(key='type_name') + + @type_name.setter + def type_name(self, new_type_name): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='type_name', + value=new_type_name, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids()), + allow_modification=False) + + @type_name.deleter + def type_name(self): + raise Exception(f"type_name {self.type_name} may not be deleted") diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py new file mode 100644 index 00000000..ac301674 --- /dev/null +++ b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py @@ -0,0 +1,28 @@ +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class AttributesRunSQLResultProcessor(AttributesGroup): + def __init__(self, attributes_run_sql, attribute_list=None): + super().__init__(key='result_processor', + permissions=DefaultPermissions.analyst_attribute(role_uuids=attributes_run_sql.get_role_uuids()), + entity=attributes_run_sql, + attribute_list=attribute_list) + + # type_name + @property + def type_name(self): + return self.get_attribute_value(key='type_name') + + @type_name.setter + def type_name(self, new_type_name): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='type_name', + value=new_type_name, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids()), + allow_modification=False) + + @type_name.deleter + def type_name(self): + raise Exception(f"type_name {self.type_name} may not be deleted") diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/run.py b/dq0/sdk/data/metadata/interface/run/v1/run/run.py new file mode 100644 index 00000000..58aa82ee --- /dev/null +++ b/dq0/sdk/data/metadata/interface/run/v1/run/run.py @@ -0,0 +1,64 @@ +from dq0.sdk.data.metadata.interface.entity import Entity +from dq0.sdk.data.metadata.interface.run.v1.run.attributes_run_data import AttributesRunData +from dq0.sdk.data.metadata.interface.run.v1.run.attributes_run_sql import AttributesRunSQL +from dq0.sdk.data.metadata.structure.attribute.attribute_list import AttributeList +from dq0.sdk.data.metadata.structure.node.node import Node +from dq0.sdk.data.metadata.structure.node.node_type import NodeType + + +class Run(Entity): + @staticmethod + def check_type_name(type_name): + if type_name not in ['sql']: + raise Exception(f"unknown type_name {type_name}") + + def __init__(self, name, parent, permissions=None, data_permissions=None, name_permissions=None, role_uuids=None, node=None): + super().__init__(name=name, type_name=NodeType.TYPE_NAME_RUN, parent=parent, + permissions=permissions, data_permissions=data_permissions, name_permissions=name_permissions, + create_attributes_group_func=self.create_attributes_group, role_uuids=role_uuids, node=node) + + def get_type_name(self): + data_attribute = self.get_node().get_attribute(key='data') if isinstance(self.get_node(), Node) else None + type_name_attribute = data_attribute.get_attribute(key='type_name') if isinstance(data_attribute, AttributeList) else None + type_name = type_name_attribute.get_value() if type_name_attribute is not None else None + if type_name is not None: + Run.check_type_name(type_name=type_name) + return type_name + + def create_attributes_group(self, key, attribute_list=None): + if key == 'data': + return AttributesRunData(run=self, attribute_list=attribute_list) + elif key == 'sql': + if self.get_type_name() is None: + raise Exception("cannot access sql without type_name being set") + if self.get_type_name() != 'sql': + raise Exception("attribute group sql not allowed for non sql run") + return AttributesRunSQL(run=self, attribute_list=attribute_list) + else: + raise Exception(f"key {key} is invalid") + + # data + @property + def data(self): + return super().get_attribute_group(key='data') + + @data.setter + def data(self, _): + raise Exception("data attribute group may not be set") + + @data.deleter + def data(self): + super().get_attribute_group(key='data').delete() + + # sql + @property + def sql(self): + return super().get_attribute_group(key='sql') + + @sql.setter + def sql(self, _): + raise Exception("sql attribute group may not be set") + + @sql.deleter + def sql(self): + super().get_attribute_group(key='sql').delete() diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_boolean.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_boolean.py index ab5bb11e..f9d0b605 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_boolean.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_boolean.py @@ -1,7 +1,7 @@ from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute -from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup -from dq0.sdk.data.metadata.specification.json_schema.column import Column as JsonSchemaColumn +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.dataset.column import Column as JsonSchemaColumn from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_datetime.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_datetime.py index 487a0c29..adada7a5 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_datetime.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_datetime.py @@ -1,7 +1,7 @@ from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute -from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup -from dq0.sdk.data.metadata.specification.json_schema.column import Column as JsonSchemaColumn +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.dataset.column import Column as JsonSchemaColumn from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_float.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_float.py index b9a9bf9a..0eceaa94 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_float.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_float.py @@ -1,7 +1,7 @@ from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute -from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup -from dq0.sdk.data.metadata.specification.json_schema.column import Column as JsonSchemaColumn +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.dataset.column import Column as JsonSchemaColumn from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_int.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_int.py index a3d51204..96d37889 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_int.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_int.py @@ -1,7 +1,7 @@ from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute -from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup -from dq0.sdk.data.metadata.specification.json_schema.column import Column as JsonSchemaColumn +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.dataset.column import Column as JsonSchemaColumn from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_string.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_string.py index a3751259..1921e486 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_string.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_string.py @@ -1,7 +1,7 @@ from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute -from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup -from dq0.sdk.data.metadata.specification.json_schema.column import Column as JsonSchemaColumn +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.dataset.column import Column as JsonSchemaColumn from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/connector.py b/dq0/sdk/data/metadata/specification/dataset/v1/connector.py index 36f2902b..32fe28f6 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/connector.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/connector.py @@ -1,5 +1,7 @@ from dq0.sdk.data.metadata.specification.dataset.v1.connector_csv import ConnectorCSV +from dq0.sdk.data.metadata.specification.dataset.v1.connector_mysql import ConnectorMySQL from dq0.sdk.data.metadata.specification.dataset.v1.connector_postgresql import ConnectorPostgreSQL +from dq0.sdk.data.metadata.specification.dataset.v1.connector_sqlite import ConnectorSQLite from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_string import AttributeString @@ -17,8 +19,12 @@ def apply_defaults(attribute, role_uuids=None): raise Exception(f"attribute type_name is not of type AttributeString, is of type {type(attribute_type_name)} instead") if attribute_type_name.get_value() == 'csv': return ConnectorCSV.apply_defaults(attribute=attribute, role_uuids=role_uuids) + elif attribute_type_name.get_value() == 'mysql': + return ConnectorMySQL.apply_defaults(attribute=attribute, role_uuids=role_uuids) elif attribute_type_name.get_value() == 'postgresql': return ConnectorPostgreSQL.apply_defaults(attribute=attribute, role_uuids=role_uuids) + elif attribute_type_name.get_value() == 'sqlite': + return ConnectorSQLite.apply_defaults(attribute=attribute, role_uuids=role_uuids) else: raise Exception(f"unknown connector type_name {attribute_type_name.get_value()}") @@ -32,7 +38,11 @@ def verify(attribute, role_uuids=None): raise Exception(f"attribute type_name is not of type AttributeString, is of type {type(attribute_type_name)} instead") if attribute_type_name.get_value() == 'csv': return ConnectorCSV.verify(attribute=attribute, role_uuids=role_uuids) + elif attribute_type_name.get_value() == 'mysql': + return ConnectorMySQL.verify(attribute=attribute, role_uuids=role_uuids) elif attribute_type_name.get_value() == 'postgresql': return ConnectorPostgreSQL.verify(attribute=attribute, role_uuids=role_uuids) + elif attribute_type_name.get_value() == 'sqlite': + return ConnectorSQLite.verify(attribute=attribute, role_uuids=role_uuids) else: raise Exception(f"unknown connector type_name {attribute_type_name.get_value()}") diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/connector_csv.py b/dq0/sdk/data/metadata/specification/dataset/v1/connector_csv.py index 34f21497..62a16974 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/connector_csv.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/connector_csv.py @@ -46,7 +46,7 @@ def verify_attributes(attributes, role_uuids=None): 'type_name': ([AttributeType.TYPE_NAME_STRING], shared_attribute), 'uri': ([AttributeType.TYPE_NAME_STRING], shared_attribute), 'use_original_header': ([AttributeType.TYPE_NAME_BOOLEAN], shared_attribute), - }, required_keys={'type_name'}) + }, required_keys={'type_name', 'uri'}) type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] if type_name_attributes[0].get_value() != 'csv': raise Exception(f"csv connector type_name value {type_name_attributes[0].get_value()} does not match 'csv'") @@ -83,10 +83,17 @@ def json_schema(): key='connector', group_name='connector csv', description="The 'connector csv' attributes group.", - additional_description="Requires 'type_name' attribute.", - contains=JsonSchemaAttribute.json_schema( - key='type_name', attribute_name="type name", description="This item ensures that the 'type_name' attribute is present.", - type_name=AttributeType.TYPE_NAME_STRING), + additional_description="Requires 'type_name' and 'uri' attributes.", + contains=[ + JsonSchemaAttribute.json_schema( + key='type_name', attribute_name="type name", description="This item ensures that the 'type_name' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='uri', attribute_name="uri", description="This item ensures that the 'uri' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + ], attributes=[ JsonSchemaAttribute.json_schema( key='decimal', diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/connector_mysql.py b/dq0/sdk/data/metadata/specification/dataset/v1/connector_mysql.py new file mode 100644 index 00000000..6a959e6d --- /dev/null +++ b/dq0/sdk/data/metadata/specification/dataset/v1/connector_mysql.py @@ -0,0 +1,107 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute +from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_list import AttributeList +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class ConnectorMySQL: + @staticmethod + def apply_defaults(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data=None) + applied_attributes = ConnectorMySQL.apply_defaults_to_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + applied_permissions = DefaultPermissions.shared_attribute(role_uuids=role_uuids) \ + if attribute.get_permissions() is None else attribute.get_permissions().copy() + return AttributeList(key=attribute.get_key(), value=applied_attributes, permissions=applied_permissions) + + @staticmethod + def apply_defaults_to_attributes(attributes, role_uuids=None): + Attribute.check_list(attribute_list=attributes, check_data=None) + applied_attributes = [] if attributes is not None else None + for attribute in attributes if attributes is not None else []: + applied_attribute = attribute.copy() + applied_attribute.set_default_permissions(default_permissions=DefaultPermissions.shared_attribute(role_uuids=role_uuids)) + applied_attributes.append(applied_attribute) + return applied_attributes + + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'connector': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.shared_attribute(role_uuids=role_uuids)), + }) + ConnectorMySQL.verify_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + owner_attribute = DefaultPermissions.owner_attribute(role_uuids=role_uuids) + Attribute.check_list(attribute_list=attributes, check_data={ + 'charset': ([AttributeType.TYPE_NAME_STRING], owner_attribute), + 'host': ([AttributeType.TYPE_NAME_STRING], owner_attribute), + 'password': ([AttributeType.TYPE_NAME_STRING], owner_attribute), + 'port': ([AttributeType.TYPE_NAME_INT], owner_attribute), + 'type_name': ([AttributeType.TYPE_NAME_STRING], DefaultPermissions.shared_attribute(role_uuids=role_uuids)), + 'username': ([AttributeType.TYPE_NAME_STRING], owner_attribute), + }, required_keys={'host', 'type_name'}) + type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] + if type_name_attributes[0].get_value() != 'mysql': + raise Exception(f"mysql connector type_name value {type_name_attributes[0].get_value()} does not match 'mysql'") + + @staticmethod + def json_schema(): + return JsonSchemaAttributesGroup.json_schema( + key='connector', + group_name='connector mysql', + description="The 'connector mysql' attributes group.", + additional_description="Requires 'host' and 'type_name' attributes.", + contains=[ + JsonSchemaAttribute.json_schema( + key='host', attribute_name="host", description="This item ensures that the 'host' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='type_name', attribute_name="type name", description="This item ensures that the 'type_name' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + ], + attributes=[ + JsonSchemaAttribute.json_schema( + key='charset', + attribute_name='charset', + description="The 'charset' attribute specifies the charset for the client connection.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='host', + attribute_name='host', + description="The 'host' attribute specifies the hostname of the server running the mysql database.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='password', + attribute_name='password', + description="The 'password' attribute specifies the password to log into the mysql server.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='port', + attribute_name='port', + description="The 'port' attribute specifies the port number, where to connect to the mysql server.", + type_name=AttributeType.TYPE_NAME_INT, + additional_value=""""minimum": 1, +"maximum": 65535""" + ), + JsonSchemaAttribute.json_schema( + key='type_name', + attribute_name="type name", + description="The 'type name' attribute specifies the type of connector. In this case it is a 'mysql' connector.", + type_name=AttributeType.TYPE_NAME_STRING, + additional_value="\"const\": \"mysql\"" + ), + JsonSchemaAttribute.json_schema( + key='username', + attribute_name='username', + description="The 'username' attribute specifies the username to log into the mysql server.", + type_name=AttributeType.TYPE_NAME_STRING + ) + ]) diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/connector_postgresql.py b/dq0/sdk/data/metadata/specification/dataset/v1/connector_postgresql.py index d739dd57..7b7bbc01 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/connector_postgresql.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/connector_postgresql.py @@ -41,7 +41,7 @@ def verify_attributes(attributes, role_uuids=None): 'port': ([AttributeType.TYPE_NAME_INT], owner_attribute), 'type_name': ([AttributeType.TYPE_NAME_STRING], DefaultPermissions.shared_attribute(role_uuids=role_uuids)), 'username': ([AttributeType.TYPE_NAME_STRING], owner_attribute), - }, required_keys={'type_name'}) + }, required_keys={'host', 'type_name'}) type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] if type_name_attributes[0].get_value() != 'postgresql': raise Exception(f"postgresql connector type_name value {type_name_attributes[0].get_value()} does not match 'postgresql'") @@ -52,10 +52,17 @@ def json_schema(): key='connector', group_name='connector postgresql', description="The 'connector postgresql' attributes group.", - additional_description="Requires 'type_name' attribute.", - contains=JsonSchemaAttribute.json_schema( - key='type_name', attribute_name="type name", description="This item ensures that the 'type_name' attribute is present.", - type_name=AttributeType.TYPE_NAME_STRING), + additional_description="Requires 'host' and 'type_name' attributes.", + contains=[ + JsonSchemaAttribute.json_schema( + key='host', attribute_name="host", description="This item ensures that the 'host' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='type_name', attribute_name="type name", description="This item ensures that the 'type_name' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + ], attributes=[ JsonSchemaAttribute.json_schema( key='host', diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/connector_sqlite.py b/dq0/sdk/data/metadata/specification/dataset/v1/connector_sqlite.py new file mode 100644 index 00000000..e30bf1aa --- /dev/null +++ b/dq0/sdk/data/metadata/specification/dataset/v1/connector_sqlite.py @@ -0,0 +1,77 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute +from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_list import AttributeList +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class ConnectorSQLite: + @staticmethod + def apply_defaults(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data=None) + applied_attributes = ConnectorSQLite.apply_defaults_to_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + applied_permissions = DefaultPermissions.shared_attribute(role_uuids=role_uuids) \ + if attribute.get_permissions() is None else attribute.get_permissions().copy() + return AttributeList(key=attribute.get_key(), value=applied_attributes, permissions=applied_permissions) + + @staticmethod + def apply_defaults_to_attributes(attributes, role_uuids=None): + Attribute.check_list(attribute_list=attributes, check_data=None) + applied_attributes = [] if attributes is not None else None + for attribute in attributes if attributes is not None else []: + applied_attribute = attribute.copy() + applied_attribute.set_default_permissions(default_permissions=DefaultPermissions.shared_attribute(role_uuids=role_uuids)) + applied_attributes.append(applied_attribute) + return applied_attributes + + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'connector': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.shared_attribute(role_uuids=role_uuids)), + }) + ConnectorSQLite.verify_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + shared_attribute = DefaultPermissions.shared_attribute(role_uuids=role_uuids) + Attribute.check_list(attribute_list=attributes, check_data={ + 'type_name': ([AttributeType.TYPE_NAME_STRING], shared_attribute), + 'uri': ([AttributeType.TYPE_NAME_STRING], shared_attribute) + }, required_keys={'type_name', 'uri'}) + type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] + if type_name_attributes[0].get_value() != 'sqlite': + raise Exception(f"sqlite connector type_name value {type_name_attributes[0].get_value()} does not match 'sqlite'") + + @staticmethod + def json_schema(): + return JsonSchemaAttributesGroup.json_schema( + key='connector', + group_name='connector sqlite', + description="The 'connector sqlite' attributes group.", + additional_description="Requires 'type_name' and 'uri' attributes.", + contains=[ + JsonSchemaAttribute.json_schema( + key='type_name', attribute_name="type name", description="This item ensures that the 'type_name' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='uri', attribute_name="uri", description="This item ensures that the 'uri' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + ], + attributes=[ + JsonSchemaAttribute.json_schema( + key='type_name', + attribute_name="type name", + description="The 'type name' attribute specifies the type of connector. In this case it is a 'sqlite' connector.", + type_name=AttributeType.TYPE_NAME_STRING, + additional_value="\"const\": \"sqlite\"" + ), + JsonSchemaAttribute.json_schema( + key='uri', + attribute_name='uri', + description="The 'uri' attribute specifies the URI pointing to the SQLite file (e.g. a filepath).", + type_name=AttributeType.TYPE_NAME_STRING + ) + ]) diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/database.py b/dq0/sdk/data/metadata/specification/dataset/v1/database.py index 27257539..ee031f5e 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/database.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/database.py @@ -1,10 +1,12 @@ from dq0.sdk.data.metadata.specification.dataset.v1.connector import Connector from dq0.sdk.data.metadata.specification.dataset.v1.connector_csv import ConnectorCSV +from dq0.sdk.data.metadata.specification.dataset.v1.connector_mysql import ConnectorMySQL from dq0.sdk.data.metadata.specification.dataset.v1.connector_postgresql import ConnectorPostgreSQL +from dq0.sdk.data.metadata.specification.dataset.v1.connector_sqlite import ConnectorSQLite from dq0.sdk.data.metadata.specification.dataset.v1.schema import Schema from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions -from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute -from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.dataset.attribute import Attribute as JsonSchemaDatasetAttribute +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaDatasetAttributesGroup from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType @@ -94,28 +96,30 @@ def json_schema(): return JsonSchemaNode.json_schema( NodeType.TYPE_NAME_DATABASE, attributes_groups=[ - JsonSchemaAttributesGroup.data( + JsonSchemaDatasetAttributesGroup.data( attributes=[ - JsonSchemaAttribute.description( + JsonSchemaDatasetAttribute.description( node_type_name=NodeType.TYPE_NAME_DATABASE ), - JsonSchemaAttribute.metadata_is_public( + JsonSchemaDatasetAttribute.metadata_is_public( node_type_name=NodeType.TYPE_NAME_DATABASE ), - JsonSchemaAttribute.name( + JsonSchemaDatasetAttribute.name( node_type_name=NodeType.TYPE_NAME_DATABASE ) ] ), - JsonSchemaAttributesGroup.differential_privacy( + JsonSchemaDatasetAttributesGroup.differential_privacy( attributes=[ - JsonSchemaAttribute.privacy_level( + JsonSchemaDatasetAttribute.privacy_level( node_type_name=NodeType.TYPE_NAME_DATABASE ) ] ), ConnectorCSV.json_schema(), - ConnectorPostgreSQL.json_schema() + ConnectorMySQL.json_schema(), + ConnectorPostgreSQL.json_schema(), + ConnectorSQLite.json_schema() ], child_node_json_schema=Schema.json_schema() ) diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/dataset.py b/dq0/sdk/data/metadata/specification/dataset/v1/dataset.py index 4c8f96cb..a38492f2 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/dataset.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/dataset.py @@ -2,6 +2,8 @@ from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.dataset.attribute import Attribute as JsonSchemaDatasetAttribute +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaDatasetAttributesGroup from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType @@ -93,15 +95,15 @@ def json_schema(): return JsonSchemaNode.json_schema( NodeType.TYPE_NAME_DATASET, attributes_groups=[ - JsonSchemaAttributesGroup.data( + JsonSchemaDatasetAttributesGroup.data( attributes=[ - JsonSchemaAttribute.description( + JsonSchemaDatasetAttribute.description( node_type_name=NodeType.TYPE_NAME_DATASET ), - JsonSchemaAttribute.metadata_is_public( + JsonSchemaDatasetAttribute.metadata_is_public( node_type_name=NodeType.TYPE_NAME_DATASET ), - JsonSchemaAttribute.name( + JsonSchemaDatasetAttribute.name( node_type_name=NodeType.TYPE_NAME_DATASET ), JsonSchemaAttribute.no_key_list( @@ -114,16 +116,16 @@ def json_schema(): ) ] ), - JsonSchemaAttributesGroup.differential_privacy( + JsonSchemaDatasetAttributesGroup.differential_privacy( attributes=[ - JsonSchemaAttribute.privacy_level( + JsonSchemaDatasetAttribute.privacy_level( node_type_name=NodeType.TYPE_NAME_DATASET ) ], contains=JsonSchemaAttribute.json_schema( key='privacy_level', attribute_name="privacy level", - description="This item ensures that the 'privacy level' attribute is present.", + description="This item ensures that the 'privacy_level' attribute is present.", type_name=AttributeType.TYPE_NAME_INT ), additional_description="Requires a 'privacy_level' attribute." diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/schema.py b/dq0/sdk/data/metadata/specification/dataset/v1/schema.py index e2a394a6..e9c3f9af 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/schema.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/schema.py @@ -1,7 +1,7 @@ from dq0.sdk.data.metadata.specification.dataset.v1.table import Table from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions -from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute -from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.dataset.attribute import Attribute as JsonSchemaDatasetAttribute +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaDatasetAttributesGroup from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType @@ -86,22 +86,22 @@ def json_schema(): return JsonSchemaNode.json_schema( NodeType.TYPE_NAME_SCHEMA, attributes_groups=[ - JsonSchemaAttributesGroup.data( + JsonSchemaDatasetAttributesGroup.data( attributes=[ - JsonSchemaAttribute.description( + JsonSchemaDatasetAttribute.description( node_type_name=NodeType.TYPE_NAME_SCHEMA ), - JsonSchemaAttribute.metadata_is_public( + JsonSchemaDatasetAttribute.metadata_is_public( node_type_name=NodeType.TYPE_NAME_SCHEMA ), - JsonSchemaAttribute.name( + JsonSchemaDatasetAttribute.name( node_type_name=NodeType.TYPE_NAME_SCHEMA ) ] ), - JsonSchemaAttributesGroup.differential_privacy( + JsonSchemaDatasetAttributesGroup.differential_privacy( attributes=[ - JsonSchemaAttribute.privacy_level( + JsonSchemaDatasetAttribute.privacy_level( node_type_name=NodeType.TYPE_NAME_SCHEMA ) ] diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/table.py b/dq0/sdk/data/metadata/specification/dataset/v1/table.py index 1f143b58..7f2ef974 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/table.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/table.py @@ -1,7 +1,8 @@ from dq0.sdk.data.metadata.specification.dataset.v1.column import Column from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute -from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.dataset.attribute import Attribute as JsonSchemaDatasetAttribute +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaDatasetAttributesGroup from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType @@ -114,15 +115,15 @@ def json_schema(): return JsonSchemaNode.json_schema( NodeType.TYPE_NAME_TABLE, attributes_groups=[ - JsonSchemaAttributesGroup.data( + JsonSchemaDatasetAttributesGroup.data( attributes=[ - JsonSchemaAttribute.description( + JsonSchemaDatasetAttribute.description( node_type_name=NodeType.TYPE_NAME_TABLE ), - JsonSchemaAttribute.metadata_is_public( + JsonSchemaDatasetAttribute.metadata_is_public( node_type_name=NodeType.TYPE_NAME_TABLE ), - JsonSchemaAttribute.name( + JsonSchemaDatasetAttribute.name( node_type_name=NodeType.TYPE_NAME_TABLE ), JsonSchemaAttribute.json_schema( @@ -135,7 +136,7 @@ def json_schema(): ) ] ), - JsonSchemaAttributesGroup.differential_privacy( + JsonSchemaDatasetAttributesGroup.differential_privacy( attributes=[ JsonSchemaAttribute.json_schema( key='budget_delta', @@ -160,12 +161,12 @@ def json_schema(): f"for the '{NodeType.TYPE_NAME_TABLE}'.", type_name=AttributeType.TYPE_NAME_STRING ), - JsonSchemaAttribute.privacy_level( + JsonSchemaDatasetAttribute.privacy_level( node_type_name=NodeType.TYPE_NAME_TABLE ) ] ), - JsonSchemaAttributesGroup.private_sql( + JsonSchemaDatasetAttributesGroup.private_sql( attributes=[ JsonSchemaAttribute.json_schema( key='censor_dims', @@ -227,7 +228,7 @@ def json_schema(): ) ] ), - JsonSchemaAttributesGroup.private_synthesis( + JsonSchemaDatasetAttributesGroup.private_synthesis( attributes=[ JsonSchemaAttribute.json_schema( key='synth_allowed', diff --git a/dq0/sdk/data/metadata/specification/default_permissions.py b/dq0/sdk/data/metadata/specification/default_permissions.py index 38b5c750..10b78fcf 100644 --- a/dq0/sdk/data/metadata/specification/default_permissions.py +++ b/dq0/sdk/data/metadata/specification/default_permissions.py @@ -58,6 +58,19 @@ def shared_node(role_uuids=None): } return Permissions(permissions=permissions) if len(permissions) != 0 else None + @staticmethod + def analyst_node(role_uuids=None): + permissions = {action: permission for (action, permission) in [ + (Action.READ, DefaultPermissions.select_uuids(role_uuids=role_uuids, role_names={DefaultPermissions.OWNER_NAME, DefaultPermissions.USER_NAME})), + (Action.WRITE_ATTRIBUTES, DefaultPermissions.select_uuids(role_uuids=role_uuids, role_names={DefaultPermissions.OWNER_NAME, + DefaultPermissions.USER_NAME})), + (Action.WRITE_CHILD_NODES, DefaultPermissions.select_uuids(role_uuids=role_uuids, role_names={DefaultPermissions.OWNER_NAME, + DefaultPermissions.USER_NAME})), + (Action.WRITE_PERMISSIONS, DefaultPermissions.select_uuids(role_uuids=role_uuids, role_names={DefaultPermissions.OWNER_NAME})), + ] if permission is not None + } + return Permissions(permissions=permissions) if len(permissions) != 0 else None + @staticmethod def json_schema_node_permissions(): return JsonSchemaPermissions.json_schema(owner='node', properties=[ diff --git a/dq0/sdk/data/metadata/specification/json_schema/attribute.py b/dq0/sdk/data/metadata/specification/json_schema/attribute.py index 9ade7fa9..dde57dfb 100644 --- a/dq0/sdk/data/metadata/specification/json_schema/attribute.py +++ b/dq0/sdk/data/metadata/specification/json_schema/attribute.py @@ -126,41 +126,3 @@ def no_key_list(key, attribute_name, description, item_attribute_name, item_desc description=description, type_name=AttributeType.TYPE_NAME_LIST, additional_value=additional_value) - - @staticmethod - def description(node_type_name): - return Attribute.json_schema( - key='description', - attribute_name='description', - description=f"The 'description' attribute. Describes the '{node_type_name}'.", - type_name=AttributeType.TYPE_NAME_STRING) - - @staticmethod - def metadata_is_public(node_type_name): - return Attribute.json_schema( - key='metadata_is_public', - attribute_name='metadata is public', - description=f"The 'metadata is public' attribute. Specifies whether the '{node_type_name}' metadata is public.", - type_name=AttributeType.TYPE_NAME_BOOLEAN) - - @staticmethod - def name(node_type_name): - return Attribute.json_schema( - key='name', - attribute_name='name', - description=f"The 'name' attribute. Mandatory and required to be unique among all elements of type '{node_type_name}'.", - type_name=AttributeType.TYPE_NAME_STRING - ) - - @staticmethod - def privacy_level(node_type_name): - additional_value = """"minimum": 0, -"maximum": 2""" - return Attribute.json_schema( - key='privacy_level', - attribute_name='privacy level', - description=f"The 'privacy level' attribute. Sets the level of privacy protection for '{node_type_name}'. " - "A child node's setting of 'privacy level' will take precedence for the respective child node. " - "Allowed values are [0, 1, 2].", - type_name=AttributeType.TYPE_NAME_INT, - additional_value=additional_value) diff --git a/dq0/sdk/data/metadata/specification/json_schema/attributes_group.py b/dq0/sdk/data/metadata/specification/json_schema/attributes_group.py index e4b3839f..c3c604d5 100644 --- a/dq0/sdk/data/metadata/specification/json_schema/attributes_group.py +++ b/dq0/sdk/data/metadata/specification/json_schema/attributes_group.py @@ -1,4 +1,4 @@ -from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType @@ -34,67 +34,7 @@ def json_schema(key, group_name, description, additional_description=None, conta }},""" additional_value = f""""minItems": 1,{contains_json}{items_json} "uniqueItemProperties": [ "key" ]""" - return Attribute.json_schema(key=key, attribute_name=f"{group_name} group", description=description, - type_name=AttributeType.TYPE_NAME_LIST, additional_value=additional_value, - additional_description=additional_description) - - @staticmethod - def data(attributes, additional_contains=None, additional_description=''): - contains = Attribute.json_schema( - key='name', - attribute_name='name', - description="This item ensures that the 'name' attribute is present.", - type_name=AttributeType.TYPE_NAME_STRING - ) - if additional_contains is not None: - additional_contains.append(contains) - contains = additional_contains - return AttributesGroup.json_schema( - key='data', - group_name='data', - description="The 'data' attributes group. This group is required.", - additional_description=f"Requires a 'name' attribute.{additional_description}", - contains=contains, - attributes=attributes) - - @staticmethod - def differential_privacy(attributes, contains=None, additional_description=None): - return AttributesGroup.json_schema( - key='differential_privacy', - group_name="differential privacy", - description="The 'differential privacy' attributes group.", - attributes=attributes, - additional_description=additional_description, - contains=contains) - - @staticmethod - def private_sql(attributes): - return AttributesGroup.json_schema( - key='private_sql', - group_name="private sql", - description="The 'private sql' attributes group.", - attributes=attributes) - - @staticmethod - def private_sql_and_synthesis(attributes): - return AttributesGroup.json_schema( - key='private_sql_and_synthesis', - group_name="private sql and synthesis", - description="The 'private sql and synthesis' attributes group.", - attributes=attributes) - - @staticmethod - def private_synthesis(attributes): - return AttributesGroup.json_schema( - key='private_synthesis', - group_name="private synthesis", - description="The 'private synthesis' attributes group.", - attributes=attributes) - - @staticmethod - def machine_learning(attributes): - return AttributesGroup.json_schema( - key='machine_learning', - group_name="machine learning", - description="The 'machine learning' attributes group.", - attributes=attributes) + return JsonSchemaAttribute.json_schema( + key=key, attribute_name=f"{group_name} group", description=description, + type_name=AttributeType.TYPE_NAME_LIST, additional_value=additional_value, + additional_description=additional_description) diff --git a/dq0/sdk/data/metadata/specification/json_schema/dataset/__init__.py b/dq0/sdk/data/metadata/specification/json_schema/dataset/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/data/metadata/specification/json_schema/dataset/attribute.py b/dq0/sdk/data/metadata/specification/json_schema/dataset/attribute.py new file mode 100644 index 00000000..a983b4b9 --- /dev/null +++ b/dq0/sdk/data/metadata/specification/json_schema/dataset/attribute.py @@ -0,0 +1,42 @@ +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class Attribute: + @staticmethod + def description(node_type_name): + return JsonSchemaAttribute.json_schema( + key='description', + attribute_name='description', + description=f"The 'description' attribute. Describes the '{node_type_name}'.", + type_name=AttributeType.TYPE_NAME_STRING) + + @staticmethod + def metadata_is_public(node_type_name): + return JsonSchemaAttribute.json_schema( + key='metadata_is_public', + attribute_name='metadata is public', + description=f"The 'metadata is public' attribute. Specifies whether the '{node_type_name}' metadata is public.", + type_name=AttributeType.TYPE_NAME_BOOLEAN) + + @staticmethod + def name(node_type_name): + return JsonSchemaAttribute.json_schema( + key='name', + attribute_name='name', + description=f"The 'name' attribute. Mandatory and required to be unique among all elements of type '{node_type_name}'.", + type_name=AttributeType.TYPE_NAME_STRING + ) + + @staticmethod + def privacy_level(node_type_name): + additional_value = """"minimum": 0, +"maximum": 2""" + return JsonSchemaAttribute.json_schema( + key='privacy_level', + attribute_name='privacy level', + description=f"The 'privacy level' attribute. Sets the level of privacy protection for '{node_type_name}'. " + "A child node's setting of 'privacy level' will take precedence for the respective child node. " + "Allowed values are [0, 1, 2].", + type_name=AttributeType.TYPE_NAME_INT, + additional_value=additional_value) diff --git a/dq0/sdk/data/metadata/specification/json_schema/dataset/attributes_group.py b/dq0/sdk/data/metadata/specification/json_schema/dataset/attributes_group.py new file mode 100644 index 00000000..c7432588 --- /dev/null +++ b/dq0/sdk/data/metadata/specification/json_schema/dataset/attributes_group.py @@ -0,0 +1,66 @@ +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute +from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class AttributesGroup: + @staticmethod + def data(attributes, additional_contains=None, additional_description=''): + contains = JsonSchemaAttribute.json_schema( + key='name', + attribute_name='name', + description="This item ensures that the 'name' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + if additional_contains is not None: + additional_contains.append(contains) + contains = additional_contains + return JsonSchemaAttributesGroup.json_schema( + key='data', + group_name='data', + description="The 'data' attributes group. This group is required.", + additional_description=f"Requires a 'name' attribute.{additional_description}", + contains=contains, + attributes=attributes) + + @staticmethod + def differential_privacy(attributes, contains=None, additional_description=None): + return JsonSchemaAttributesGroup.json_schema( + key='differential_privacy', + group_name="differential privacy", + description="The 'differential privacy' attributes group.", + attributes=attributes, + additional_description=additional_description, + contains=contains) + + @staticmethod + def private_sql(attributes): + return JsonSchemaAttributesGroup.json_schema( + key='private_sql', + group_name="private sql", + description="The 'private sql' attributes group.", + attributes=attributes) + + @staticmethod + def private_sql_and_synthesis(attributes): + return JsonSchemaAttributesGroup.json_schema( + key='private_sql_and_synthesis', + group_name="private sql and synthesis", + description="The 'private sql and synthesis' attributes group.", + attributes=attributes) + + @staticmethod + def private_synthesis(attributes): + return JsonSchemaAttributesGroup.json_schema( + key='private_synthesis', + group_name="private synthesis", + description="The 'private synthesis' attributes group.", + attributes=attributes) + + @staticmethod + def machine_learning(attributes): + return JsonSchemaAttributesGroup.json_schema( + key='machine_learning', + group_name="machine learning", + description="The 'machine learning' attributes group.", + attributes=attributes) diff --git a/dq0/sdk/data/metadata/specification/json_schema/column.py b/dq0/sdk/data/metadata/specification/json_schema/dataset/column.py similarity index 69% rename from dq0/sdk/data/metadata/specification/json_schema/column.py rename to dq0/sdk/data/metadata/specification/json_schema/dataset/column.py index 72065d8a..de970e77 100644 --- a/dq0/sdk/data/metadata/specification/json_schema/column.py +++ b/dq0/sdk/data/metadata/specification/json_schema/dataset/column.py @@ -1,5 +1,6 @@ -from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute -from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute +from dq0.sdk.data.metadata.specification.json_schema.dataset.attribute import Attribute as JsonSchemaDatasetAttribute +from dq0.sdk.data.metadata.specification.json_schema.dataset.attributes_group import AttributesGroup as JsonSchemaDatasetAttributesGroup from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType from dq0.sdk.data.metadata.structure.node.node_type import NodeType @@ -7,9 +8,9 @@ class Column: @staticmethod def data(data_type_name): - return AttributesGroup.data( + return JsonSchemaDatasetAttributesGroup.data( attributes=[ - Attribute.json_schema( + JsonSchemaAttribute.json_schema( key='data_type_name', attribute_name='data type name', description="The 'data type name' attribute. Specifies the name of the data type " @@ -17,10 +18,10 @@ def data(data_type_name): type_name=AttributeType.TYPE_NAME_STRING, additional_value=f"\"const\": \"{data_type_name}\"" ), - Attribute.description(node_type_name=NodeType.TYPE_NAME_COLUMN), - Attribute.metadata_is_public(node_type_name=NodeType.TYPE_NAME_COLUMN), - Attribute.name(node_type_name=NodeType.TYPE_NAME_COLUMN), - Attribute.json_schema( + JsonSchemaDatasetAttribute.description(node_type_name=NodeType.TYPE_NAME_COLUMN), + JsonSchemaDatasetAttribute.metadata_is_public(node_type_name=NodeType.TYPE_NAME_COLUMN), + JsonSchemaDatasetAttribute.name(node_type_name=NodeType.TYPE_NAME_COLUMN), + JsonSchemaAttribute.json_schema( key='selectable', attribute_name='selectable', description="The 'selectable' attribute. Specifies whether a selection may happen " @@ -29,7 +30,7 @@ def data(data_type_name): ) ], additional_contains=[ - Attribute.json_schema( + JsonSchemaAttribute.json_schema( key='data_type_name', attribute_name='data type name', description="This item ensures that the 'data_type_name' attribute is present.", @@ -41,15 +42,15 @@ def data(data_type_name): @staticmethod def machine_learning(): - return AttributesGroup.machine_learning( + return JsonSchemaDatasetAttributesGroup.machine_learning( attributes=[ - Attribute.json_schema( + JsonSchemaAttribute.json_schema( key='is_feature', attribute_name="is feature", description=f"The 'is feature' attribute. Specifies whether this '{NodeType.TYPE_NAME_COLUMN}' is a feature.", type_name=AttributeType.TYPE_NAME_BOOLEAN ), - Attribute.json_schema( + JsonSchemaAttribute.json_schema( key='is_target', attribute_name="is target", description=f"The 'is target' attribute. Specifies whether this '{NodeType.TYPE_NAME_COLUMN}' is a target.", diff --git a/dq0/sdk/data/metadata/specification/json_schema/metadata.py b/dq0/sdk/data/metadata/specification/json_schema/metadata.py index 6452ab1f..a6f5c9b1 100644 --- a/dq0/sdk/data/metadata/specification/json_schema/metadata.py +++ b/dq0/sdk/data/metadata/specification/json_schema/metadata.py @@ -1,5 +1,6 @@ from dq0.sdk.data.metadata.specification.dataset.v1.dataset import Dataset from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.run.v1.run import Run class Metadata: @@ -8,6 +9,7 @@ def json_schema(): attribute_permissions = DefaultPermissions.json_schema_attribute_permissions().replace('\n', "\n ") node_permissions = DefaultPermissions.json_schema_node_permissions().replace('\n', "\n ") dataset = Dataset.json_schema().replace('\n', "\n ") + run = Run.json_schema().replace('\n', "\n ") return f"""{{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://dq0.io/metadata.schema.json", @@ -36,6 +38,25 @@ def json_schema(): }}, "required": [ "format", "specification" ], "additionalProperties": false + }}, + "meta_run": {{ + "description": "The run root node in the metadata.", + "type": "object", + "properties": {{ + "format": {{ + "description": "The format of the run metadata.", + "type": "string", + "const": "full" + }}, + "node": {run}, + "specification": {{ + "description": "The specification to validate the run metadata against.", + "type": "string", + "const": "run_v1" + }} + }}, + "required": [ "format", "specification" ], + "additionalProperties": false }} }}, "additionalProperties": false diff --git a/dq0/sdk/data/metadata/specification/json_schema/run/__init__.py b/dq0/sdk/data/metadata/specification/json_schema/run/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/data/metadata/specification/json_schema/run/attribute.py b/dq0/sdk/data/metadata/specification/json_schema/run/attribute.py new file mode 100644 index 00000000..f7d23080 --- /dev/null +++ b/dq0/sdk/data/metadata/specification/json_schema/run/attribute.py @@ -0,0 +1,49 @@ +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class Attribute: + @staticmethod + def description(node_type_name): + return JsonSchemaAttribute.json_schema( + key='description', + attribute_name='description', + description=f"The 'description' attribute. Describes the '{node_type_name}'.", + type_name=AttributeType.TYPE_NAME_STRING) + + @staticmethod + def name(node_type_name): + return JsonSchemaAttribute.json_schema( + key='name', + attribute_name='name', + description=f"The 'name' attribute. Mandatory for elements of type '{node_type_name}'.", + type_name=AttributeType.TYPE_NAME_STRING + ) + + @staticmethod + def run_type_name(node_type_name, additional_value): + return JsonSchemaAttribute.json_schema( + key='type_name', + attribute_name='type name', + description=f"The 'type name' attribute. Specifies the type of '{node_type_name}' and is mandatory.", + type_name=AttributeType.TYPE_NAME_STRING, + additional_value=additional_value + ) + + @staticmethod + def query_string(): + return JsonSchemaAttribute.json_schema( + key='query_string', + attribute_name='query string', + description="The 'query string' attribute. Specifies the input sql and is mandatory.", + type_name=AttributeType.TYPE_NAME_STRING + ) + + @staticmethod + def processor_type_name(group_type_name): + return JsonSchemaAttribute.json_schema( + key='type_name', + attribute_name='type name', + description=f"The 'type name' attribute. Specifies the type of '{group_type_name}' and is mandatory.", + type_name=AttributeType.TYPE_NAME_STRING + ) diff --git a/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py b/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py new file mode 100644 index 00000000..d01c0923 --- /dev/null +++ b/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py @@ -0,0 +1,77 @@ +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute +from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class AttributesGroup: + @staticmethod + def data(attributes): + contains = [ + JsonSchemaAttribute.json_schema( + key='name', + attribute_name='name', + description="This item ensures that the 'name' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='run_type_name', + attribute_name='run type name', + description="This item ensures that the 'run_type_name' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + ] + return JsonSchemaAttributesGroup.json_schema( + key='data', + group_name='data', + description="The 'data' attributes group. This group is required.", + additional_description="Requires 'name' and 'type_name' attributes.", + contains=contains, + attributes=attributes) + + @staticmethod + def sql(attributes): + contains = JsonSchemaAttribute.json_schema( + key='query_string', + attribute_name='query string', + description="This item ensures that the 'query_string' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + return JsonSchemaAttributesGroup.json_schema( + key='sql', + group_name='sql', + description="The 'sql' attributes group.", + additional_description="Requires the 'query string' attribute.", + contains=contains, + attributes=attributes) + + @staticmethod + def query_processor(attributes): + contains = JsonSchemaAttribute.json_schema( + key='processor_type_name', + attribute_name='processor type name', + description="This item ensures that the 'processor_type_name' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + return JsonSchemaAttributesGroup.json_schema( + key='query_processor', + group_name='query processor', + description="The 'query processor' attributes group.", + additional_description="Requires the 'processor type name' attribute.", + contains=contains, + attributes=attributes) + + @staticmethod + def result_processor(attributes): + contains = JsonSchemaAttribute.json_schema( + key='processor_type_name', + attribute_name='processor type name', + description="This item ensures that the 'processor_type_name' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + return JsonSchemaAttributesGroup.json_schema( + key='result_processor', + group_name='result processor', + description="The 'result processor' attributes group.", + additional_description="Requires the 'processor type name' attribute.", + contains=contains, + attributes=attributes) diff --git a/dq0/sdk/data/metadata/specification/run/__init__.py b/dq0/sdk/data/metadata/specification/run/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/data/metadata/specification/run/specification_version.py b/dq0/sdk/data/metadata/specification/run/specification_version.py new file mode 100644 index 00000000..c5e263be --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/specification_version.py @@ -0,0 +1,12 @@ +class SpecificationVersion: + # No underscores allowed in these type names + VERSION_V1 = 'v1' + + @staticmethod + def is_valid_version(version): + if not isinstance(version, str): + return False + if \ + version == SpecificationVersion.VERSION_V1: + return True + return False diff --git a/dq0/sdk/data/metadata/specification/run/v1/__init__.py b/dq0/sdk/data/metadata/specification/run/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dq0/sdk/data/metadata/specification/run/v1/run.py b/dq0/sdk/data/metadata/specification/run/v1/run.py new file mode 100644 index 00000000..c0ad2719 --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/run.py @@ -0,0 +1,126 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.attributes_group import AttributesGroup as JsonSchemaAttributesGroup +from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode +from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType +from dq0.sdk.data.metadata.structure.node.node import Node +from dq0.sdk.data.metadata.structure.node.node_type import NodeType + + +class Run: + @staticmethod + def apply_defaults(node, role_uuids=None): + Node.check(node=node, allowed_type_names=None, allowed_permissions=None) + applied_attributes = Run.apply_defaults_to_attributes(attributes=node.get_attributes(), role_uuids=role_uuids) + applied_permissions = DefaultPermissions.analyst_node(role_uuids=role_uuids) if node.get_permissions() is None else node.get_permissions().copy() + return Node(type_name=node.get_type_name(), attributes=applied_attributes, permissions=applied_permissions) + + @staticmethod + def apply_defaults_to_attributes(attributes, role_uuids=None): + Attribute.check_list(attribute_list=attributes, check_data=None) + analyst_attribute = DefaultPermissions.analyst_attribute(role_uuids=role_uuids) + applied_attributes = [] if attributes is not None else None + for attribute in attributes if attributes is not None else []: + applied_attribute = attribute.copy() + applied_attribute.set_default_permissions(default_permissions=analyst_attribute) + applied_attributes.append(applied_attribute) + return applied_attributes + + @staticmethod + def verify(node, role_uuids=None): + Node.check(node=node, allowed_type_names=[NodeType.TYPE_NAME_RUN], allowed_permissions=DefaultPermissions.analyst_attribute(role_uuids=role_uuids)) + Run.verify_attributes(attributes=node.get_attributes(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + analyst_attribute = DefaultPermissions.analyst_attribute(role_uuids=role_uuids) + Attribute.check_list(attribute_list=attributes, check_data={ + 'data': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), + 'sql': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), + }, required_keys={'data'}) + data_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'data'] \ + if attributes is not None else [] + if 0 < len(data_attributes): + Attribute.check_list(attribute_list=data_attributes[0].get_value(), check_data={ + 'description': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'type_name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + }, required_keys={'name', 'type_name'}) + sql_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'sql'] \ + if attributes is not None else [] + if 0 < len(sql_attributes): + Attribute.check_list(attribute_list=sql_attributes[0].get_value(), check_data={ + 'query_string': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'query_processor': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), + 'result_processor': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), + }, required_keys={'query_string'}) + query_processor_attributes = [tmp_attribute for tmp_attribute in sql_attributes if tmp_attribute.get_key() == 'query_processor'] \ + if sql_attributes is not None else [] + if 0 < len(query_processor_attributes): + Attribute.check_list(attribute_list=query_processor_attributes[0].get_value(), check_data={ + 'type_name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + }, required_keys={'type_name'}) + result_processor_attributes = [tmp_attribute for tmp_attribute in sql_attributes if tmp_attribute.get_key() == 'result_processor'] \ + if sql_attributes is not None else [] + if 0 < len(result_processor_attributes): + Attribute.check_list(attribute_list=result_processor_attributes[0].get_value(), check_data={ + 'type_name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + }, required_keys={'type_name'}) + + @staticmethod + def json_schema(): + run_sql = Run.sql_json_schema().replace('\n', "\n ") + return f"""{{ + "oneOf": [ + {run_sql} + ] +}}""" + + @staticmethod + def sql_json_schema(): + return JsonSchemaNode.json_schema( + NodeType.TYPE_NAME_RUN, + attributes_groups=[ + JsonSchemaRunAttributesGroup.data( + attributes=[ + JsonSchemaRunAttribute.description( + node_type_name=NodeType.TYPE_NAME_RUN + ), + JsonSchemaRunAttribute.name( + node_type_name=NodeType.TYPE_NAME_RUN + ), + JsonSchemaRunAttribute.run_type_name( + node_type_name=NodeType.TYPE_NAME_RUN, + additional_value="\"const\": \"sql\"" + ) + ] + ), + JsonSchemaRunAttributesGroup.sql( + attributes=[ + JsonSchemaRunAttribute.query_string(), + JsonSchemaRunAttributesGroup.query_processor( + attributes=[ + JsonSchemaRunAttribute.processor_type_name( + group_type_name='query_processor' + ) + ] + ), + JsonSchemaRunAttributesGroup.result_processor( + attributes=[ + JsonSchemaRunAttribute.processor_type_name( + group_type_name='result_processor' + ) + ] + ) + ] + ) + ], + attributes_groups_additional_contains=JsonSchemaAttributesGroup.json_schema( + key='sql', + group_name="sql", + description="This item ensures, that the 'sql' attributes group is present." + ), + attributes_groups_additional_description="Requires 'sql' attributes group.", + ) diff --git a/dq0/sdk/data/metadata/specification/run/v1/specification_v1.py b/dq0/sdk/data/metadata/specification/run/v1/specification_v1.py new file mode 100644 index 00000000..e3f3ed6f --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/specification_v1.py @@ -0,0 +1,26 @@ +from dq0.sdk.data.metadata.specification.run.specification_version import SpecificationVersion +from dq0.sdk.data.metadata.specification.run.v1.run import Run +from dq0.sdk.data.metadata.specification.specification import Specification +from dq0.sdk.data.metadata.structure.node.node_type import NodeType + + +class SpecificationV1(Specification): + def check_version(version): + if not SpecificationVersion.is_valid_version(version=version): + raise Exception(f"version {version} is invalid") + if version != SpecificationVersion.VERSION_V1: + raise Exception(f"version mismatch {version} != {SpecificationVersion.VERSION_V1}") + + @staticmethod + def check(specification): + if not isinstance(specification, SpecificationV1): + raise Exception(f"specification is not of type SpecificationV1, is of type {type(specification)} instead") + SpecificationV1.check_version(specification.version) + + def __init__(self, role_uuids=None): + super().__init__( + node_type_name=NodeType.TYPE_NAME_RUN, + version=SpecificationVersion.VERSION_V1, + apply_defaults_func=Run.apply_defaults, + verify_func=Run.verify, + role_uuids=role_uuids) diff --git a/dq0/sdk/data/metadata/specification/specification_factory.py b/dq0/sdk/data/metadata/specification/specification_factory.py index eb45b703..dfd82a3f 100644 --- a/dq0/sdk/data/metadata/specification/specification_factory.py +++ b/dq0/sdk/data/metadata/specification/specification_factory.py @@ -1,5 +1,7 @@ -from dq0.sdk.data.metadata.specification.dataset.specification_version import SpecificationVersion +from dq0.sdk.data.metadata.specification.dataset.specification_version import SpecificationVersion as DatasetSpecificationVersion from dq0.sdk.data.metadata.specification.dataset.v1.specification_v1 import SpecificationV1 as DatasetSpecificationV1 +from dq0.sdk.data.metadata.specification.run.specification_version import SpecificationVersion as RunSpecificationVersion +from dq0.sdk.data.metadata.specification.run.v1.specification_v1 import SpecificationV1 as RunSpecificationV1 from dq0.sdk.data.metadata.structure.node.node_type import NodeType @@ -19,12 +21,19 @@ def from_specification_string(specification_string, role_uuids=None): def from_type_and_version(node_type_name, version, role_uuids=None): if not NodeType.is_valid_type_name(type_name=node_type_name): raise Exception(f"node_type_name {node_type_name} is invalid") - if not SpecificationVersion.is_valid_version(version=version): - raise Exception(f"version {version} is invalid") if node_type_name == NodeType.TYPE_NAME_DATASET: - if version == SpecificationVersion.VERSION_V1: + if not DatasetSpecificationVersion.is_valid_version(version=version): + raise Exception(f"version {version} is invalid") + if version == DatasetSpecificationVersion.VERSION_V1: return DatasetSpecificationV1(role_uuids=role_uuids) else: raise Exception(f"unknown version {version}") + if node_type_name == NodeType.TYPE_NAME_RUN: + if not RunSpecificationVersion.is_valid_version(version=version): + raise Exception(f"version {version} is invalid") + if version == RunSpecificationVersion.VERSION_V1: + return RunSpecificationV1(role_uuids=role_uuids) + else: + raise Exception(f"unknown version {version}") else: raise Exception(f"unknown node_type_name {node_type_name}") diff --git a/dq0/sdk/data/metadata/structure/metadata.py b/dq0/sdk/data/metadata/structure/metadata.py index a6d6c658..53eeb251 100644 --- a/dq0/sdk/data/metadata/structure/metadata.py +++ b/dq0/sdk/data/metadata/structure/metadata.py @@ -9,7 +9,7 @@ class Metadata: - ROOT_KEYS = {'dataset'} + ROOT_KEYS = {'dataset', 'run'} @staticmethod def from_yaml_file(filename, role_uuids=None): diff --git a/dq0/sdk/data/metadata/structure/node/node_type.py b/dq0/sdk/data/metadata/structure/node/node_type.py index 05eabfc9..9d215558 100644 --- a/dq0/sdk/data/metadata/structure/node/node_type.py +++ b/dq0/sdk/data/metadata/structure/node/node_type.py @@ -1,10 +1,13 @@ class NodeType: # No underscores allowed in these type names + # dataset TYPE_NAME_DATASET = 'dataset' TYPE_NAME_DATABASE = 'database' TYPE_NAME_SCHEMA = 'schema' TYPE_NAME_TABLE = 'table' TYPE_NAME_COLUMN = 'column' + # run + TYPE_NAME_RUN = 'run' @staticmethod def is_valid_type_name(type_name): @@ -15,6 +18,7 @@ def is_valid_type_name(type_name): type_name == NodeType.TYPE_NAME_DATABASE or \ type_name == NodeType.TYPE_NAME_SCHEMA or \ type_name == NodeType.TYPE_NAME_TABLE or \ - type_name == NodeType.TYPE_NAME_COLUMN: + type_name == NodeType.TYPE_NAME_COLUMN or \ + type_name == NodeType.TYPE_NAME_RUN: return True return False From f0f3184c2fb0d112fd6d7a8d4ae73c1b4b19ed87 Mon Sep 17 00:00:00 2001 From: Reinhard Munz Date: Tue, 15 Mar 2022 23:08:42 +0100 Subject: [PATCH 3/8] moved kind specific factory functions into each kind --- dq0/sdk/connector/connector_factory.py | 58 ++----------------- dq0/sdk/connector/kind/csv/csv.py | 18 ++++++ dq0/sdk/connector/kind/sql/db/mysql/mysql.py | 13 +++++ .../kind/sql/db/postgresql/postgresql.py | 12 ++++ .../connector/kind/sql/db/sqlite/sqlite.py | 7 +++ 5 files changed, 54 insertions(+), 54 deletions(-) diff --git a/dq0/sdk/connector/connector_factory.py b/dq0/sdk/connector/connector_factory.py index c437a068..4b6622cd 100644 --- a/dq0/sdk/connector/connector_factory.py +++ b/dq0/sdk/connector/connector_factory.py @@ -10,61 +10,11 @@ def connector_from(meta_database): meta_connector = meta_database.connector type_name = meta_connector.type_name if type_name == 'csv': - return ConnectorFactory.csv_from(meta_connector=meta_connector) + return CSV.csv_from(meta_connector=meta_connector) if type_name == 'mysql': - return ConnectorFactory.mysql_from(meta_database=meta_database) + return MySQL.mysql_from(meta_database=meta_database) if type_name == 'postgresql': - return ConnectorFactory.postgresql_from(meta_database=meta_database) + return PostgreSQL.postgresql_from(meta_database=meta_database) if type_name == 'sqlite': - return ConnectorFactory.sqlite_from(meta_connector=meta_connector) + return SQLite.sqlite_from(meta_connector=meta_connector) raise ValueError(f"type_name={type_name} is unknown") - - @staticmethod - def csv_from(meta_connector): - if meta_connector.type_name != 'csv': - raise ValueError(f"type_name={meta_connector.type_name} != csv") - filepath = meta_connector.uri - if len(filepath) == 0: - raise ValueError("uri not specified") - sep = meta_connector.sep if meta_connector.sep is not None else ',' - header = meta_connector.header_row if meta_connector.header_row is not None else 'infer' - names = meta_connector.header_columns - index_col = meta_connector.index_col - skipinitialspace = meta_connector.skipinitialspace \ - if meta_connector.skipinitialspace is not None else False - na_values = meta_connector.na_values - decimal = meta_connector.decimal if meta_connector.decimal is not None else '.' - return CSV(filepath=filepath, sep=sep, header=header, names=names, index_col=index_col, - skipinitialspace=skipinitialspace, na_values=na_values, decimal=decimal) - - @staticmethod - def mysql_from(meta_database): - meta_connector = meta_database.connector - if meta_connector.type_name != 'mysql': - raise ValueError(f"type_name={meta_connector.type_name} != mysql") - username = meta_connector.username if meta_connector.username is not None else '' - password = meta_connector.password if meta_connector.password is not None else '' - host = meta_connector.host if meta_connector.host is not None else '' - port = meta_connector.port if meta_connector.port is not None else '' - database = meta_database.data.name if meta_database.data.name is not None else '' - charset = meta_connector.charset if meta_connector.charset is not None else '' - return MySQL(username=username, password=password, host=host, port=port, database=database, charset=charset) - - @staticmethod - def postgresql_from(meta_database): - meta_connector = meta_database.connector - if meta_connector.type_name != 'postgresql': - raise ValueError(f"type_name={meta_connector.type_name} != postgresql") - username = meta_connector.username if meta_connector.username is not None else '' - password = meta_connector.password if meta_connector.password is not None else '' - host = meta_connector.host if meta_connector.host is not None else '' - port = meta_connector.port if meta_connector.port is not None else '' - database = meta_database.data.name if meta_database.data.name is not None else '' - return PostgreSQL(username=username, password=password, host=host, port=port, database=database) - - @staticmethod - def sqlite_from(meta_connector): - if meta_connector.type_name != 'sqlite': - raise ValueError(f"type_name={meta_connector.type_name} != sqlite") - uri = meta_connector.uri if meta_connector.uri is not None else '' - return SQLite(filepath=uri) diff --git a/dq0/sdk/connector/kind/csv/csv.py b/dq0/sdk/connector/kind/csv/csv.py index b93b9680..4264b30f 100644 --- a/dq0/sdk/connector/kind/csv/csv.py +++ b/dq0/sdk/connector/kind/csv/csv.py @@ -7,6 +7,24 @@ class CSV(Connector): + @staticmethod + def csv_from(meta_connector): + if meta_connector.type_name != 'csv': + raise ValueError(f"type_name={meta_connector.type_name} != csv") + filepath = meta_connector.uri + if len(filepath) == 0: + raise ValueError("uri not specified") + sep = meta_connector.sep if meta_connector.sep is not None else ',' + header = meta_connector.header_row if meta_connector.header_row is not None else 'infer' + names = meta_connector.header_columns + index_col = meta_connector.index_col + skipinitialspace = meta_connector.skipinitialspace \ + if meta_connector.skipinitialspace is not None else False + na_values = meta_connector.na_values + decimal = meta_connector.decimal if meta_connector.decimal is not None else '.' + return CSV(filepath=filepath, sep=sep, header=header, names=names, index_col=index_col, + skipinitialspace=skipinitialspace, na_values=na_values, decimal=decimal) + def __init__(self, filepath, sep=',', header='infer', names=None, index_col=None, skipinitialspace=False, na_values=None, decimal='.'): super().__init__(ConnectorKind.CSV) diff --git a/dq0/sdk/connector/kind/sql/db/mysql/mysql.py b/dq0/sdk/connector/kind/sql/db/mysql/mysql.py index d36a687c..272e2350 100644 --- a/dq0/sdk/connector/kind/sql/db/mysql/mysql.py +++ b/dq0/sdk/connector/kind/sql/db/mysql/mysql.py @@ -6,6 +6,19 @@ class MySQL(SQL): + @staticmethod + def mysql_from(meta_database): + meta_connector = meta_database.connector + if meta_connector.type_name != 'mysql': + raise ValueError(f"type_name={meta_connector.type_name} != mysql") + username = meta_connector.username if meta_connector.username is not None else '' + password = meta_connector.password if meta_connector.password is not None else '' + host = meta_connector.host if meta_connector.host is not None else '' + port = meta_connector.port if meta_connector.port is not None else '' + database = meta_database.data.name if meta_database.data.name is not None else '' + charset = meta_connector.charset if meta_connector.charset is not None else '' + return MySQL(username=username, password=password, host=host, port=port, database=database, charset=charset) + @staticmethod def connection_uri(username='', password='', host='', port='', database='', charset=''): if len(username) == 0: diff --git a/dq0/sdk/connector/kind/sql/db/postgresql/postgresql.py b/dq0/sdk/connector/kind/sql/db/postgresql/postgresql.py index cfbd8a69..736b66f2 100644 --- a/dq0/sdk/connector/kind/sql/db/postgresql/postgresql.py +++ b/dq0/sdk/connector/kind/sql/db/postgresql/postgresql.py @@ -6,6 +6,18 @@ class PostgreSQL(SQL): + @staticmethod + def postgresql_from(meta_database): + meta_connector = meta_database.connector + if meta_connector.type_name != 'postgresql': + raise ValueError(f"type_name={meta_connector.type_name} != postgresql") + username = meta_connector.username if meta_connector.username is not None else '' + password = meta_connector.password if meta_connector.password is not None else '' + host = meta_connector.host if meta_connector.host is not None else '' + port = meta_connector.port if meta_connector.port is not None else '' + database = meta_database.data.name if meta_database.data.name is not None else '' + return PostgreSQL(username=username, password=password, host=host, port=port, database=database) + @staticmethod def connection_uri(username='', password='', host='', port='', database=''): if len(username) == 0: diff --git a/dq0/sdk/connector/kind/sql/db/sqlite/sqlite.py b/dq0/sdk/connector/kind/sql/db/sqlite/sqlite.py index 5f33b76d..0fc30cb6 100644 --- a/dq0/sdk/connector/kind/sql/db/sqlite/sqlite.py +++ b/dq0/sdk/connector/kind/sql/db/sqlite/sqlite.py @@ -8,6 +8,13 @@ class SQLite(SQL): + @staticmethod + def sqlite_from(meta_connector): + if meta_connector.type_name != 'sqlite': + raise ValueError(f"type_name={meta_connector.type_name} != sqlite") + uri = meta_connector.uri if meta_connector.uri is not None else '' + return SQLite(filepath=uri) + @staticmethod def connection_uri(filepath): absolute_path = filepath.resolve() From 1870967855d90856c9b5be617e6ca86aa0873a1b Mon Sep 17 00:00:00 2001 From: Reinhard Munz Date: Tue, 5 Apr 2022 15:51:57 +0200 Subject: [PATCH 4/8] extended metadata specification and interface for run to have separate sql with separate processor files --- dq0/sdk/connector/kind/sql/query.py | 2 + .../run/attributes_run_sql_query_processor.py | 6 +++ .../attributes_run_sql_result_processor.py | 48 ++++++++++++++++++ .../json_schema/run/attribute.py | 7 +-- .../json_schema/run/attributes_group.py | 18 +++---- .../specification/run/v1/query_processor.py | 23 +++++++++ .../run/v1/query_processor_dummy.py | 34 +++++++++++++ .../run/v1/query_processor_opendp.py | 34 +++++++++++++ .../specification/run/v1/result_processor.py | 23 +++++++++ .../run/v1/result_processor_dummy.py | 34 +++++++++++++ .../run/v1/result_processor_opendp.py | 50 +++++++++++++++++++ .../data/metadata/specification/run/v1/run.py | 39 ++------------- .../data/metadata/specification/run/v1/sql.py | 49 ++++++++++++++++++ 13 files changed, 319 insertions(+), 48 deletions(-) create mode 100644 dq0/sdk/data/metadata/specification/run/v1/query_processor.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/query_processor_dummy.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/query_processor_opendp.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/result_processor.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/result_processor_dummy.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/result_processor_opendp.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/sql.py diff --git a/dq0/sdk/connector/kind/sql/query.py b/dq0/sdk/connector/kind/sql/query.py index 0c15ecb9..0e07234e 100644 --- a/dq0/sdk/connector/kind/sql/query.py +++ b/dq0/sdk/connector/kind/sql/query.py @@ -5,6 +5,8 @@ class Query: def __init__(self, connector, query_string): if not ConnectorKind.is_valid_sql(kind=connector.get_kind()): raise ValueError(f"connector with kind={connector.get_kind()} is not of valid sql kind") + if not isinstance(query_string, str): + raise ValueError(f"query_string={query_string} is not of type str, is of type {type(query_string)} instead") self._connector = connector self._query_string = query_string diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_query_processor.py b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_query_processor.py index 901c9f46..f7981e2e 100644 --- a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_query_processor.py +++ b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_query_processor.py @@ -10,6 +10,10 @@ def __init__(self, attributes_run_sql, attribute_list=None): entity=attributes_run_sql, attribute_list=attribute_list) + def type_names_check(self, type_names): + if self.type_name not in type_names: + raise Exception(f"access forbidden, typename mismatch: {self.type_name} not in {type_names}") + # type_name @property def type_name(self): @@ -17,6 +21,8 @@ def type_name(self): @type_name.setter def type_name(self, new_type_name): + if self.type_name is not None: + raise Exception(f"type_name {self.type_name} may not be modified") self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, key='type_name', value=new_type_name, diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py index ac301674..c25fd1b8 100644 --- a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py +++ b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py @@ -10,6 +10,10 @@ def __init__(self, attributes_run_sql, attribute_list=None): entity=attributes_run_sql, attribute_list=attribute_list) + def type_names_check(self, type_names): + if self.type_name not in type_names: + raise Exception(f"access forbidden, typename mismatch: {self.type_name} not in {type_names}") + # type_name @property def type_name(self): @@ -17,6 +21,8 @@ def type_name(self): @type_name.setter def type_name(self, new_type_name): + if self.type_name is not None: + raise Exception(f"type_name {self.type_name} may not be modified") self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, key='type_name', value=new_type_name, @@ -26,3 +32,45 @@ def type_name(self, new_type_name): @type_name.deleter def type_name(self): raise Exception(f"type_name {self.type_name} may not be deleted") + + # ========================================================================================================================================================== + # modular properties + # ========================================================================================================================================================== + + # epsilon + @property + def epsilon(self): + self.type_names_check(type_names={'opendp'}) + return self.get_attribute_value(key='epsilon') + + @epsilon.setter + def epsilon(self, new_epsilon): + self.type_names_check(type_names={'opendp'}) + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_FLOAT, + key='epsilon', + value=new_epsilon, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + + @epsilon.deleter + def epsilon(self): + self.type_names_check(type_names={'opendp'}) + self.delete_attribute(key='epsilon') + + # delta + @property + def delta(self): + self.type_names_check(type_names={'opendp'}) + return self.get_attribute_value(key='delta') + + @delta.setter + def delta(self, new_delta): + self.type_names_check(type_names={'opendp'}) + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_FLOAT, + key='delta', + value=new_delta, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + + @delta.deleter + def delta(self): + self.type_names_check(type_names={'opendp'}) + self.delete_attribute(key='delta') diff --git a/dq0/sdk/data/metadata/specification/json_schema/run/attribute.py b/dq0/sdk/data/metadata/specification/json_schema/run/attribute.py index f7d23080..fbdc6ca1 100644 --- a/dq0/sdk/data/metadata/specification/json_schema/run/attribute.py +++ b/dq0/sdk/data/metadata/specification/json_schema/run/attribute.py @@ -40,10 +40,11 @@ def query_string(): ) @staticmethod - def processor_type_name(group_type_name): + def processor_type_name(group_name, additional_value): return JsonSchemaAttribute.json_schema( key='type_name', attribute_name='type name', - description=f"The 'type name' attribute. Specifies the type of '{group_type_name}' and is mandatory.", - type_name=AttributeType.TYPE_NAME_STRING + description=f"The 'type name' attribute. Specifies the type of '{group_name}' and is mandatory.", + type_name=AttributeType.TYPE_NAME_STRING, + additional_value=additional_value ) diff --git a/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py b/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py index d01c0923..c55c234e 100644 --- a/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py +++ b/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py @@ -40,38 +40,38 @@ def sql(attributes): key='sql', group_name='sql', description="The 'sql' attributes group.", - additional_description="Requires the 'query string' attribute.", + additional_description="Requires the 'query_string' attribute.", contains=contains, attributes=attributes) @staticmethod def query_processor(attributes): contains = JsonSchemaAttribute.json_schema( - key='processor_type_name', - attribute_name='processor type name', - description="This item ensures that the 'processor_type_name' attribute is present.", + key='type_name', + attribute_name='type name', + description="This item ensures that the 'type_name' attribute is present.", type_name=AttributeType.TYPE_NAME_STRING ) return JsonSchemaAttributesGroup.json_schema( key='query_processor', group_name='query processor', description="The 'query processor' attributes group.", - additional_description="Requires the 'processor type name' attribute.", + additional_description="Requires the 'type_name' attribute.", contains=contains, attributes=attributes) @staticmethod def result_processor(attributes): contains = JsonSchemaAttribute.json_schema( - key='processor_type_name', - attribute_name='processor type name', - description="This item ensures that the 'processor_type_name' attribute is present.", + key='type_name', + attribute_name='type name', + description="This item ensures that the 'type_name' attribute is present.", type_name=AttributeType.TYPE_NAME_STRING ) return JsonSchemaAttributesGroup.json_schema( key='result_processor', group_name='result processor', description="The 'result processor' attributes group.", - additional_description="Requires the 'processor type name' attribute.", + additional_description="Requires the 'type_name' attribute.", contains=contains, attributes=attributes) diff --git a/dq0/sdk/data/metadata/specification/run/v1/query_processor.py b/dq0/sdk/data/metadata/specification/run/v1/query_processor.py new file mode 100644 index 00000000..ed86c98c --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/query_processor.py @@ -0,0 +1,23 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.run.v1.query_processor_dummy import QueryProcessorDummy +from dq0.sdk.data.metadata.specification.run.v1.query_processor_opendp import QueryProcessorOpenDP +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_string import AttributeString +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class QueryProcessor: + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'query_processor': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }) + attribute_type_name = attribute.get_attribute(key='type_name') + if not isinstance(attribute_type_name, AttributeString): + raise Exception(f"attribute type_name is not of type AttributeString, is of type {type(attribute_type_name)} instead") + if attribute_type_name.get_value() == 'dummy': + return QueryProcessorDummy.verify(attribute=attribute, role_uuids=role_uuids) + elif attribute_type_name.get_value() == 'opendp': + return QueryProcessorOpenDP.verify(attribute=attribute, role_uuids=role_uuids) + else: + raise Exception(f"unknown query processor type_name {attribute_type_name.get_value()}") diff --git a/dq0/sdk/data/metadata/specification/run/v1/query_processor_dummy.py b/dq0/sdk/data/metadata/specification/run/v1/query_processor_dummy.py new file mode 100644 index 00000000..0088aa9f --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/query_processor_dummy.py @@ -0,0 +1,34 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class QueryProcessorDummy: + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'query_processor': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }) + QueryProcessorDummy.verify_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + Attribute.check_list(attribute_list=attributes, check_data={ + 'type_name': ([AttributeType.TYPE_NAME_STRING], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }, required_keys={'type_name'}) + type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] + if type_name_attributes[0].get_value() != 'dummy': + raise Exception(f"query processor type_name value {type_name_attributes[0].get_value()} does not match 'dummy'") + + @staticmethod + def json_schema(): + return JsonSchemaRunAttributesGroup.query_processor( + attributes=[ + JsonSchemaRunAttribute.processor_type_name( + group_name='query_processor', + additional_value="\"const\": \"dummy\"" + ) + ] + ) diff --git a/dq0/sdk/data/metadata/specification/run/v1/query_processor_opendp.py b/dq0/sdk/data/metadata/specification/run/v1/query_processor_opendp.py new file mode 100644 index 00000000..eda41687 --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/query_processor_opendp.py @@ -0,0 +1,34 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class QueryProcessorOpenDP: + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'query_processor': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }) + QueryProcessorOpenDP.verify_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + Attribute.check_list(attribute_list=attributes, check_data={ + 'type_name': ([AttributeType.TYPE_NAME_STRING], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }, required_keys={'type_name'}) + type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] + if type_name_attributes[0].get_value() != 'opendp': + raise Exception(f"query processor type_name value {type_name_attributes[0].get_value()} does not match 'opendp'") + + @staticmethod + def json_schema(): + return JsonSchemaRunAttributesGroup.query_processor( + attributes=[ + JsonSchemaRunAttribute.processor_type_name( + group_name='query_processor', + additional_value="\"const\": \"opendp\"" + ) + ] + ) diff --git a/dq0/sdk/data/metadata/specification/run/v1/result_processor.py b/dq0/sdk/data/metadata/specification/run/v1/result_processor.py new file mode 100644 index 00000000..bb3a7a8a --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/result_processor.py @@ -0,0 +1,23 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.run.v1.result_processor_dummy import ResultProcessorDummy +from dq0.sdk.data.metadata.specification.run.v1.result_processor_opendp import ResultProcessorOpenDP +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_string import AttributeString +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class ResultProcessor: + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'result_processor': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }) + attribute_type_name = attribute.get_attribute(key='type_name') + if not isinstance(attribute_type_name, AttributeString): + raise Exception(f"attribute type_name is not of type AttributeString, is of type {type(attribute_type_name)} instead") + if attribute_type_name.get_value() == 'dummy': + return ResultProcessorDummy.verify(attribute=attribute, role_uuids=role_uuids) + elif attribute_type_name.get_value() == 'opendp': + return ResultProcessorOpenDP.verify(attribute=attribute, role_uuids=role_uuids) + else: + raise Exception(f"unknown result processor type_name {attribute_type_name.get_value()}") diff --git a/dq0/sdk/data/metadata/specification/run/v1/result_processor_dummy.py b/dq0/sdk/data/metadata/specification/run/v1/result_processor_dummy.py new file mode 100644 index 00000000..70eacee5 --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/result_processor_dummy.py @@ -0,0 +1,34 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class ResultProcessorDummy: + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'result_processor': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }) + ResultProcessorDummy.verify_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + Attribute.check_list(attribute_list=attributes, check_data={ + 'type_name': ([AttributeType.TYPE_NAME_STRING], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }, required_keys={'type_name'}) + type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] + if type_name_attributes[0].get_value() != 'dummy': + raise Exception(f"result processor type_name value {type_name_attributes[0].get_value()} does not match 'dummy'") + + @staticmethod + def json_schema(): + return JsonSchemaRunAttributesGroup.result_processor( + attributes=[ + JsonSchemaRunAttribute.processor_type_name( + group_name='result_processor', + additional_value="\"const\": \"dummy\"" + ) + ] + ) diff --git a/dq0/sdk/data/metadata/specification/run/v1/result_processor_opendp.py b/dq0/sdk/data/metadata/specification/run/v1/result_processor_opendp.py new file mode 100644 index 00000000..6534009e --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/result_processor_opendp.py @@ -0,0 +1,50 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class ResultProcessorOpenDP: + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'result_processor': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }) + ResultProcessorOpenDP.verify_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + analyst_attribute = DefaultPermissions.analyst_attribute(role_uuids=role_uuids) + Attribute.check_list(attribute_list=attributes, check_data={ + 'type_name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'epsilon': ([AttributeType.TYPE_NAME_FLOAT], analyst_attribute), + 'delta': ([AttributeType.TYPE_NAME_FLOAT], analyst_attribute), + }, required_keys={'type_name'}) + type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] + if type_name_attributes[0].get_value() != 'opendp': + raise Exception(f"result processor type_name value {type_name_attributes[0].get_value()} does not match 'opendp'") + + @staticmethod + def json_schema(): + return JsonSchemaRunAttributesGroup.result_processor( + attributes=[ + JsonSchemaRunAttribute.processor_type_name( + group_name='result_processor', + additional_value="\"const\": \"opendp\"" + ), + JsonSchemaAttribute.json_schema( + key='epsilon', + attribute_name='epsilon', + description="The 'epsilon' attribute.", + type_name=AttributeType.TYPE_NAME_FLOAT + ), + JsonSchemaAttribute.json_schema( + key='delta', + attribute_name='delta', + description="The 'delta' attribute.", + type_name=AttributeType.TYPE_NAME_FLOAT + ) + ] + ) diff --git a/dq0/sdk/data/metadata/specification/run/v1/run.py b/dq0/sdk/data/metadata/specification/run/v1/run.py index c0ad2719..41de489a 100644 --- a/dq0/sdk/data/metadata/specification/run/v1/run.py +++ b/dq0/sdk/data/metadata/specification/run/v1/run.py @@ -3,6 +3,7 @@ from dq0.sdk.data.metadata.specification.json_schema.node import Node as JsonSchemaNode from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup +from dq0.sdk.data.metadata.specification.run.v1.sql import Sql from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType from dq0.sdk.data.metadata.structure.node.node import Node @@ -51,23 +52,7 @@ def verify_attributes(attributes, role_uuids=None): sql_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'sql'] \ if attributes is not None else [] if 0 < len(sql_attributes): - Attribute.check_list(attribute_list=sql_attributes[0].get_value(), check_data={ - 'query_string': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), - 'query_processor': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), - 'result_processor': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), - }, required_keys={'query_string'}) - query_processor_attributes = [tmp_attribute for tmp_attribute in sql_attributes if tmp_attribute.get_key() == 'query_processor'] \ - if sql_attributes is not None else [] - if 0 < len(query_processor_attributes): - Attribute.check_list(attribute_list=query_processor_attributes[0].get_value(), check_data={ - 'type_name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), - }, required_keys={'type_name'}) - result_processor_attributes = [tmp_attribute for tmp_attribute in sql_attributes if tmp_attribute.get_key() == 'result_processor'] \ - if sql_attributes is not None else [] - if 0 < len(result_processor_attributes): - Attribute.check_list(attribute_list=result_processor_attributes[0].get_value(), check_data={ - 'type_name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), - }, required_keys={'type_name'}) + Sql.verify(attribute=sql_attributes[0], role_uuids=role_uuids) @staticmethod def json_schema(): @@ -97,25 +82,7 @@ def sql_json_schema(): ) ] ), - JsonSchemaRunAttributesGroup.sql( - attributes=[ - JsonSchemaRunAttribute.query_string(), - JsonSchemaRunAttributesGroup.query_processor( - attributes=[ - JsonSchemaRunAttribute.processor_type_name( - group_type_name='query_processor' - ) - ] - ), - JsonSchemaRunAttributesGroup.result_processor( - attributes=[ - JsonSchemaRunAttribute.processor_type_name( - group_type_name='result_processor' - ) - ] - ) - ] - ) + Sql.json_schema() ], attributes_groups_additional_contains=JsonSchemaAttributesGroup.json_schema( key='sql', diff --git a/dq0/sdk/data/metadata/specification/run/v1/sql.py b/dq0/sdk/data/metadata/specification/run/v1/sql.py new file mode 100644 index 00000000..e6026f47 --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/sql.py @@ -0,0 +1,49 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup +from dq0.sdk.data.metadata.specification.run.v1.query_processor import QueryProcessor +from dq0.sdk.data.metadata.specification.run.v1.query_processor_dummy import QueryProcessorDummy +from dq0.sdk.data.metadata.specification.run.v1.query_processor_opendp import QueryProcessorOpenDP +from dq0.sdk.data.metadata.specification.run.v1.result_processor import ResultProcessor +from dq0.sdk.data.metadata.specification.run.v1.result_processor_dummy import ResultProcessorDummy +from dq0.sdk.data.metadata.specification.run.v1.result_processor_opendp import ResultProcessorOpenDP +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class Sql: + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'sql': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }) + Sql.verify_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + analyst_attribute = DefaultPermissions.analyst_attribute(role_uuids=role_uuids) + Attribute.check_list(attribute_list=attributes, check_data={ + 'query_string': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'query_processor': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), + 'result_processor': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), + }, required_keys={'query_string'}) + query_processor_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'query_processor'] \ + if attributes is not None else [] + if 0 < len(query_processor_attributes): + QueryProcessor.verify(attribute=query_processor_attributes[0], role_uuids=role_uuids) + result_processor_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'result_processor'] \ + if attributes is not None else [] + if 0 < len(result_processor_attributes): + ResultProcessor.verify(attribute=result_processor_attributes[0], role_uuids=role_uuids) + + @staticmethod + def json_schema(): + return JsonSchemaRunAttributesGroup.sql( + attributes=[ + JsonSchemaRunAttribute.query_string(), + QueryProcessorDummy.json_schema(), + QueryProcessorOpenDP.json_schema(), + ResultProcessorDummy.json_schema(), + ResultProcessorOpenDP.json_schema() + ] + ) From 2d09455213fa6424bfe7d2de730bf643f390afae Mon Sep 17 00:00:00 2001 From: Reinhard Munz Date: Fri, 8 Apr 2022 11:41:08 +0200 Subject: [PATCH 5/8] changed sqlite connector to use new metadata --- dq0/sdk/data/sql/sqlite.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dq0/sdk/data/sql/sqlite.py b/dq0/sdk/data/sql/sqlite.py index 54ba4c2e..0fe6aefe 100644 --- a/dq0/sdk/data/sql/sqlite.py +++ b/dq0/sdk/data/sql/sqlite.py @@ -27,7 +27,12 @@ class SQLite(SQL): connection_string (:obj:`str`): The sqlite connection string. """ - def __init__(self, connection_string): + def __init__(self, meta_database): + meta_connector = meta_database.connector + if meta_connector.type_name != 'sqlite': + raise Exception(f"type_name {meta_connector.type_name} does not match sqlite") + uri = meta_connector.uri if isinstance(meta_connector.uri, str) else '' + connection_string = f"sqlite:///{uri}" super().__init__(connection_string) self.type = 'sqlite' self.engine = sqlalchemy.create_engine(connection_string) From 00540cb5c30b88141da21cf0e1ef7aee9ae9a38e Mon Sep 17 00:00:00 2001 From: Reinhard Munz Date: Tue, 12 Apr 2022 03:17:55 +0200 Subject: [PATCH 6/8] extended metadata for additional run data for instance logging --- .../run/v1/run/attributes_run_logging.py | 91 +++++++++++++++++++ .../data/metadata/interface/run/v1/run/run.py | 16 ++++ .../json_schema/run/attributes_group.py | 62 +++++++++++++ .../data/metadata/specification/run/v1/run.py | 12 +++ 4 files changed, 181 insertions(+) create mode 100644 dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_logging.py diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_logging.py b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_logging.py new file mode 100644 index 00000000..1c38399c --- /dev/null +++ b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_logging.py @@ -0,0 +1,91 @@ +from dq0.sdk.data.metadata.interface.attributes_group import AttributesGroup +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class AttributesRunLogging(AttributesGroup): + def __init__(self, run, attribute_list=None): + super().__init__(key='logging', + permissions=DefaultPermissions.analyst_attribute(role_uuids=run.get_role_uuids()), + entity=run, + attribute_list=attribute_list) + + # log_key_string + @property + def log_key_string(self): + return self.get_attribute_value(key='log_key_string') + + @log_key_string.setter + def log_key_string(self, new_log_key_string): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='log_key_string', + value=new_log_key_string, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + + @log_key_string.deleter + def log_key_string(self): + self.delete_attribute(key='log_key_string') + + # tracker_output_path + @property + def tracker_output_path(self): + return self.get_attribute_value(key='tracker_output_path') + + @tracker_output_path.setter + def tracker_output_path(self, new_tracker_output_path): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='tracker_output_path', + value=new_tracker_output_path, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + + @tracker_output_path.deleter + def tracker_output_path(self): + self.delete_attribute(key='tracker_output_path') + + # tracker_type + @property + def tracker_type(self): + return self.get_attribute_value(key='tracker_type') + + @tracker_type.setter + def tracker_type(self, new_tracker_type): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='tracker_type', + value=new_tracker_type, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + + @tracker_type.deleter + def tracker_type(self): + self.delete_attribute(key='tracker_type') + + # tracker_group_uuid + @property + def tracker_group_uuid(self): + return self.get_attribute_value(key='tracker_group_uuid') + + @tracker_group_uuid.setter + def tracker_group_uuid(self, new_tracker_group_uuid): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='tracker_group_uuid', + value=new_tracker_group_uuid, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + + @tracker_group_uuid.deleter + def tracker_group_uuid(self): + self.delete_attribute(key='tracker_group_uuid') + + # tracker_run_uuid + @property + def tracker_run_uuid(self): + return self.get_attribute_value(key='tracker_run_uuid') + + @tracker_run_uuid.setter + def tracker_run_uuid(self, new_tracker_run_uuid): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_STRING, + key='tracker_run_uuid', + value=new_tracker_run_uuid, + permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids())) + + @tracker_run_uuid.deleter + def tracker_run_uuid(self): + self.delete_attribute(key='tracker_run_uuid') diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/run.py b/dq0/sdk/data/metadata/interface/run/v1/run/run.py index 58aa82ee..fc2e27f4 100644 --- a/dq0/sdk/data/metadata/interface/run/v1/run/run.py +++ b/dq0/sdk/data/metadata/interface/run/v1/run/run.py @@ -1,5 +1,6 @@ from dq0.sdk.data.metadata.interface.entity import Entity from dq0.sdk.data.metadata.interface.run.v1.run.attributes_run_data import AttributesRunData +from dq0.sdk.data.metadata.interface.run.v1.run.attributes_run_logging import AttributesRunLogging from dq0.sdk.data.metadata.interface.run.v1.run.attributes_run_sql import AttributesRunSQL from dq0.sdk.data.metadata.structure.attribute.attribute_list import AttributeList from dq0.sdk.data.metadata.structure.node.node import Node @@ -28,6 +29,8 @@ def get_type_name(self): def create_attributes_group(self, key, attribute_list=None): if key == 'data': return AttributesRunData(run=self, attribute_list=attribute_list) + elif key == 'logging': + return AttributesRunLogging(run=self, attribute_list=attribute_list) elif key == 'sql': if self.get_type_name() is None: raise Exception("cannot access sql without type_name being set") @@ -50,6 +53,19 @@ def data(self, _): def data(self): super().get_attribute_group(key='data').delete() + # logging + @property + def logging(self): + return super().get_attribute_group(key='logging') + + @logging.setter + def logging(self, _): + raise Exception("logging attribute group may not be set") + + @logging.deleter + def logging(self): + super().get_attribute_group(key='logging').delete() + # sql @property def sql(self): diff --git a/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py b/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py index c55c234e..ed6a7e6c 100644 --- a/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py +++ b/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py @@ -28,6 +28,68 @@ def data(attributes): contains=contains, attributes=attributes) + @staticmethod + def logging(): + contains = [ + JsonSchemaAttribute.json_schema( + key='log_key_string', + attribute_name='log key string', + description="This item ensures that the 'log_key_string' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='tracker_output_path', + attribute_name='tracker output path', + description="This item ensures that the 'tracker_output_path' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='tracker_type', + attribute_name='tracker type', + description="This item ensures that the 'tracker_type' attribute is present.", + type_name=AttributeType.TYPE_NAME_STRING + ) + ] + attributes = [ + JsonSchemaAttribute.json_schema( + key='log_key_string', + attribute_name='log key string', + description="The 'log_key_string' attribute is mandatory.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='tracker_output_path', + attribute_name='tracker output path', + description="The 'tracker_output_path' attribute is mandatory.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='tracker_type', + attribute_name='tracker type', + description="The 'tracker_type' attribute is mandatory.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='tracker_group_uuid', + attribute_name='tracker group uuid', + description="The 'tracker_group_uuid' attribute.", + type_name=AttributeType.TYPE_NAME_STRING + ), + JsonSchemaAttribute.json_schema( + key='tracker_run_uuid', + attribute_name='tracker run uuid', + description="The 'tracker_run_uuid' attribute.", + type_name=AttributeType.TYPE_NAME_STRING + ), + ] + return JsonSchemaAttributesGroup.json_schema( + key='logging', + group_name='logging', + description="The 'logging' attributes group.", + additional_description="Requires 'log_key_string', 'tracker_output_path', and 'tracker_type' attribute.", + contains=contains, + attributes=attributes) + @staticmethod def sql(attributes): contains = JsonSchemaAttribute.json_schema( diff --git a/dq0/sdk/data/metadata/specification/run/v1/run.py b/dq0/sdk/data/metadata/specification/run/v1/run.py index 41de489a..16d355c3 100644 --- a/dq0/sdk/data/metadata/specification/run/v1/run.py +++ b/dq0/sdk/data/metadata/specification/run/v1/run.py @@ -39,6 +39,7 @@ def verify_attributes(attributes, role_uuids=None): analyst_attribute = DefaultPermissions.analyst_attribute(role_uuids=role_uuids) Attribute.check_list(attribute_list=attributes, check_data={ 'data': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), + 'logging': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), 'sql': ([AttributeType.TYPE_NAME_LIST], analyst_attribute), }, required_keys={'data'}) data_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'data'] \ @@ -49,6 +50,16 @@ def verify_attributes(attributes, role_uuids=None): 'name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), 'type_name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), }, required_keys={'name', 'type_name'}) + logging_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'logging'] \ + if attributes is not None else [] + if 0 < len(logging_attributes): + Attribute.check_list(attribute_list=logging_attributes[0].get_value(), check_data={ + 'log_key_string': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'tracker_output_path': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'tracker_type': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'tracker_group_uuid': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'tracker_run_uuid': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + }, required_keys={'log_key_string', 'tracker_output_path', 'tracker_type'}) sql_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'sql'] \ if attributes is not None else [] if 0 < len(sql_attributes): @@ -82,6 +93,7 @@ def sql_json_schema(): ) ] ), + JsonSchemaRunAttributesGroup.logging(), Sql.json_schema() ], attributes_groups_additional_contains=JsonSchemaAttributesGroup.json_schema( From 6c38760b90204b2f74ed9a8009941e49d6244ad8 Mon Sep 17 00:00:00 2001 From: Reinhard Munz Date: Wed, 13 Apr 2022 04:51:05 +0200 Subject: [PATCH 7/8] fixed metadata bugs regarding new run root node and root node creation/deletion --- .../metadata/interface/attributes_group.py | 4 +- dq0/sdk/data/metadata/interface/interface.py | 88 +++++++++++-------- .../json_schema/run/attributes_group.py | 6 +- 3 files changed, 54 insertions(+), 44 deletions(-) diff --git a/dq0/sdk/data/metadata/interface/attributes_group.py b/dq0/sdk/data/metadata/interface/attributes_group.py index 36ef9732..39c74f7e 100644 --- a/dq0/sdk/data/metadata/interface/attributes_group.py +++ b/dq0/sdk/data/metadata/interface/attributes_group.py @@ -90,8 +90,8 @@ def delete_attribute(self, key): self._attribute_list = None def add_attribute(self, attribute): - attribute = self.get_attribute(key=attribute.get_key()) - if attribute is not None: + internal_attribute = self.get_attribute(key=attribute.get_key()) + if internal_attribute is not None: raise Exception(f"attribute {attribute.get_key()} is already set and cannot be modified") if self._attribute_list is None: attribute_list = AttributeList(key=self.get_key(), value=[], permissions=self._permissions) diff --git a/dq0/sdk/data/metadata/interface/interface.py b/dq0/sdk/data/metadata/interface/interface.py index f67b22f0..0a6c063d 100644 --- a/dq0/sdk/data/metadata/interface/interface.py +++ b/dq0/sdk/data/metadata/interface/interface.py @@ -53,18 +53,24 @@ def delete(self): def add_child_node(self, child_node): if not isinstance(child_node, Node): raise Exception(f"child_node is not of type Node, is of type {type(child_node)} instead") - if child_node.get_type_name() != NodeType.TYPE_NAME_DATASET: - raise Exception(f"child_node.get_type_name() mismatch: {child_node.get_type_name()} != {NodeType.TYPE_NAME_DATASET}") - self._metadata.set_node(root_key='dataset', node=child_node) - - def remove_child_node(self, name): - if self._metadata.get_node(root_key='dataset') is None: + child_node_type_name = child_node.get_type_name() + if child_node_type_name == NodeType.TYPE_NAME_DATASET: + self._metadata.set_node(root_key=NodeType.TYPE_NAME_DATASET, node=child_node) + elif child_node_type_name == NodeType.TYPE_NAME_RUN: + self._metadata.set_node(root_key=NodeType.TYPE_NAME_RUN, node=child_node) + else: + raise Exception(f"unknown child_node_type_name={child_node_type_name}, must be one of [{NodeType.TYPE_NAME_DATASET}, {NodeType.TYPE_NAME_RUN}]") + + def remove_child_node(self, type_name, name): + if self._metadata.get_node(root_key=NodeType.TYPE_NAME_DATASET) is None and self._metadata.get_node(root_key=NodeType.TYPE_NAME_RUN) is None: return - self._metadata.delete_node(root_key='dataset') - if self._entities['dataset'] is not None: - if name != self._entities['dataset'].get_name(): - raise Exception(f"name mismatch: {name} != {self._entities['dataset'].get_name()}") - self._entities['dataset'].wipe() + if type_name not in [NodeType.TYPE_NAME_DATASET, NodeType.TYPE_NAME_RUN]: + raise ValueError(f"unknown type_name={type_name}, must be one of [{NodeType.TYPE_NAME_DATASET}, {NodeType.TYPE_NAME_RUN}]") + self._metadata.delete_node(root_key=type_name) + if self._entities[type_name] is not None: + if name != self._entities[type_name].get_name(): + raise Exception(f"name mismatch: {name} != {self._entities[type_name].get_name()}") + self._entities[type_name].wipe() def set_name(self, old_name, new_name): pass @@ -75,42 +81,46 @@ def set_child_name(self, old_name, new_name): def apply_defaults_and_verify(self, specifications=None): return Interface(metadata=self._metadata.apply_defaults_and_verify(specifications=specifications), role_uuids=self.get_role_uuids()) - def dataset(self, name=None): - node = self._metadata.get_node(root_key='dataset') + def dataset(self, name=None, specification=None): + node = self._metadata.get_node(root_key=NodeType.TYPE_NAME_DATASET) if name is None: if node is None: raise Exception("get without name may only work for existing dataset node") name = Entity.name_of(node=node) - if 'dataset' not in self._entities: - specification = self._metadata.get_specification(root_key='dataset') - if isinstance(specification, DatasetSpecificationV1): - self._entities['dataset'] = Dataset(name=name, parent=self, - permissions=DefaultPermissions.shared_node(role_uuids=self.get_role_uuids()), - data_permissions=DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()), - name_permissions=DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()), - role_uuids=self.get_role_uuids(), node=node) + if NodeType.TYPE_NAME_DATASET not in self._entities: + internal_specification = self._metadata.get_specification(root_key=NodeType.TYPE_NAME_DATASET) + if internal_specification is None and specification is not None: + internal_specification = specification + if isinstance(internal_specification, DatasetSpecificationV1): + self._entities[NodeType.TYPE_NAME_DATASET] = Dataset(name=name, parent=self, + permissions=DefaultPermissions.shared_node(role_uuids=self.get_role_uuids()), + data_permissions=DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()), + name_permissions=DefaultPermissions.shared_attribute(role_uuids=self.get_role_uuids()), + role_uuids=self.get_role_uuids(), node=node) else: - raise Exception("no interface for specified version available") - if name != self._entities['dataset'].get_name(): - raise Exception(f"name mismatch: {name} != {self._entities['dataset'].get_name()}") - return self._entities['dataset'] + raise Exception(f"no interface for specified version={internal_specification} available") + if name != self._entities[NodeType.TYPE_NAME_DATASET].get_name(): + raise Exception(f"name mismatch: {name} != {self._entities[NodeType.TYPE_NAME_DATASET].get_name()}") + return self._entities[NodeType.TYPE_NAME_DATASET] - def run(self, name=None): - node = self._metadata.get_node(root_key='run') + def run(self, name=None, specification=None): + node = self._metadata.get_node(root_key=NodeType.TYPE_NAME_RUN) if name is None: if node is None: raise Exception("get without name may only work for existing run node") name = Entity.name_of(node=node) - if 'run' not in self._entities: - specification = self._metadata.get_specification(root_key='run') - if isinstance(specification, RunSpecificationV1): - self._entities['run'] = Run(name=name, parent=self, - permissions=DefaultPermissions.analyst_node(role_uuids=self.get_role_uuids()), - data_permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids()), - name_permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids()), - role_uuids=self.get_role_uuids(), node=node) + if NodeType.TYPE_NAME_RUN not in self._entities: + internal_specification = self._metadata.get_specification(root_key=NodeType.TYPE_NAME_RUN) + if internal_specification is None and specification is not None: + internal_specification = specification + if isinstance(internal_specification, RunSpecificationV1): + self._entities[NodeType.TYPE_NAME_RUN] = Run(name=name, parent=self, + permissions=DefaultPermissions.analyst_node(role_uuids=self.get_role_uuids()), + data_permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids()), + name_permissions=DefaultPermissions.analyst_attribute(role_uuids=self.get_role_uuids()), + role_uuids=self.get_role_uuids(), node=node) else: - raise Exception("no interface for specified version available") - if name != self._entities['run'].get_name(): - raise Exception(f"name mismatch: {name} != {self._entities['run'].get_name()}") - return self._entities['run'] + raise Exception(f"no interface for specified version={internal_specification} available") + if name != self._entities[NodeType.TYPE_NAME_RUN].get_name(): + raise Exception(f"name mismatch: {name} != {self._entities[NodeType.TYPE_NAME_RUN].get_name()}") + return self._entities[NodeType.TYPE_NAME_RUN] diff --git a/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py b/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py index ed6a7e6c..ea5bafc1 100644 --- a/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py +++ b/dq0/sdk/data/metadata/specification/json_schema/run/attributes_group.py @@ -14,9 +14,9 @@ def data(attributes): type_name=AttributeType.TYPE_NAME_STRING ), JsonSchemaAttribute.json_schema( - key='run_type_name', - attribute_name='run type name', - description="This item ensures that the 'run_type_name' attribute is present.", + key='type_name', + attribute_name='type name', + description="This item ensures that the 'type_name' attribute is present.", type_name=AttributeType.TYPE_NAME_STRING ) ] From c9ad33d85f25639df6daf3f8fb70ea6ca50dfea8 Mon Sep 17 00:00:00 2001 From: Reinhard Munz Date: Fri, 13 May 2022 10:07:35 +0200 Subject: [PATCH 8/8] at the point where python does not work any longer --- ...ibutes_column_private_sql_and_synthesis.py | 16 +++++++ .../attributes_run_sql_result_processor.py | 6 +-- .../specification/dataset/v1/column.py | 1 + .../dataset/v1/column_boolean.py | 16 +++++++ .../dataset/v1/column_datetime.py | 7 +++ .../specification/dataset/v1/column_float.py | 7 +++ .../specification/dataset/v1/column_int.py | 7 +++ .../specification/dataset/v1/column_string.py | 7 +++ .../specification/run/v1/query_processor.py | 3 ++ .../run/v1/query_processor_classic_dp.py | 34 +++++++++++++++ .../specification/run/v1/result_processor.py | 3 ++ .../run/v1/result_processor_classic_dp.py | 43 +++++++++++++++++++ .../data/metadata/specification/run/v1/sql.py | 4 ++ 13 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 dq0/sdk/data/metadata/specification/run/v1/query_processor_classic_dp.py create mode 100644 dq0/sdk/data/metadata/specification/run/v1/result_processor_classic_dp.py diff --git a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql_and_synthesis.py b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql_and_synthesis.py index b2d5d8f5..f6472d50 100644 --- a/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql_and_synthesis.py +++ b/dq0/sdk/data/metadata/interface/dataset/v1/column/attributes_column_private_sql_and_synthesis.py @@ -85,3 +85,19 @@ def upper(self, new_upper): @upper.deleter def upper(self): self.delete_attribute(key='upper') + + # carries_null + @property + def carries_null(self): + return self.get_attribute_value(key='carries_null') + + @carries_null.setter + def carries_null(self, new_carries_null): + self.set_attribute_value(type_name=AttributeType.TYPE_NAME_BOOLEAN, + key='carries_null', + value=new_carries_null, + permissions=DefaultPermissions.owner_attribute(role_uuids=self.get_role_uuids())) + + @carries_null.deleter + def carries_null(self): + self.delete_attribute(key='carries_null') diff --git a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py index c25fd1b8..e849cfb2 100644 --- a/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py +++ b/dq0/sdk/data/metadata/interface/run/v1/run/attributes_run_sql_result_processor.py @@ -40,12 +40,12 @@ def type_name(self): # epsilon @property def epsilon(self): - self.type_names_check(type_names={'opendp'}) + self.type_names_check(type_names={'opendp', 'classic_dp'}) return self.get_attribute_value(key='epsilon') @epsilon.setter def epsilon(self, new_epsilon): - self.type_names_check(type_names={'opendp'}) + self.type_names_check(type_names={'opendp', 'classic_dp'}) self.set_attribute_value(type_name=AttributeType.TYPE_NAME_FLOAT, key='epsilon', value=new_epsilon, @@ -53,7 +53,7 @@ def epsilon(self, new_epsilon): @epsilon.deleter def epsilon(self): - self.type_names_check(type_names={'opendp'}) + self.type_names_check(type_names={'opendp', 'classic_dp'}) self.delete_attribute(key='epsilon') # delta diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column.py b/dq0/sdk/data/metadata/specification/dataset/v1/column.py index 8ef35f89..e9d2ad37 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column.py @@ -94,6 +94,7 @@ def verify_private_sql_and_synthesis_attributes(attributes, role_uuids=None): Attribute.check_list(attribute_list=private_sql_and_synthesis_attributes[0].get_value(), check_data={ 'bounded': ([AttributeType.TYPE_NAME_BOOLEAN], owner_attribute), 'cardinality': ([AttributeType.TYPE_NAME_INT], owner_attribute), + 'carries_null': ([AttributeType.TYPE_NAME_BOOLEAN], owner_attribute), 'lower': ([AttributeType.TYPE_NAME_DATETIME, AttributeType.TYPE_NAME_FLOAT, AttributeType.TYPE_NAME_INT], owner_attribute), 'upper': ([AttributeType.TYPE_NAME_DATETIME, AttributeType.TYPE_NAME_FLOAT, AttributeType.TYPE_NAME_INT], owner_attribute), }) diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_boolean.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_boolean.py index f9d0b605..1bbbb4f7 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_boolean.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_boolean.py @@ -34,6 +34,12 @@ def verify_attributes(attributes, role_uuids=None): Attribute.check_list(attribute_list=private_sql_attributes[0].get_value(), check_data={ 'private_id': ([AttributeType.TYPE_NAME_BOOLEAN], owner_attribute), }) + private_sql_and_synthesis_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'private_sql_and_synthesis'] \ + if attributes is not None else [] + if 0 < len(private_sql_and_synthesis_attributes): + Attribute.check_list(attribute_list=private_sql_and_synthesis_attributes[0].get_value(), check_data={ + 'carries_null': ([AttributeType.TYPE_NAME_BOOLEAN], owner_attribute), + }) private_synthesis_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'private_synthesis'] \ if attributes is not None else [] if 0 < len(private_synthesis_attributes): @@ -64,6 +70,16 @@ def json_schema(): ) ] ), + JsonSchemaAttributesGroup.private_sql_and_synthesis( + attributes=[ + JsonSchemaAttribute.json_schema( + key='carries_null', + attribute_name="carries null", + description=f"The 'carries null' attribute. Specifies whether this '{NodeType.TYPE_NAME_COLUMN}' may contain null values.", + type_name=AttributeType.TYPE_NAME_BOOLEAN + ) + ] + ), JsonSchemaAttributesGroup.private_synthesis( attributes=[ JsonSchemaAttribute.json_schema( diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_datetime.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_datetime.py index adada7a5..98e1b6da 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_datetime.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_datetime.py @@ -55,6 +55,7 @@ def verify_attributes(attributes, role_uuids=None): 'cardinality': ([AttributeType.TYPE_NAME_INT], owner_attribute), 'lower': ([AttributeType.TYPE_NAME_DATETIME], owner_attribute), 'upper': ([AttributeType.TYPE_NAME_DATETIME], owner_attribute), + 'carries_null': ([AttributeType.TYPE_NAME_BOOLEAN], owner_attribute), }) private_synthesis_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'private_synthesis'] \ if attributes is not None else [] @@ -147,6 +148,12 @@ def json_schema(): attribute_name='upper', description=f"The 'upper' attribute. Specifies the upper bound for the '{NodeType.TYPE_NAME_COLUMN}'.", type_name=AttributeType.TYPE_NAME_DATETIME + ), + JsonSchemaAttribute.json_schema( + key='carries_null', + attribute_name="carries null", + description=f"The 'carries null' attribute. Specifies whether this '{NodeType.TYPE_NAME_COLUMN}' may contain null values.", + type_name=AttributeType.TYPE_NAME_BOOLEAN ) ] ), diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_float.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_float.py index 0eceaa94..cfdd7a1d 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_float.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_float.py @@ -55,6 +55,7 @@ def verify_attributes(attributes, role_uuids=None): 'cardinality': ([AttributeType.TYPE_NAME_INT], owner_attribute), 'lower': ([AttributeType.TYPE_NAME_FLOAT], owner_attribute), 'upper': ([AttributeType.TYPE_NAME_FLOAT], owner_attribute), + 'carries_null': ([AttributeType.TYPE_NAME_BOOLEAN], owner_attribute), }) private_synthesis_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'private_synthesis'] \ if attributes is not None else [] @@ -149,6 +150,12 @@ def json_schema(): attribute_name='upper', description=f"The 'upper' attribute. Specifies the upper bound for the '{NodeType.TYPE_NAME_COLUMN}'.", type_name=AttributeType.TYPE_NAME_FLOAT + ), + JsonSchemaAttribute.json_schema( + key='carries_null', + attribute_name="carries null", + description=f"The 'carries null' attribute. Specifies whether this '{NodeType.TYPE_NAME_COLUMN}' may contain null values.", + type_name=AttributeType.TYPE_NAME_BOOLEAN ) ] ), diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_int.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_int.py index 96d37889..81e02a7b 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_int.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_int.py @@ -56,6 +56,7 @@ def verify_attributes(attributes, role_uuids=None): 'cardinality': ([AttributeType.TYPE_NAME_INT], owner_attribute), 'lower': ([AttributeType.TYPE_NAME_INT], owner_attribute), 'upper': ([AttributeType.TYPE_NAME_INT], owner_attribute), + 'carries_null': ([AttributeType.TYPE_NAME_BOOLEAN], owner_attribute), }) private_synthesis_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'private_synthesis'] \ if attributes is not None else [] @@ -149,6 +150,12 @@ def json_schema(): attribute_name='upper', description=f"The 'upper' attribute. Specifies the upper bound for the '{NodeType.TYPE_NAME_COLUMN}'.", type_name=AttributeType.TYPE_NAME_INT + ), + JsonSchemaAttribute.json_schema( + key='carries_null', + attribute_name="carries null", + description=f"The 'carries null' attribute. Specifies whether this '{NodeType.TYPE_NAME_COLUMN}' may contain null values.", + type_name=AttributeType.TYPE_NAME_BOOLEAN ) ] ), diff --git a/dq0/sdk/data/metadata/specification/dataset/v1/column_string.py b/dq0/sdk/data/metadata/specification/dataset/v1/column_string.py index 1921e486..b60d0d54 100644 --- a/dq0/sdk/data/metadata/specification/dataset/v1/column_string.py +++ b/dq0/sdk/data/metadata/specification/dataset/v1/column_string.py @@ -49,6 +49,7 @@ def verify_attributes(attributes, role_uuids=None): if 0 < len(private_sql_and_synthesis_attributes): Attribute.check_list(attribute_list=private_sql_and_synthesis_attributes[0].get_value(), check_data={ 'cardinality': ([AttributeType.TYPE_NAME_INT], owner_attribute), + 'carries_null': ([AttributeType.TYPE_NAME_BOOLEAN], owner_attribute), }) private_synthesis_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'private_synthesis'] \ if attributes is not None else [] @@ -101,6 +102,12 @@ def json_schema(): attribute_name='cardinality', description=f"The 'cardinality' attribute. Specifies the cardinality for the '{NodeType.TYPE_NAME_COLUMN}'.", type_name=AttributeType.TYPE_NAME_INT + ), + JsonSchemaAttribute.json_schema( + key='carries_null', + attribute_name="carries null", + description=f"The 'carries null' attribute. Specifies whether this '{NodeType.TYPE_NAME_COLUMN}' may contain null values.", + type_name=AttributeType.TYPE_NAME_BOOLEAN ) ] ), diff --git a/dq0/sdk/data/metadata/specification/run/v1/query_processor.py b/dq0/sdk/data/metadata/specification/run/v1/query_processor.py index ed86c98c..fc39d55b 100644 --- a/dq0/sdk/data/metadata/specification/run/v1/query_processor.py +++ b/dq0/sdk/data/metadata/specification/run/v1/query_processor.py @@ -1,4 +1,5 @@ from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.run.v1.query_processor_classic_dp import QueryProcessorClassicDP from dq0.sdk.data.metadata.specification.run.v1.query_processor_dummy import QueryProcessorDummy from dq0.sdk.data.metadata.specification.run.v1.query_processor_opendp import QueryProcessorOpenDP from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute @@ -19,5 +20,7 @@ def verify(attribute, role_uuids=None): return QueryProcessorDummy.verify(attribute=attribute, role_uuids=role_uuids) elif attribute_type_name.get_value() == 'opendp': return QueryProcessorOpenDP.verify(attribute=attribute, role_uuids=role_uuids) + elif attribute_type_name.get_value() == 'classic_dp': + return QueryProcessorClassicDP.verify(attribute=attribute, role_uuids=role_uuids) else: raise Exception(f"unknown query processor type_name {attribute_type_name.get_value()}") diff --git a/dq0/sdk/data/metadata/specification/run/v1/query_processor_classic_dp.py b/dq0/sdk/data/metadata/specification/run/v1/query_processor_classic_dp.py new file mode 100644 index 00000000..0ff6b1af --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/query_processor_classic_dp.py @@ -0,0 +1,34 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class QueryProcessorClassicDP: + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'query_processor': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }) + QueryProcessorClassicDP.verify_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + Attribute.check_list(attribute_list=attributes, check_data={ + 'type_name': ([AttributeType.TYPE_NAME_STRING], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }, required_keys={'type_name'}) + type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] + if type_name_attributes[0].get_value() != 'classic_dp': + raise Exception(f"query processor type_name value {type_name_attributes[0].get_value()} does not match 'classic_dp'") + + @staticmethod + def json_schema(): + return JsonSchemaRunAttributesGroup.query_processor( + attributes=[ + JsonSchemaRunAttribute.processor_type_name( + group_name='query_processor', + additional_value="\"const\": \"classic_dp\"" + ) + ] + ) diff --git a/dq0/sdk/data/metadata/specification/run/v1/result_processor.py b/dq0/sdk/data/metadata/specification/run/v1/result_processor.py index bb3a7a8a..2005fb0d 100644 --- a/dq0/sdk/data/metadata/specification/run/v1/result_processor.py +++ b/dq0/sdk/data/metadata/specification/run/v1/result_processor.py @@ -1,4 +1,5 @@ from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.run.v1.result_processor_classic_dp import ResultProcessorClassicDP from dq0.sdk.data.metadata.specification.run.v1.result_processor_dummy import ResultProcessorDummy from dq0.sdk.data.metadata.specification.run.v1.result_processor_opendp import ResultProcessorOpenDP from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute @@ -19,5 +20,7 @@ def verify(attribute, role_uuids=None): return ResultProcessorDummy.verify(attribute=attribute, role_uuids=role_uuids) elif attribute_type_name.get_value() == 'opendp': return ResultProcessorOpenDP.verify(attribute=attribute, role_uuids=role_uuids) + elif attribute_type_name.get_value() == 'classic_dp': + return ResultProcessorClassicDP.verify(attribute=attribute, role_uuids=role_uuids) else: raise Exception(f"unknown result processor type_name {attribute_type_name.get_value()}") diff --git a/dq0/sdk/data/metadata/specification/run/v1/result_processor_classic_dp.py b/dq0/sdk/data/metadata/specification/run/v1/result_processor_classic_dp.py new file mode 100644 index 00000000..565c432f --- /dev/null +++ b/dq0/sdk/data/metadata/specification/run/v1/result_processor_classic_dp.py @@ -0,0 +1,43 @@ +from dq0.sdk.data.metadata.specification.default_permissions import DefaultPermissions +from dq0.sdk.data.metadata.specification.json_schema.attribute import Attribute as JsonSchemaAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute +from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup +from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute +from dq0.sdk.data.metadata.structure.attribute.attribute_type import AttributeType + + +class ResultProcessorClassicDP: + @staticmethod + def verify(attribute, role_uuids=None): + Attribute.check(attribute=attribute, check_data={ + 'result_processor': ([AttributeType.TYPE_NAME_LIST], DefaultPermissions.analyst_attribute(role_uuids=role_uuids)), + }) + ResultProcessorClassicDP.verify_attributes(attributes=attribute.get_value(), role_uuids=role_uuids) + + @staticmethod + def verify_attributes(attributes, role_uuids=None): + analyst_attribute = DefaultPermissions.analyst_attribute(role_uuids=role_uuids) + Attribute.check_list(attribute_list=attributes, check_data={ + 'type_name': ([AttributeType.TYPE_NAME_STRING], analyst_attribute), + 'epsilon': ([AttributeType.TYPE_NAME_FLOAT], analyst_attribute), + }, required_keys={'type_name'}) + type_name_attributes = [tmp_attribute for tmp_attribute in attributes if tmp_attribute.get_key() == 'type_name'] if attributes is not None else [] + if type_name_attributes[0].get_value() != 'classic_dp': + raise Exception(f"result processor type_name value {type_name_attributes[0].get_value()} does not match 'classic_dp'") + + @staticmethod + def json_schema(): + return JsonSchemaRunAttributesGroup.result_processor( + attributes=[ + JsonSchemaRunAttribute.processor_type_name( + group_name='result_processor', + additional_value="\"const\": \"classic_dp\"" + ), + JsonSchemaAttribute.json_schema( + key='epsilon', + attribute_name='epsilon', + description="The 'epsilon' attribute.", + type_name=AttributeType.TYPE_NAME_FLOAT + ) + ] + ) diff --git a/dq0/sdk/data/metadata/specification/run/v1/sql.py b/dq0/sdk/data/metadata/specification/run/v1/sql.py index e6026f47..4f167b90 100644 --- a/dq0/sdk/data/metadata/specification/run/v1/sql.py +++ b/dq0/sdk/data/metadata/specification/run/v1/sql.py @@ -2,9 +2,11 @@ from dq0.sdk.data.metadata.specification.json_schema.run.attribute import Attribute as JsonSchemaRunAttribute from dq0.sdk.data.metadata.specification.json_schema.run.attributes_group import AttributesGroup as JsonSchemaRunAttributesGroup from dq0.sdk.data.metadata.specification.run.v1.query_processor import QueryProcessor +from dq0.sdk.data.metadata.specification.run.v1.query_processor_classic_dp import QueryProcessorClassicDP from dq0.sdk.data.metadata.specification.run.v1.query_processor_dummy import QueryProcessorDummy from dq0.sdk.data.metadata.specification.run.v1.query_processor_opendp import QueryProcessorOpenDP from dq0.sdk.data.metadata.specification.run.v1.result_processor import ResultProcessor +from dq0.sdk.data.metadata.specification.run.v1.result_processor_classic_dp import ResultProcessorClassicDP from dq0.sdk.data.metadata.specification.run.v1.result_processor_dummy import ResultProcessorDummy from dq0.sdk.data.metadata.specification.run.v1.result_processor_opendp import ResultProcessorOpenDP from dq0.sdk.data.metadata.structure.attribute.attribute import Attribute @@ -41,8 +43,10 @@ def json_schema(): return JsonSchemaRunAttributesGroup.sql( attributes=[ JsonSchemaRunAttribute.query_string(), + QueryProcessorClassicDP.json_schema(), QueryProcessorDummy.json_schema(), QueryProcessorOpenDP.json_schema(), + ResultProcessorClassicDP.json_schema(), ResultProcessorDummy.json_schema(), ResultProcessorOpenDP.json_schema() ]