DQ0-1160: dq0-sql: refactoring/completion

dq0/examples/census/_data/adult_with_rand_names_datatopredict_full.yaml

@@ -1,5 +1,6 @@
 meta_dataset:
   format: 'full'
+  specification: 'dataset_v1'
   node:
     type_name: 'dataset'
     attributes:
@@ -584,4 +585,3 @@ meta_dataset:
               - type_name: 'boolean'
                 key: 'synthesizable'
                 value: true
-  specification: 'dataset_v1'

dq0/examples/census/_data/adult_with_rand_names_datatopredict_simple.yaml

@@ -1,5 +1,6 @@
 meta_dataset:
   format: simple
+  specification: 'dataset_v1'
   node:
     dataset:
       attributes:
@@ -230,4 +231,3 @@ meta_dataset:
                           'cardinality': 9
                         'private_synthesis':
                           'synthesizable': true
-  specification: 'dataset_v1'

dq0/examples/census/_data/adult_with_rand_names_minimal_full.yaml

@@ -1,5 +1,6 @@
 meta_dataset:
   format: 'full'
+  specification: 'dataset_v1'
   node:
     type_name: 'dataset'
     attributes:
@@ -29,7 +30,7 @@ meta_dataset:
           value: 'csv'
         - type_name: 'string'
           key: 'uri'
-          value: "/Users/wolfganggross/code/dq0-sdk/dq0/examples/census/_data/adult_with_rand_names.csv"
+          value: "../dq0-sdk/dq0/examples/census/_data/adult_with_rand_names.csv"
       - type_name: 'list'
         key: 'data'
         value:
@@ -54,4 +55,3 @@ meta_dataset:
                     - type_name: 'string'
                       key: 'name'
                       value: 'ACI Table'
-  specification: 'dataset_v1'

dq0/examples/census/_data/adult_with_rand_names_minimal_simple.yaml

@@ -1,5 +1,6 @@
 meta_dataset:
   format: simple
+  specification: 'dataset_v1'
   node:
     dataset:
       attributes:
@@ -13,7 +14,6 @@ meta_dataset:
           attributes:
             'connector':
               'type_name': 'csv'
-              'uri': "/Users/wolfganggross/code/dq0-sdk/dq0/examples/census/_data/adult_with_rand_names.csv"
+              'uri': "../dq0-sdk/dq0/examples/census/_data/adult_with_rand_names.csv"
             'data':
               'name': "adult database"
-  specification: 'dataset_v1'

dq0/examples/census/_data/adult_with_rand_names_regular_full.yaml

@@ -1,5 +1,6 @@
 meta_dataset:
   format: 'full'
+  specification: 'dataset_v1'
   node:
     type_name: 'dataset'
     attributes:
@@ -328,15 +329,6 @@ meta_dataset:
               - type_name: 'boolean'
                 key: 'is_feature'
                 value: true
-            # - type_name: 'list'
-            #   key: 'private_sql_and_synthesis'
-            #   value:
-            #   - type_name: 'int'
-            #     key: 'lower'
-            #     value: 39824
-            #   - type_name: 'int'
-            #     key: 'upper'
-            #     value: 334679
             - type_name: 'list'
               key: 'private_synthesis'
               value:
@@ -642,4 +634,3 @@ meta_dataset:
               - type_name: 'boolean'
                 key: 'synthesizable'
                 value: true
-  specification: 'dataset_v1'

dq0/examples/census/_data/adult_with_rand_names_regular_simple.yaml

@@ -1,5 +1,6 @@
 meta_dataset:
   format: simple
+  specification: 'dataset_v1'
   node:
     dataset:
       attributes:
@@ -111,9 +112,6 @@ meta_dataset:
                           'name': 'education-num'
                         'machine_learning':
                           'is_feature': true
-                        # 'private_sql_and_synthesis':
-                        #   'lower': 6
-                        #   'upper': 14
                         'private_sql_and_synthesis':
                           'cardinality': 16
                         'private_synthesis':
@@ -136,9 +134,6 @@ meta_dataset:
                           'name': 'fnlwgt'
                         'machine_learning':
                           'is_feature': true
-                        # 'private_sql_and_synthesis':
-                        #   'lower': 39824
-                        #   'upper': 334679
                         'private_sql_and_synthesis':
                           'cardinality': 28521
                         'private_synthesis':
@@ -254,4 +249,3 @@ meta_dataset:
                           'cardinality': 9
                         'private_synthesis':
                           'synthesizable': true
-  specification: 'dataset_v1'

dq0/examples/census/_data/adult_with_rand_names_w_header_full.yaml

@@ -1,5 +1,6 @@
 meta_dataset:
   format: 'full'
+  specification: 'dataset_v1'
   node:
     type_name: 'dataset'
     attributes:
@@ -209,15 +210,6 @@ meta_dataset:
               - type_name: 'boolean'
                 key: 'is_feature'
                 value: true
-            # - type_name: 'list'
-            #   key: 'private_sql_and_synthesis'
-            #   value:
-            #   - type_name: 'int'
-            #     key: 'lower'
-            #     value: 6
-            #   - type_name: 'int'
-            #     key: 'upper'
-            #     value: 14
             - type_name: 'list'
               key: 'private_synthesis'
               value:
@@ -276,15 +268,6 @@ meta_dataset:
               - type_name: 'boolean'
                 key: 'is_feature'
                 value: true
-            # - type_name: 'list'
-            #   key: 'private_sql_and_synthesis'
-            #   value:
-            #   - type_name: 'int'
-            #     key: 'lower'
-            #     value: 45868
-            #   - type_name: 'int'
-            #     key: 'upper'
-            #     value: 349148
             - type_name: 'list'
               key: 'private_synthesis'
               value:
@@ -590,4 +573,3 @@ meta_dataset:
               - type_name: 'boolean'
                 key: 'synthesizable'
                 value: true
-  specification: 'dataset_v1'

dq0/examples/census/_data/adult_with_rand_names_w_header_simple.yaml

@@ -1,5 +1,6 @@
 meta_dataset:
   format: simple
+  specification: 'dataset_v1'
   node:
     dataset:
       attributes:
@@ -85,9 +86,6 @@ meta_dataset:
                           'name': 'education-num'
                         'machine_learning':
                           'is_feature': true
-                        # 'private_sql_and_synthesis':
-                        #   'lower': 6
-                        #   'upper': 14
                         'private_synthesis':
                           'synthesizable': false
                         'private_sql_and_synthesis':
@@ -111,8 +109,6 @@ meta_dataset:
                         'machine_learning':
                           'is_feature': true
                         'private_sql_and_synthesis':
-                          # 'lower': 45868
-                          # 'upper': 349148
                           'cardinality': 28521
                         'private_synthesis':
                           'synthesizable': false
@@ -227,4 +223,3 @@ meta_dataset:
                           'cardinality': 9
                         'private_synthesis':
                           'synthesizable': true
-  specification: 'dataset_v1'

dq0/sdk/connector/connector.py

@@ -0,0 +1,17 @@
+from dq0.sdk.connector.connector_kind import ConnectorKind
+
+
+class Connector:
+    def __init__(self, kind):
+        if not ConnectorKind.is_valid(kind=kind):
+            raise ValueError(f"kind {kind} is not valid")
+        self._kind = kind
+
+    def get_kind(self):
+        return self._kind
+
+    def to_pandas(self):
+        raise NotImplementedError
+
+    def new_query(self, query_string):
+        raise NotImplementedError

dq0/sdk/connector/connector_factory.py

@@ -0,0 +1,20 @@
+from dq0.sdk.connector.kind.csv.csv import CSV
+from dq0.sdk.connector.kind.sql.db.mysql.mysql import MySQL
+from dq0.sdk.connector.kind.sql.db.postgresql.postgresql import PostgreSQL
+from dq0.sdk.connector.kind.sql.db.sqlite.sqlite import SQLite
+
+
+class ConnectorFactory:
+    @staticmethod
+    def connector_from(meta_database):
+        meta_connector = meta_database.connector
+        type_name = meta_connector.type_name
+        if type_name == 'csv':
+            return CSV.csv_from(meta_connector=meta_connector)
+        if type_name == 'mysql':
+            return MySQL.mysql_from(meta_database=meta_database)
+        if type_name == 'postgresql':
+            return PostgreSQL.postgresql_from(meta_database=meta_database)
+        if type_name == 'sqlite':
+            return SQLite.sqlite_from(meta_connector=meta_connector)
+        raise ValueError(f"type_name={type_name} is unknown")

dq0/sdk/connector/connector_kind.py

@@ -0,0 +1,18 @@
+class ConnectorKind:
+    CSV = 'csv'
+    SQL_MYSQL = 'sql_mysql'
+    SQL_POSTGRESQL = 'sql_postgresql'
+    SQL_SQLITE = 'sql_sqlite'
+
+    @staticmethod
+    def is_valid(kind):
+        return \
+            kind == ConnectorKind.CSV or \
+            ConnectorKind.is_valid_sql(kind=kind)
+
+    @staticmethod
+    def is_valid_sql(kind):
+        return \
+            kind == ConnectorKind.SQL_MYSQL or \
+            kind == ConnectorKind.SQL_POSTGRESQL or \
+            kind == ConnectorKind.SQL_SQLITE

dq0/sdk/connector/kind/csv/csv.py

@@ -0,0 +1,70 @@
+from pathlib import Path
+
+from dq0.sdk.connector.connector import Connector
+from dq0.sdk.connector.connector_kind import ConnectorKind
+
+import pandas
+
+
+class CSV(Connector):
+    @staticmethod
+    def csv_from(meta_connector):
+        if meta_connector.type_name != 'csv':
+            raise ValueError(f"type_name={meta_connector.type_name} != csv")
+        filepath = meta_connector.uri
+        if len(filepath) == 0:
+            raise ValueError("uri not specified")
+        sep = meta_connector.sep if meta_connector.sep is not None else ','
+        header = meta_connector.header_row if meta_connector.header_row is not None else 'infer'
+        names = meta_connector.header_columns
+        index_col = meta_connector.index_col
+        skipinitialspace = meta_connector.skipinitialspace \
+            if meta_connector.skipinitialspace is not None else False
+        na_values = meta_connector.na_values
+        decimal = meta_connector.decimal if meta_connector.decimal is not None else '.'
+        return CSV(filepath=filepath, sep=sep, header=header, names=names, index_col=index_col,
+                   skipinitialspace=skipinitialspace, na_values=na_values, decimal=decimal)
+
+    def __init__(self, filepath, sep=',', header='infer', names=None, index_col=None,
+                 skipinitialspace=False, na_values=None, decimal='.'):
+        super().__init__(ConnectorKind.CSV)
+        self._filepath = Path(filepath)
+        if not self._filepath.is_file():
+            raise ValueError(f"filepath={filepath} is not a file")
+        self._sep = sep
+        self._header = header
+        self._names = names
+        self._index_col = index_col
+        self._skipinitialspace = skipinitialspace
+        self._na_values = na_values
+        self._decimal = decimal
+
+    def get_filepath(self):
+        return self._filepath
+
+    def get_sep(self):
+        return self._sep
+
+    def get_header(self):
+        return self._header
+
+    def get_names(self):
+        return self._names
+
+    def get_index_col(self):
+        return self._index_col
+
+    def get_skipinitialspace(self):
+        return self._skipinitialspace
+
+    def get_na_values(self):
+        return self._na_values
+
+    def get_decimal(self):
+        return self._decimal
+
+    def to_pandas(self):
+        return pandas.read_csv(filepath=self._filepath, sep=self._sep, header=self._header,
+                               names=self._names, index_col=self._index_col,
+                               skipinitialspace=self._skipinitialspace, na_values=self._na_values,
+                               decimal=self._decimal)

dq0/sdk/connector/kind/sql/db/mysql/mysql.py

@@ -0,0 +1,47 @@
+from dq0.sdk.connector.connector_kind import ConnectorKind
+from dq0.sdk.connector.kind.sql.db.mysql.mysql_query import MySQLQuery
+from dq0.sdk.connector.kind.sql.sql import SQL
+
+import sqlalchemy
+
+
+class MySQL(SQL):
+    @staticmethod
+    def mysql_from(meta_database):
+        meta_connector = meta_database.connector
+        if meta_connector.type_name != 'mysql':
+            raise ValueError(f"type_name={meta_connector.type_name} != mysql")
+        username = meta_connector.username if meta_connector.username is not None else ''
+        password = meta_connector.password if meta_connector.password is not None else ''
+        host = meta_connector.host if meta_connector.host is not None else ''
+        port = meta_connector.port if meta_connector.port is not None else ''
+        database = meta_database.data.name if meta_database.data.name is not None else ''
+        charset = meta_connector.charset if meta_connector.charset is not None else ''
+        return MySQL(username=username, password=password, host=host, port=port, database=database, charset=charset)
+
+    @staticmethod
+    def connection_uri(username='', password='', host='', port='', database='', charset=''):
+        if len(username) == 0:
+            password = ''
+        if len(host) == 0:
+            raise Exception("host not provided")
+        password_sep = ':' if 0 < len(password) else ''
+        user_sep = '@' if 0 < len(username) else ''
+        port_sep = ':' if 0 < len(port) else ''
+        database_sep = '/' if 0 < len(database) else ''
+        charset_sep = '?charset=' if 0 < len(charset) else ''
+        return f"mysql+mysqlconnector://{username}{password_sep}{password}{user_sep}{host}{port_sep}{port}{database_sep}{database}{charset_sep}{charset}"
+
+    def __init__(self, username='', password='', host='', port='', database='', charset=''):
+        super().__init__(ConnectorKind.SQL_MYSQL)
+        self._connection_uri = MySQL.connection_uri(username=username, password=password, host=host, port=port, database=database, charset=charset)
+        self._engine = sqlalchemy.create_engine(self.get_connection_uri())
+
+    def get_connection_uri(self):
+        return self._connection_uri
+
+    def get_engine(self):
+        return self._engine
+
+    def new_query(self, query_string):
+        return MySQLQuery(connector=self, query_string=query_string)

dq0/sdk/connector/kind/sql/db/mysql/mysql_query.py

@@ -0,0 +1,20 @@
+from dq0.sdk.connector.connector_kind import ConnectorKind
+from dq0.sdk.connector.kind.sql.query import Query
+
+import pandas
+
+
+class MySQLQuery(Query):
+    def __init__(self, connector, query_string):
+        super().__init__(connector, query_string)
+        if not connector.get_kind() == ConnectorKind.SQL_MYSQL:
+            raise ValueError(f"connector with kind={connector.get_kind()} is not of valid {ConnectorKind.SQL_MYSQL} kind")
+
+    def to_pandas(self):
+        engine = self.get_connector().get_engine()
+        if engine is None:
+            raise ValueError("engine is None")
+        connection = engine.raw_connection()
+        df = pandas.read_sql_query(sql=self.get_query_string(), con=connection)
+        connection.close()
+        return df