diff --git a/dq0/sdk/cli/experiment.py b/dq0/sdk/cli/experiment.py index 0f8960fa..65cb0059 100644 --- a/dq0/sdk/cli/experiment.py +++ b/dq0/sdk/cli/experiment.py @@ -15,6 +15,7 @@ """ from dq0.sdk.cli.api import routes +from dq0.sdk.cli import Data from dq0.sdk.cli.runner import DataRunner, ModelRunner from dq0.sdk.errors import DQ0SDKError, checkSDKResponse @@ -54,6 +55,7 @@ def __init__(self, project=None, name=None): raise ValueError('You need to set the "name" argument') self.project = project self.name = name + self.datasets_used = None def get_last_model_run(self): """Returns the latest ModelRunner. @@ -75,7 +77,21 @@ def get_last_data_run(self): """ return DataRunner(self.project) - def run(self, entry_point='train_use_dq0_makedp', args=''): + def get_dataset_uuids(self, datasets=None): + """Returns used dataset uuids as a single comma-separated string""" + if not datasets: + datasets = self.datasets_used + + data_uuids = [] + for dataset in datasets: + if isinstance(dataset, Data): + data_uuids.append(dataset.uuid) + elif type(dataset) == str: + data_uuids.append(dataset) + + return ','.join(data_uuids) + + def run(self, args, datasets=None): """Starts a training run It calls the CLI command `model train` and returns @@ -84,26 +100,53 @@ def run(self, entry_point='train_use_dq0_makedp', args=''): Returns: A new instance of the ModelRunner class for the train run. """ + if not isinstance(args, dict): + raise TypeError('args need to passed as a dict') + response = self.project._deploy() checkSDKResponse(response) self.project.update_commit_uuid(response['message']) + data_uuids = self.get_dataset_uuids(datasets) + + if not data_uuids or not len(data_uuids): + raise DQ0SDKError('No datasets provided. Please choose which datasets to use for this run using the' + 'datasets parameter or the .for_data() method') + data = { 'project_uuid': self.project.project_uuid, 'commit_uuid': self.project.commit_uuid, - 'ml_project_entry_point': entry_point, - 'args': args + 'experiment_name': self.name, + 'args': args, + 'datasets': data_uuids } response = self.project.client.post(routes.runs.create, data=data) checkSDKResponse(response) - print(response['message']) + print(response) try: job_uuid = response['message'].split(' ')[-1] except Exception: raise DQ0SDKError('Could not parse new commit uuid') return ModelRunner(self.project, job_uuid) + def for_data(self, data): + """ + Specifiy which datasets are used in query. + Args: + data (:obj:`list`) list of :obj:`dq0.sdk.cli.Data` instances included in query. Alternatively, pass a single + :obj:`dq0.sdk.cli.Data` instance. + + Returns: + :obj:`dq0.sdk.cli.Query` instance with set datasets + """ + if isinstance(data, Data): + data = [data] + elif not isinstance(data, list): + raise DQ0SDKError('Please provide datasets either as list of Data objects or a single Data instance') + self.datasets_used = data + return self + def preprocess(self): """Starts a preprocessing run @@ -118,5 +161,5 @@ def preprocess(self): response = self.project.post(routes.data.preprocess, id=self.project.data_source_uuid) checkSDKResponse(response) - print(response['message']) + print(response) return DataRunner(self.project) diff --git a/dq0/sdk/cli/model.py b/dq0/sdk/cli/model.py index 3cbc457d..7ae508b6 100644 --- a/dq0/sdk/cli/model.py +++ b/dq0/sdk/cli/model.py @@ -126,7 +126,7 @@ def predict(self, test_data): response = self.project.client.post( routes.model.predict, uuid=self.model_uuid, data=data) checkSDKResponse(response) - print(response['message']) + print(response) try: job_uuid = response['message'].split(' ')[-1] except Exception: diff --git a/dq0/sdk/cli/project.py b/dq0/sdk/cli/project.py index 7e88076d..6a1fc512 100644 --- a/dq0/sdk/cli/project.py +++ b/dq0/sdk/cli/project.py @@ -49,6 +49,8 @@ class Project: Args: name (:obj:`str`): The name of the new project + type (:obj:`str`): Type of project template to use. Can be either 'ml', 'query', 'synth' or 'estimator'. + Defaults to 'ml'. create (bool): True to create a new project via DQ0 CLI. Default is True. @@ -62,10 +64,11 @@ class Project: """ - def __init__(self, name=None, create=True): + def __init__(self, name=None, project_type='ml', create=True): if name is None: raise ValueError('You need to set the "name" argument') self.name = name + self.project_type = project_type self.commit_uuid = '' self.datasets = [] self.experiment_name = '' @@ -128,9 +131,11 @@ def _create_new(self, name): name (:obj:`str`): The name of the new project """ working_dir = os.getcwd() - response = self.client.post(routes.project.create, data={'working_dir': working_dir, 'name': name}) + response = self.client.post(routes.project.create, data={'working_dir': working_dir, + 'name': name, + 'type': self.project_type}) checkSDKResponse(response) - print(response['message']) + print(response) # change to working directory where the new project was created os.chdir(working_dir) @@ -242,7 +247,7 @@ def attach_data_source(self, data=None, data_uuid=None, data_name=None): else: raise ValueError('Missing required parameter: data (Data instance) or data_uuid and data_name') - print(response['message']) + print(response) def detach_data_source(self, data=None, data_uuid=None, data_name=None): """Detaches a new data source to the project. @@ -270,7 +275,7 @@ def detach_data_source(self, data=None, data_uuid=None, data_name=None): else: raise ValueError('Missing required parameter: data (Data instance) or data_uuid and data_name') - print(response['message']) + print(response) def get_attached_data_sources(self): pass diff --git a/dq0/sdk/cli/query.py b/dq0/sdk/cli/query.py index 9d139298..536321b1 100644 --- a/dq0/sdk/cli/query.py +++ b/dq0/sdk/cli/query.py @@ -6,6 +6,8 @@ Copyright 2020, Gradient Zero All rights reserved """ +import base64 + from dq0.sdk.cli import Project from dq0.sdk.cli.api import Client, routes from dq0.sdk.cli.data import Data @@ -58,31 +60,62 @@ def get_dataset_names(self): """Returns used dataset names as a single comma-separated string""" return ','.join([dataset.name for dataset in self.datasets_used]) - def execute(self, query, epsilon=1.0, tau=0.0, private_column='', permissions=None, params=None): + def get_dataset_uuids(self, datasets=None): + """Returns used dataset uuids as a single comma-separated string""" + if not datasets: + datasets = self.datasets_used + + data_uuids = [] + for dataset in datasets: + if isinstance(dataset, Data): + data_uuids.append(dataset.uuid) + elif type(dataset) == str: + data_uuids.append(dataset) + + return ','.join(data_uuids) + + def execute(self, query, args): """Run a query on the data sources defined by this Query instance. Args: query: string containing SQL - epsilon: float; Epsilon value for differential private query. Default: 1.0 - tau: float; Tau threshold value for private query. Default: 0.0 - private_column: string; Private column for this query. Leave empty or omit for default value from metadata. - permissions: optional; e.g. 'households<75' - params: optional; e.g. 'p1=123' + args: + entry-point: string; + epsilon: float; Epsilon value for differential private query. Default: 1.0 + tau: float; Tau threshold value for private query. Default: 0.0 + private_column: string; Private column for this query. Leave empty or omit for default value from metadata. + permissions: optional; e.g. 'households<75' + params: optional; e.g. 'p1=123' Returns: :obj:`dq0.sdk.cli.runner.QueryRunner` instance """ + if not isinstance(args, dict): + raise TypeError('args need to passed as a dict') + + response = self.project._deploy() + checkSDKResponse(response) + self.project.update_commit_uuid(response['message']) + + if 'epsilon' not in args: + args['epsilon'] = '1.0' + if 'tau' not in args: + args['tau'] = '0' + if 'entry-point' not in args: + args['entry-point'] = 'execute' + args['job-type'] = 'query.run' + + query_encoded = base64.b64encode(query.encode('UTF-8')) + args['query-encoded'] = query_encoded.decode('UTF-8') + if not self.datasets_used: - raise DQ0SDKError('Please specify which datasets to use for query') + raise DQ0SDKError('Please specify which datasets to use for query using the .for_data() method') response = self.client.post( - route=routes.query.create, - data={'query': query, - 'datasets': self.get_dataset_names(), - 'epsilon': epsilon, - 'tau': tau, - 'private_column': private_column, - 'permissions': permissions, - 'params': params, - 'project_uuid': self.project.project_uuid + route=routes.runs.create, + data={'datasets': self.get_dataset_uuids(), + 'project_uuid': self.project.project_uuid, + 'commit_uuid': self.project.commit_uuid, + 'job-type': 'query.run', + 'args': args } ) checkSDKResponse(response) diff --git a/dq0/sdk/cli/runner/runner.py b/dq0/sdk/cli/runner/runner.py index 05d50e7f..a1d4b903 100644 --- a/dq0/sdk/cli/runner/runner.py +++ b/dq0/sdk/cli/runner/runner.py @@ -128,7 +128,7 @@ def _cancel(self, route, uuid): """ response = self.project.client.post(route, uuid=uuid) checkSDKResponse(response) - print(response['message']) + print(response) def wait_for_completion(self, verbose=False): """Loops until the state reflects the end of the run. diff --git a/dq0/sdk/cli/runner/state.py b/dq0/sdk/cli/runner/state.py index c7e5b6ca..2084f7d3 100644 --- a/dq0/sdk/cli/runner/state.py +++ b/dq0/sdk/cli/runner/state.py @@ -44,7 +44,7 @@ def __init__(self): def update(self, response): """Updates the state representation""" - self.message = response['job_state'] + self.message = response.get('job_state') try: self.job_uuid = response.get('job_uuid') self.state = response.get('job_state') diff --git a/notebooks/DQ0SDK-Quickstart.ipynb b/notebooks/DQ0SDK-Quickstart.ipynb index 20a206af..7c40ac8b 100644 --- a/notebooks/DQ0SDK-Quickstart.ipynb +++ b/notebooks/DQ0SDK-Quickstart.ipynb @@ -1,5 +1,18 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# DQ0 SDK Demo\n", + "## Prerequistes\n", + "* Installed DQ0 SDK. Install with `pip install dq0-sdk`\n", + "* Installed DQ0 CLI.\n", + "* Proxy running and registered from the DQ0 CLI with `dq0 proxy add ...`\n", + "* Valid session of DQ0. Log in with `dq0 user login`\n", + "* Running instance of DQ0 CLI server: `dq0 server start`" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -23,6 +36,7 @@ "metadata": {}, "outputs": [], "source": [ + "# ensure we are in the dq0-sdk directory\n", "%cd ../" ] }, @@ -40,40 +54,50 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Create a project\n", + "## 1a. Create a project\n", "Projects act as the working environment for model development.\n", "Each project has a model directory with a .meta file containing the model uuid, attached data sources etc.\n", - "Creating a project with `Project.create(name='model_1')` is equivalent to calling the DQ0 Cli command `dq0-cli project create model_1`" + "Creating a project with `Project.create(name='model_1')` is equivalent to calling the DQ0 Cli command `dq0 project create model_1`. Note that this newly created project only exists locally on your filesystem as long as you dont start a run or commit/deploy manually." ] }, { - "cell_type": "markdown", - "metadata": {}, + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ - "# DQ0 SDK Demo\n", - "## Prerequistes\n", - "* Installed DQ0 SDK. Install with `pip install dq0-sdk`\n", - "* Installed DQ0 CLI.\n", - "* Proxy running and registered from the DQ0 CLI with `dq0-cli proxy add ...`\n", - "* Valid session of DQ0. Log in with `dq0 user login`\n", - "* Running instance of DQ0 CLI server: `dq0 server start`" + "# create a project with name 'model_1' and type 'ml', which will provide us a \n", + "# project template for machine learning purposes. \n", + "# Automatically creates the 'model_1' directory and changes to this directory.\n", + "project = Project(name='model_1', project_type='ml')" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# create a project with name 'model_1'. Automatically creates the 'model_1' directory and changes to this directory.\n", - "project = Project(name='model_1')" + "We chose to create this project using the default machine learning template. This will define several entry points for executing runs in an *MLproject* file, including:\n", + "\n", + "- my_model.py:\n", + "A model training demo with DQ0's Differential Privacy Module (dq0.makedp). Uses DQ0's census dataset.\n", + "\n", + "- mlflow_quickstart.py:\n", + "A simple Python module to run any code with DQ0's managed mlflow.\n", + "\n", + "- keras_model_simple.py:\n", + "A simple model training example without using DQ0's Differential Privacy Module (dq0.makedp)\n", + "\n", + "- transform.py:\n", + "A basic data transformation module for creating new transformed datasets in DQ0. Uses DQ0's census dataset." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Load a project\n", + "## 1b. Load a project\n", "Alternatively, you can load an existing project by first cd'ing into this directory and then call Project.load()\n", "This will read in the .meta file of this directory" ] @@ -100,7 +124,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "project.project_uuid" @@ -110,27 +136,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Create Experiment\n", - "To execute DQ0 training commands inside the quarantine you define experiments for your projects.\n", - "You can create as many experiments as you like for one project." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Create experiment for project\n", - "experiment = Experiment(project=project, name='experiment_1')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Get and attach data source\n", - "For new projects you need to attach a data source. Existing (loaded) projects usually already have data sources attached." + "## 2. Check data sources\n", + "New projects, by default, have all available datasets already attached to them. These data sources are typically defined by the data owner." ] }, { @@ -144,7 +151,7 @@ "\n", "# get info about the first source\n", "info = project.get_data_info(sources[0])\n", - "info" + "info # make sure this is the census dataset" ] }, { @@ -185,31 +192,28 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now, attach the dataset to our project" + "If you want to detach a dataset, type the following:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ - "# attach the first dataset\n", - "project.attach_data_source(sources[0])" + "# if you want to detach some data\n", + "# project.detach_data_source(sources[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Define the model\n", - "Working with DQ0 is basically about defining two functions:\n", - "* setup_data() - called right before model training to prepare attached data sources\n", - "* setup_model() - actual model definition code\n", - "The easiest way to define those functions is to write them in the notebook (inline) and pass them to the project before calling deploy. Alternatively, the user can write the complete user_model.py to the project's directory.\n", - "\n", - "### Define fuctions inline\n", - "First variant with functions passed to the project instance. Note that you need to define imports inline inside the functions as only those code blocks are replaced in the source files." + "## 3. Create Experiment\n", + "To execute DQ0 training commands inside the quarantine you define experiments for your projects.\n", + "You can create as many experiments as you like for one project." ] }, { @@ -218,402 +222,29 @@ "metadata": {}, "outputs": [], "source": [ - "# define functions\n", - "\n", - "def setup_data():\n", - " # load input data\n", - " if self.data_source is None:\n", - " logger.error('No data source found')\n", - " return\n", - "\n", - " data = self.data_source.read()\n", - "\n", - " # read and preprocess the data\n", - " dataset_df = self.preprocess()\n", - "\n", - " from sklearn.model_selection import train_test_split\n", - " X_train_df, X_test_df, y_train_ts, y_test_ts =\\\n", - " train_test_split(dataset_df.iloc[:, :-1],\n", - " dataset_df.iloc[:, -1],\n", - " test_size=0.33,\n", - " random_state=42)\n", - " self.input_dim = X_train_df.shape[1]\n", - "\n", - " # set data member variables\n", - " self.X_train = X_train_df\n", - " self.X_test = X_test_df\n", - " self.y_train = y_train_ts\n", - " self.y_test = y_test_ts\n", - " \n", - "def setup_model():\n", - " import tensorflow.compat.v1 as tf\n", - " self.optimizer = tf.keras.optimizers.Adam(learning_rate=0.1)\n", - " self.loss = tf.keras.losses.SparseCategoricalCrossentropy()\n", - " # As an alternative, define the loss function with a string\n", - " self.epochs = 10\n", - " self.batch_size = 250\n", - " # self.optimizer = tf.keras.optimizers.Adam(learning_rate=self.learning_rate)\n", - " self.optimizer = 'Adam'\n", - " self.num_microbatches = 250\n", - " self.metrics = ['accuracy']\n", - " self.loss = tf.keras.losses.SparseCategoricalCrossentropy()\n", - " self.model = tf.keras.Sequential([\n", - " tf.keras.layers.Input(self.input_dim),\n", - " tf.keras.layers.Dense(10, activation='tanh'),\n", - " tf.keras.layers.Dense(10, activation='tanh'),\n", - " tf.keras.layers.Dense(2, activation='softmax')])\n", - " \n", - "def preprocess():\n", - " # columns\n", - " column_names_list = [\n", - " 'lastname',\n", - " 'firstname',\n", - " 'age',\n", - " 'workclass',\n", - " 'fnlwgt',\n", - " 'education',\n", - " 'education-num',\n", - " 'marital-status',\n", - " 'occupation',\n", - " 'relationship',\n", - " 'race',\n", - " 'sex',\n", - " 'capital-gain',\n", - " 'capital-loss',\n", - " 'hours-per-week',\n", - " 'native-country',\n", - " 'income'\n", - " ]\n", - "\n", - " # columns types list drawn from data source types information above.\n", - " columns_types_list = [\n", - " {\n", - " 'name': 'age',\n", - " 'type': 'int'\n", - " },\n", - " {\n", - " 'name': 'workclass',\n", - " 'type': 'string',\n", - " 'values': [\n", - " 'Private',\n", - " 'Self-emp-not-inc',\n", - " 'Self-emp-inc',\n", - " 'Federal-gov',\n", - " 'Local-gov',\n", - " 'State-gov',\n", - " 'Without-pay',\n", - " 'Never-worked',\n", - " 'Unknown'\n", - " ]\n", - " },\n", - " {\n", - " 'name': 'fnlwgt',\n", - " 'type': 'int'\n", - " },\n", - " {\n", - " 'name': 'education',\n", - " 'type': 'string',\n", - " 'values': [\n", - " 'Bachelors',\n", - " 'Some-college',\n", - " '11th',\n", - " 'HS-grad',\n", - " 'Prof-school',\n", - " 'Assoc-acdm',\n", - " 'Assoc-voc',\n", - " '9th',\n", - " '7th-8th',\n", - " '12th',\n", - " 'Masters',\n", - " '1st-4th',\n", - " '10th',\n", - " 'Doctorate',\n", - " '5th-6th',\n", - " 'Preschool'\n", - " ]\n", - " },\n", - " {\n", - " 'name': 'education-num',\n", - " 'type': 'int'\n", - " },\n", - " {\n", - " 'name': 'marital-status',\n", - " 'type': 'string',\n", - " 'values': [\n", - " 'Married-civ-spouse',\n", - " 'Divorced',\n", - " 'Never-married',\n", - " 'Separated',\n", - " 'Widowed',\n", - " 'Married-spouse-absent',\n", - " 'Married-AF-spouse'\n", - " ]\n", - " },\n", - " {\n", - " 'name': 'occupation',\n", - " 'type': 'string',\n", - " 'values': [\n", - " 'Tech-support',\n", - " 'Craft-repair',\n", - " 'Other-service',\n", - " 'Sales',\n", - " 'Exec-managerial',\n", - " 'Prof-specialty',\n", - " 'Handlers-cleaners',\n", - " 'Machine-op-inspct',\n", - " 'Adm-clerical',\n", - " 'Farming-fishing',\n", - " 'Transport-moving',\n", - " 'Priv-house-serv',\n", - " 'Protective-serv',\n", - " 'Armed-Forces',\n", - " 'Unknown'\n", - " ]\n", - " },\n", - " {\n", - " 'name': 'relationship',\n", - " 'type': 'string',\n", - " 'values': [\n", - " 'Wife',\n", - " 'Own-child',\n", - " 'Husband',\n", - " 'Not-in-family',\n", - " 'Other-relative',\n", - " 'Unmarried'\n", - " ]\n", - " },\n", - " {\n", - " 'name': 'race',\n", - " 'type': 'string',\n", - " 'values': [\n", - " 'White',\n", - " 'Asian-Pac-Islander',\n", - " 'Amer-Indian-Eskimo',\n", - " 'Other',\n", - " 'Black'\n", - " ]\n", - " },\n", - " {\n", - " 'name': 'sex',\n", - " 'type': 'string',\n", - " 'values': [\n", - " 'Female',\n", - " 'Male'\n", - " ]\n", - " },\n", - " {\n", - " 'name': 'capital-gain',\n", - " 'type': 'int'\n", - " },\n", - " {\n", - " 'name': 'capital-loss',\n", - " 'type': 'int'\n", - " },\n", - " {\n", - " 'name': 'hours-per-week',\n", - " 'type': 'int'\n", - " },\n", - " {\n", - " 'name': 'native-country',\n", - " 'type': 'string',\n", - " 'values': [\n", - " 'United-States',\n", - " 'Cambodia',\n", - " 'England',\n", - " 'Puerto-Rico',\n", - " 'Canada',\n", - " 'Germany',\n", - " 'Outlying-US(Guam-USVI-etc)',\n", - " 'India',\n", - " 'Japan',\n", - " 'Greece',\n", - " 'South',\n", - " 'China',\n", - " 'Cuba',\n", - " 'Iran',\n", - " 'Honduras',\n", - " 'Philippines',\n", - " 'Italy',\n", - " 'Poland',\n", - " 'Jamaica',\n", - " 'Vietnam',\n", - " 'Mexico',\n", - " 'Portugal',\n", - " 'Ireland',\n", - " 'France',\n", - " 'Dominican-Republic',\n", - " 'Laos',\n", - " 'Ecuador',\n", - " 'Taiwan',\n", - " 'Haiti',\n", - " 'Columbia',\n", - " 'Hungary',\n", - " 'Guatemala',\n", - " 'Nicaragua',\n", - " 'Scotland',\n", - " 'Thailand',\n", - " 'Yugoslavia',\n", - " 'El-Salvador',\n", - " 'Trinadad&Tobago',\n", - " 'Peru',\n", - " 'Hong',\n", - " 'Holand-Netherlands',\n", - " 'Unknown'\n", - " ]\n", - " }\n", - " ]\n", - " \n", - " from dq0.sdk.data.preprocessing import preprocessing\n", - " import sklearn.preprocessing\n", - " import pandas as pd\n", - "\n", - " if 'dataset' in globals():\n", - " # local testing mode\n", - " dataset = globals()['dataset']\n", - " else:\n", - " # get the input dataset\n", - " if self.data_source is None:\n", - " logger.error('No data source found')\n", - " return\n", - "\n", - " # read the data via the attached input data source\n", - " dataset = self.data_source.read(\n", - " names=column_names_list,\n", - " sep=',',\n", - " skiprows=1,\n", - " index_col=None,\n", - " skipinitialspace=True,\n", - " na_values={\n", - " 'capital-gain': 99999,\n", - " 'capital-loss': 99999,\n", - " 'hours-per-week': 99,\n", - " 'workclass': '?',\n", - " 'native-country': '?',\n", - " 'occupation': '?'}\n", - " )\n", - "\n", - " # drop unused columns\n", - " dataset.drop(['lastname', 'firstname'], axis=1, inplace=True)\n", - " column_names_list.remove('lastname')\n", - " column_names_list.remove('firstname')\n", - "\n", - " # define target feature\n", - " target_feature = 'income'\n", - "\n", - " # get categorical features\n", - " categorical_features_list = [\n", - " col['name'] for col in columns_types_list\n", - " if col['type'] == 'string']\n", - "\n", - " # get categorical features\n", - " quantitative_features_list = [\n", - " col['name'] for col in columns_types_list\n", - " if col['type'] == 'int' or col['type'] == 'float']\n", - "\n", - " # get arguments\n", - " approach_for_missing_feature = 'imputation'\n", - " imputation_method_for_cat_feats = 'unknown'\n", - " imputation_method_for_quant_feats = 'median'\n", - " features_to_drop_list = None\n", - "\n", - " # handle missing data\n", - " dataset = preprocessing.handle_missing_data(\n", - " dataset,\n", - " mode=approach_for_missing_feature,\n", - " imputation_method_for_cat_feats=imputation_method_for_cat_feats,\n", - " imputation_method_for_quant_feats=imputation_method_for_quant_feats, # noqa: E501\n", - " categorical_features_list=categorical_features_list,\n", - " quantitative_features_list=quantitative_features_list)\n", - "\n", - " if features_to_drop_list is not None:\n", - " dataset.drop(features_to_drop_list, axis=1, inplace=True)\n", - "\n", - " # get dummy columns\n", - " dataset = pd.get_dummies(dataset, columns=categorical_features_list, dummy_na=False) \n", - "\n", - " # unzip categorical features with dummies\n", - " categorical_features_list_with_dummies = []\n", - " for col in columns_types_list:\n", - " if col['type'] == 'string':\n", - " for value in col['values']:\n", - " categorical_features_list_with_dummies.append('{}_{}'.format(col['name'], value))\n", - "\n", - " # add missing columns\n", - " missing_columns = set(categorical_features_list_with_dummies) - set(dataset.columns)\n", - " for col in missing_columns:\n", - " dataset[col] = 0\n", - " \n", - " # and sort the columns\n", - " dataset = dataset.reindex(sorted(dataset.columns), axis=1)\n", - "\n", - " # Scale values to the range from 0 to 1 to be precessed by the neural network\n", - " dataset[quantitative_features_list] = sklearn.preprocessing.minmax_scale(dataset[quantitative_features_list])\n", - "\n", - " # label target\n", - " y_ts = dataset[target_feature]\n", - " le = sklearn.preprocessing.LabelEncoder()\n", - " y_bin_nb = le.fit_transform(y_ts)\n", - " y_bin = pd.Series(index=y_ts.index, data=y_bin_nb)\n", - " dataset.drop([target_feature], axis=1, inplace=True)\n", - " dataset[target_feature] = y_bin\n", - "\n", - " return dataset\n", - " \n", - "# set model code in project\n", - "project.set_model_code(setup_data=setup_data, setup_model=setup_model, preprocess=preprocess, parent_class_name='NeuralNetworkClassification')" + "# Create experiment for project\n", + "experiment = Experiment(project=project, name='experiment_1')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Define functions as source code\n", - "Second variant, writing the complete model. Template can be retrieved by `!cat models/user_model.py` which is created by Project create." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile models/user_model.py\n", - "\n", - "import logging\n", - "\n", - "from dq0.sdk.models.tf import NeuralNetworkClassification\n", - "\n", - "logger = logging.getLogger()\n", - "\n", - "\n", - "class UserModel(NeuralNetworkClassification):\n", - " \"\"\"Derived from dq0.sdk.models.tf.NeuralNetwork class\n", - "\n", - " Model classes provide a setup method for data and model\n", - " definitions.\n", - " \"\"\"\n", - " def __init__(self):\n", - " super().__init__()\n", - "\n", - " def setup_data(self):\n", - " \"\"\"Setup data function. See code above...\"\"\"\n", - " pass\n", + "## 4. Create a training run using dq0-makedp\n", + "Working with DQ0 is basically about defining two functions:\n", + "* setup_data() - called right before model training to prepare attached data sources\n", + "* setup_model() - actual model definition code\n", "\n", - " def preprocess(self):\n", - " \"\"\"Preprocess the data. See code above...\"\"\"\n", - " pass\n", + "The easiest way to define those functions is to write them in the notebook (inline) and pass them to the project before starting a run. Alternatively, the user can write the complete user_model.py to the project's directory.\n", "\n", - " def setup_model(self):\n", - " \"\"\"Setup model function See code above...\"\"\"\n", - " pass\n" + "The machine learning project template ('ml') we chose earlier when creating the project already includes these classes for us. They can be found and adjusted in the project folder. We will continue with the template code in this tutorial." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Train the model\n", - "After testing the model locally directly in this notebook, it's time to train it inside the DQ0 quarantine. This is done by calling experiment.train() which in turn calls the Cli commands `dq0-cli project deploy` and `dq0-cli model train`" + "To start a run, we call experiment.run(args, datasets) which in turn calls the Cli commands `dq0 project commit` and `dq0 commit run`." ] }, { @@ -622,7 +253,18 @@ "metadata": {}, "outputs": [], "source": [ - "run = experiment.run()" + "# set run parameters\n", + "args = {\n", + " 'DP-epsilon': \"1\",\n", + " 'entry-point': \"train_dq0_makedp\",\n", + " 'job-type': \"commit.run.train\",\n", + " 'module-path': \"my_model.py\",\n", + "}\n", + "\n", + "# define datasets for this run - we choose to run it on only one dataset. \n", + "datasets = [sources[4]]\n", + "\n", + "run = experiment.run(args, datasets=datasets)" ] }, { @@ -653,39 +295,32 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "# get training results\n", "print(run.get_results())" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After train dq0 will run the model checker to evaluate if the trained model is safe and allowed for prediction. Get the state of the checker run together with the other state information with the get_state() function:" - ] - }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ - "# get the state whenever you like\n", - "print(run.get_state())" + "# if an error occured, you can query it with\n", + "run.get_error()" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# get the model\n", - "model = run.get_model()\n", - "model.__dict__" + "After train dq0 will run the model checker to evaluate if the trained model is safe and allowed for prediction. Get the state of the checker run together with the other state information with the get_state() function:" ] }, { @@ -694,8 +329,8 @@ "metadata": {}, "outputs": [], "source": [ - "# register the model\n", - "model.register()" + "# get the state whenever you like\n", + "print(run.get_state())" ] }, { diff --git a/notebooks/DQ0SDK-Quickstart_Queries.ipynb b/notebooks/DQ0SDK-Quickstart_Queries.ipynb index 5df4ee1d..7c9b9b01 100644 --- a/notebooks/DQ0SDK-Quickstart_Queries.ipynb +++ b/notebooks/DQ0SDK-Quickstart_Queries.ipynb @@ -4,16 +4,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Querying Data\n", + "# Querying Data in DQ0 Demo\n", "\n", "In order to query data you will need:\n", "* Installed DQ0 SDK. Install with `pip install dq0-sdk`\n", "* Installed DQ0 CLI.\n", - "* Proxy running and registered from the DQ0 CLI with `dq0-cli proxy add ...`\n", + "* Proxy running and registered from the DQ0 CLI with `dq0 proxy add ...`\n", "* Valid session of DQ0. Log in with `dq0 user login`\n", "* Running instance of DQ0 CLI server: `dq0 server start`\n", "\n", - "* DQ0 Project with Data attached to it.\n", + "* DQ0 Project with Data attached to it - ideally using a database backend like PostgreSQL.\n", "Keep in mind that a query is always executed within the context of a project.\n", "\n", "Start by importing the core classes" @@ -21,24 +21,16 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/Users/dominic/Projects/dq0-sdk\n" - ] - } - ], + "outputs": [], "source": [ "%cd ../" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -50,58 +42,45 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Create or load a project\n", - "Projects act as the working environment for model development.\n", - "Each project has a model directory with a .meta file containing the model uuid, attached data sources etc.\n", - "Creating a project with `Project.create(name='model_1')` is equivalent to calling the DQ0 Cli command `dq0-cli project create model_1`.\n", - "Alternatively, if you want to load an existing project, navigate your current working directory to the project directory so you can use the Project.load() method." + "## 1. Create or load a project\n", + "Let's create a new project for our query use case." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# create a project with name 'model_1'. Automatically creates the 'model_1' directory and changes to this directory.\n", - "# project = Project(name='project_1')" + "# create a project with name 'query_1'. Automatically creates the 'query_1' directory and changes to this directory.\n", + "project = Project(name='query_1', project_type='query')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Load a project\n", "Alternatively, you can load an existing project by first cd'ing into this directory and then call Project.load()\n", "This will read in the .meta file of this directory." ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Errno 20] Not a directory: '../dq0-cli/MyNewProject'\n", - "/Users/dominic/go/src/dq0-cli/MyNewProject\n" - ] - } - ], + "outputs": [], "source": [ - "%cd ../dq0-cli/MyNewProject" + "# %cd ../dq0-cli/MyNewProject" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Alternative: load a project from the current model directory\n", - "project = Project.load()" + "# project = Project.load()" ] }, { @@ -113,20 +92,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'04a74fa8-8af0-4fb5-82c5-06e6966702fe'" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "project.project_uuid" ] @@ -135,67 +103,32 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Get and attach data source\n", - "For new projects you need to attach a data source. Existing (loaded) projects usually already have data sources attached." + "## 2. Check the data source\n", + "\n", + "All datasets should already be attached. You will need a valid dataset for this demo, ideally using a database backend." ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'commit_uuid': '360ac136-c6a1-4ff6-9328-37b66ebb23c4',\n", - " 'data_uuid': 'b1ed3b6c-1a71-4597-98ec-6583e1b4ee99',\n", - " 'data_name': 'cpg_segments',\n", - " 'data_type': 'CSV',\n", - " 'data_description': 'some description',\n", - " 'privacy_budget': {'initial': 1000,\n", - " 'current': 994,\n", - " 'created_at': 1610619263,\n", - " 'updated_at': 1610622299},\n", - " 'data_usage': 89,\n", - " 'data_privacy_column': 'idl',\n", - " 'data_size': 1000,\n", - " 'data_meta': 'bmFtZTogY3BnX3NlZ21lbnRzCmRlc2NyaXB0aW9uOiBzb21lIGRlc2NyaXB0aW9uCnR5cGU6IENTVgpjb25uZWN0aW9uOiBmaWxlOi8vL1VzZXJzL2RvbWluaWMvUHJvamVjdHMvZHEwLXNxbC90ZXN0cy9kYXRhL2NwZ19zZWdtZW50cy5jc3YKcHJpdmFjeV9idWRnZXQ6IDEwMDAKcHJpdmFjeV9idWRnZXRfaW50ZXJ2YWxfZGF5czogMzAKc3ludGhfYWxsb3dlZDogZmFsc2UKcHJpdmFjeV9sZXZlbDogMgpwcml2YWN5X2NvbHVtbjogaWRsCnNpemU6IDEwMDAKTFI6CiAgY3BnX3NlZ21lbnRzOgogICAgcm93X3ByaXZhY3k6IGZhbHNlCiAgICByb3dzOiAxMDAwCiAgICBtYXhfaWRzOiAxCiAgICBzYW1wbGVfbWF4X2lkczogdHJ1ZQogICAgY2Vuc29yX2RpbXM6IHRydWUKICAgIGNsYW1wX2NvdW50czogZmFsc2UKICAgIGNsYW1wX2NvbHVtbnM6IHRydWUKICAgIGlkbDoKICAgICAgcHJpdmF0ZV9pZDogdHJ1ZQogICAgICB0eXBlOiBzdHJpbmcKICAgIGFjdGl2ZV9jb21wbGFpbnQ6CiAgICAgIHR5cGU6IGludAogICAgbG95YWx0eV90aWVyczoKICAgICAgdHlwZTogc3RyaW5nCg==',\n", - " 'created_at': 1610619263,\n", - " 'updated_at': 1610619263}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# first get some info about available data sources\n", "sources = Data.get_available_data_sources()\n", "\n", "# get info about the first source\n", - "info = Data.get_data_info(sources[1])\n", + "info = Data.get_data_info(sources[0])\n", "info" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "scrolled": false }, - "outputs": [ - { - "data": { - "text/plain": [ - "'cpg_segments'" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# print information about column types and values, description. This may be helpful for creating your queries.\n", "info['data_name']" @@ -203,59 +136,30 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'CSV'" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "info['data_type']" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'some description'" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "info['data_description']" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Suppose are happy with this information and now want to query this dataset in our project." - ] - }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# set data\n", - "data = sources[1]\n", + "data = sources[0]\n", "\n", "# alternatively, if you already know the name of the dataset:\n", "# data = Data('name_of_dataset')" @@ -265,56 +169,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### New projects\n", - "For new projects, we need to attach this data source first." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "data is already attached to project\n" - ] - } - ], - "source": [ - "project.attach_data_source(data=data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Existing projects\n", - "For existing projects, we need to check whether the dataset of interest is already attached to our project." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "project.get_attached_data_sources()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create Query\n", + "## 3. Create Query\n", "\n", "Once we have a project with data attached to it we can create our query. Think of this object like a query manager that can create multiple query runs." ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -330,20 +192,9 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "query.for_data(data)" ] @@ -357,11 +208,18 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "stmt = \"\"\"SELECT SUM(active_complaint), COUNT(*) as tx_count, c.loyalty_tiers FROM LR.cpg_segments as c WHERE c.loyalty_tiers = 'silver' AND c.active_complaint > 0 GROUP BY loyalty_tiers ORDER BY tx_count DESC LIMIT 600\"\"\"" + "stmt = \"\"\"SELECT SUM(active_complaint), \n", + " COUNT(*) as tx_count, c.loyalty_tiers \n", + " FROM LR.cpg_segments as c \n", + " WHERE c.loyalty_tiers = 'silver' \n", + " AND c.active_complaint > 0 \n", + " GROUP BY loyalty_tiers \n", + " ORDER BY tx_count \n", + " DESC LIMIT 600\"\"\"" ] }, { @@ -374,36 +232,26 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "run = query.execute(stmt)" + "args = {\n", + " 'entry-point': 'execute',\n", + " 'epsilon': '100',\n", + " 'private-column': 'idl',\n", + " 'tau': '0'\n", + "}\n", + "run = query.execute(stmt, args)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "running\n", - "Waiting for job to complete...\n", - "finished\n", - "finished\n", - "Job completed\n", - "finished\n", - "0x7be3,tx_count,c_loyalty_tiers\n", - "2556,2561,silver\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "# check status\n", "run.get_state()\n", @@ -416,6 +264,15 @@ "print(result)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "run.get_error()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -427,62 +284,13 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "running\n", - "error\n", - "2021-01-19T15:50:55Z | dq0.sql.runner | INFO | [__KEYWORD_STARTED__] Started with args: Namespace(_loglevel='debug', epsilon=1.0, meta=['bmFtZTogY3BnX3NlZ21lbnRzCmRlc2NyaXB0aW9uOiBzb21lIGRlc2NyaXB0aW9uCnR5cGU6IENTVgpjb25uZWN0aW9uOiBmaWxlOi8vL1VzZXJzL2RvbWluaWMvUHJvamVjdHMvZHEwLXNxbC90ZXN0cy9kYXRhL2NwZ19zZWdtZW50cy5jc3YKcHJpdmFjeV9idWRnZXQ6IDEwMDAKcHJpdmFjeV9idWRnZXRfaW50ZXJ2YWxfZGF5czogMzAKc3ludGhfYWxsb3dlZDogZmFsc2UKcHJpdmFjeV9sZXZlbDogMgpwcml2YWN5X2NvbHVtbjogaWRsCnNpemU6IDEwMDAKTFI6CiAgY3BnX3NlZ21lbnRzOgogICAgcm93X3ByaXZhY3k6IGZhbHNlCiAgICByb3dzOiAxMDAwCiAgICBtYXhfaWRzOiAxCiAgICBzYW1wbGVfbWF4X2lkczogdHJ1ZQogICAgY2Vuc29yX2RpbXM6IHRydWUKICAgIGNsYW1wX2NvdW50czogZmFsc2UKICAgIGNsYW1wX2NvbHVtbnM6IHRydWUKICAgIGlkbDoKICAgICAgcHJpdmF0ZV9pZDogdHJ1ZQogICAgICB0eXBlOiBzdHJpbmcKICAgIGFjdGl2ZV9jb21wbGFpbnQ6CiAgICAgIHR5cGU6IGludAogICAgbG95YWx0eV90aWVyczoKICAgICAgdHlwZTogc3RyaW5nCg=='], metapath=[], param=None, private_column='idl', query='foo', tau=None, tracker_group_uuid='38', tracker_output_path=None, tracker_run_uuid='e4378645-709d-432e-afef-241a4efcbb25', tracker_type='mlflow')\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | Traceback (most recent call last):\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/opt/miniconda3/envs/dq0/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | \"__main__\", mod_spec)\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/opt/miniconda3/envs/dq0/lib/python3.7/runpy.py\", line 85, in _run_code\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | exec(code, run_globals)\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/Projects/dq0-sql/dq0/sql/__main__.py\", line 141, in \n", - "2021-01-19T15:50:55Z | dq0 | ERROR | run()\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/Projects/dq0-sql/dq0/sql/__main__.py\", line 131, in run\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | ret = instance.run()\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/Projects/dq0-sql/dq0/sql/runner.py\", line 268, in run\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | ret = self.prepare_metadata()\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/Projects/dq0-sql/dq0/sql/runner.py\", line 117, in prepare_metadata\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | queries = QueryParser(sm_meta).queries(self.query)\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/Projects/dq0-sql/opendp/smartnoise/sql/parse.py\", line 31, in queries\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | queries = [q for q in bv.visit(parser.batch()).queries]\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/Projects/dq0-sql/opendp/smartnoise/sql/parser/SqlSmallParser.py\", line 608, in batch\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | self.query()\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/Projects/dq0-sql/opendp/smartnoise/sql/parser/SqlSmallParser.py\", line 703, in query\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | self.selectClause()\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/Projects/dq0-sql/opendp/smartnoise/sql/parser/SqlSmallParser.py\", line 953, in selectClause\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | self.match(SqlSmallParser.SELECT)\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/opt/miniconda3/envs/dq0/lib/python3.7/site-packages/antlr4/Parser.py\", line 121, in match\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | t = self._errHandler.recoverInline(self)\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/opt/miniconda3/envs/dq0/lib/python3.7/site-packages/antlr4/error/ErrorStrategy.py\", line 392, in recoverInline\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | if self.singleTokenInsertion(recognizer):\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/opt/miniconda3/envs/dq0/lib/python3.7/site-packages/antlr4/error/ErrorStrategy.py\", line 425, in singleTokenInsertion\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | self.reportMissingToken(recognizer)\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/opt/miniconda3/envs/dq0/lib/python3.7/site-packages/antlr4/error/ErrorStrategy.py\", line 333, in reportMissingToken\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | recognizer.notifyErrorListeners(msg, t, None)\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/opt/miniconda3/envs/dq0/lib/python3.7/site-packages/antlr4/Parser.py\", line 317, in notifyErrorListeners\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | listener.syntaxError(self, offendingToken, line, column, msg, e)\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/opt/miniconda3/envs/dq0/lib/python3.7/site-packages/antlr4/error/ErrorListener.py\", line 60, in syntaxError\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | delegate.syntaxError(recognizer, offendingSymbol, line, column, msg, e)\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | File \"/Users/dominic/Projects/dq0-sql/opendp/smartnoise/sql/parser/SqlSmallErrorListener.py\", line 10, in syntaxError\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | raise ValueError(\"Bad token {0} at line {1} column {2}. Message: {3}\".format(offendingToken.text, line, column, msg))\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | ValueError\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | :\n", - "2021-01-19T15:50:55Z | dq0 | ERROR | Bad token foo at line 1 column 0. Message: missing SELECT at 'foo'\n", - "\n" - ] - } - ], - "source": [ - "run2 = query.execute('foo')\n", + "outputs": [], + "source": [ + "run2 = query.execute('foo', args)\n", "run2.wait_for_completion()\n", "run2.get_error()" ] @@ -505,33 +313,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Waiting for job to complete...\n", - "running\n", - "running\n", - "finished\n", - "finished\n", - "Job completed\n", - "finished\n" - ] - }, - { - "data": { - "text/plain": [ - "'0x4e9b,tx_count,c_loyalty_tiers\\n2556,2548,silver\\n'" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "run3 = query.execute(stmt, epsilon=1.5, tau=100, private_column='idl')\n", "run3.wait_for_completion(verbose=True)\n", @@ -549,58 +333,11 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
0x4e9btx_countc_loyalty_tiers
025562548silver
\n", - "
" - ], - "text/plain": [ - " 0x4e9b tx_count c_loyalty_tiers\n", - "0 2556 2548 silver" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "from io import StringIO\n",