28-Ability to run each stage independently

.gitignore

@@ -148,3 +148,7 @@ transform_vacant_table.csv
 
 # sqlite test db
 vacancy.sqlite
+
+# config files for unit tests
+data/source/test_sources.yml
+data/transform_tasks/test_transform_tasks.yml

README.md

@@ -59,7 +59,44 @@ python3 ./app.py --db prod
 ```
 :warning: Example 3. will not work if you don't have the database admin credentials. For more details, [Go to Running with Production Database](#running-with-production-database).  
 
+#### Running individual stages
+To run individual stages (i.e. Fetcher only, Transformer only) without running the entire application, use the following commands:
+1. Run `Fetcher` only:
+Run with default `test_sources.yml`:
+```bash
+python3 tests/test_fetcher.py
+```
+To run with specific source YAML, run the following command replacing last argument with path to custom YAML:
+```bash
+python3 tests/test_fetcher.py ./data/sources/sources.yml
+```
+
+2. Run `Parser` only:
+Use --local-sources to specify local files to parse:
+```bash
+python3 tests/test_parser.py --local-sources ./src/prcl.mdb ./src/par.dbf ./src/prcl_shape.zip
+```
+
+3. Run `Extractor` only:
+```bash
+python3 tests/test_extractor.py --local-sources ./src/prcl.mdb ./src/par.dbf ./src/prcl_shape.zip
+```
 
+4. Run `Transformer` only:
+```bash
+python3 tests/test_transformer.py --local-sources src/BldgCom.csv src/BldgRes.csv src/par.dbf.csv src/prcl.shp.csv src/Prcl.csv
+```
+
+5. Run `Loader` only:
+```bash
+python3 tests/test_loader.py --local-sources ./src/vacant_table.csv
+```
+
+#### Running unit tests
+To run unit tests, run the following command from project root directory:
+```bash
+pytest
+```
 
 #### Deactivating Virtual Environment
 

app.py

@@ -3,83 +3,60 @@
 '''
 
 import os
-import logging
-from etl import command_line_args, extractor, fetcher, fetcher_local, loader, parser, transformer, utils
-
-CSV = '.csv'  # comma separated values
-DBF = '.dbf'  # dbase
-MDB = '.mdb'  # microsoft access database (jet, access, etc.)
-PRJ = '.prj'  # .shp support file
-SBN = '.sbn'  # .shp support file
-SBX = '.sbx'  # .shp support file
-SHP = '.shp'  # shapes
-SHX = '.shx'  # .shp support file
+import sys
+import logging.config
+from etl.constants import *
+from etl import command_line_args, utils
+from etl.fetcher import Fetcher
+from etl.fetcher_local import FetcherLocal
+from etl.extractor import Extractor
+from etl.parser import Parser
+from etl.transformer import Transformer
+from etl.loader import Loader
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
-SUPPORTED_FILE_EXT = [CSV, DBF, MDB, PRJ, SBN, SBX, SHP, SHX]
 
 if __name__ == '__main__':
+    # Parse Command line arguments
     commandLineArgs = command_line_args.getCommandLineArgs()
+    # Setup logging
+    logging.config.fileConfig('data/logger/config.ini')
+    logger = logging.getLogger(__name__)
 
-    # Load database config
     # notify user if the app will be using test or prod db
     if (commandLineArgs.db == 'prod'):
-        print('Using production database...')
+        logger.info('Using production database...')
         db_yaml = utils.get_yaml('data/database/config_prod.yml')
     else:
-        print('Using development database...')
+        logger.info('Using development database...')
         db_yaml = utils.get_yaml('data/database/config_dev.yml')
         # delete local db from previous run
         utils.silentremove(db_yaml['database_credentials']['db_name'])
 
     # Fetcher
     if (commandLineArgs.local_sources):
-        print('using local data files', commandLineArgs.local_sources)
-        fetcher = fetcher_local.FetcherLocal()
+        logger.info("Using local data files: {}".format(' '.join(map(str, commandLineArgs.local_sources))))
+        fetcher = FetcherLocal()
         filenames = commandLineArgs.local_sources
         responses = fetcher.fetch_all(filenames)
     else:
-        fetcher = fetcher.Fetcher()
+        fetcher = Fetcher()
         src_yaml = utils.get_yaml('data/sources/sources.yml')
         responses = fetcher.fetch_all(src_yaml)
 
     # Parser
-    parser = parser.Parser()
-    for response in responses:
-        try:
-            response.payload = parser.flatten(response, SUPPORTED_FILE_EXT)
-        except Exception as err:
-            print(err)
+    parser = Parser()
+    responses = parser.parse_all(responses)
 
     # Extractor
-    extractor = extractor.Extractor()
-    # Master entity list
-    entity_dict = dict()
-    entities = []
-    for response in responses:
-        for payload in response.payload:
-            if utils.get_file_ext(payload.filename) == CSV:
-                entities = extractor.get_csv_data(payload)
-            elif utils.get_file_ext(payload.filename) == MDB:
-                entities = extractor.get_mdb_data(payload)
-            elif utils.get_file_ext(payload.filename) == DBF:
-                entities = extractor.get_dbf_data(payload)
-            elif utils.get_file_ext(payload.filename) == SHP:
-                entities = extractor.get_shp_data(response, payload)
-            else:
-                entities = {}
-            # Add to master entity list
-            entity_dict.update(entities)
+    extractor = Extractor()
+    entity_dict = extractor.extract_all(responses)
 
     # Transformer
     transform_tasks = utils.get_yaml('data/transform_tasks/transform_tasks.yml')
-    transformer = transformer.Transformer()
+    transformer = Transformer()
     transformed_dict = transformer.transform_all(entity_dict, transform_tasks)
 
     # Loader
-    # read loader config
-    loader = loader.Loader(db_yaml)
-    # connect to database
-    loader.connect()
-    for tablename, transformed_df in transformed_dict.items():
-        loader.insert(tablename, transformed_df)
+    loader = Loader(db_yaml)
+    loader.load_all(transformed_dict)

data/logger/config.ini

@@ -0,0 +1,23 @@
+[loggers]
+keys=root
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+disable_existing_loggers=False
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[formatter_simpleFormatter]
+format=%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s
+datefmt=%Y-%m-%d:%H:%M:%S

data/sources/sources.yml

@@ -1,20 +1,27 @@
-prcl_shape:
+ESRI_Parcels_Shapefile:
+  info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=84
   url: https://www.stlouis-mo.gov/data/upload/data-files/prcl_shape.zip
 
-prcl:
+Parcels_Key:
+  info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=83
   url: https://www.stlouis-mo.gov/data/upload/data-files/prcl.zip
 
-par:
+Parcel_Data:
+  info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=85
   url: https://www.stlouis-mo.gov/data/upload/data-files/par.zip
 
-lra_public:
+LRA_Iventory_Records:
+  info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=65
   url: https://www.stlouis-mo.gov/data/upload/data-files/lra_public.zip
 
-bldginsp:
+Building_Inspections:
+  info: https://www.stlouis-mo.gov/data/datasets/dataset.cfm?id=11
   url: https://www.stlouis-mo.gov/data/upload/data-files/bldginsp.zip
 
-prmbdo:
+Building_Permits:
+  info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=3
   url: https://www.stlouis-mo.gov/data/upload/data-files/prmbdo.zip
 
-forestry_maintenance_properties:
+Forestry_Property_Maintenance_Data:
+  Info: https://www.stlouis-mo.gov/data/datasets/dataset.cfm?id=64
   url: https://www.stlouis-mo.gov/data/upload/data-files/forestry-maintenance-properties.csv

data/sources/test_sources.yml

@@ -0,0 +1,27 @@
+ESRI_Parcels_Shapefile:
+  info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=84
+  url: https://www.stlouis-mo.gov/data/upload/data-files/prcl_shape.zip
+
+Parcels_Key:
+  info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=83
+  url: https://www.stlouis-mo.gov/data/upload/data-files/prcl.zip
+
+# Parcel_Data:
+#   info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=85
+#   url: https://www.stlouis-mo.gov/data/upload/data-files/par.zip
+#
+# LRA_Iventory_Records:
+#   info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=65
+#   url: https://www.stlouis-mo.gov/data/upload/data-files/lra_public.zip
+#
+# Building_Inspections:
+#   info: https://www.stlouis-mo.gov/data/datasets/dataset.cfm?id=11
+#   url: https://www.stlouis-mo.gov/data/upload/data-files/bldginsp.zip
+#
+# Building_Permits:
+#   info: https://www.stlouis-mo.gov/data/datasets/distribution.cfm?id=3
+#   url: https://www.stlouis-mo.gov/data/upload/data-files/prmbdo.zip
+#
+# Forestry_Property_Maintenance_Data:
+#   Info: https://www.stlouis-mo.gov/data/datasets/dataset.cfm?id=64
+#   url: https://www.stlouis-mo.gov/data/upload/data-files/forestry-maintenance-properties.csv

data/transform_tasks/test_transform_tasks.yml

@@ -0,0 +1 @@
+- vacant_table

etl/command_line_args.py

@@ -1,8 +1,10 @@
 import argparse
 
-def getCommandLineArgs():
+def getCommandLineArgs(local_source=True, db=True):
     parser = argparse.ArgumentParser()
-    parser.add_argument('--db', nargs='?', type=str, choices=['dev','prod'], default='dev', help='dev: use local database; prod: use production database')
-    parser.add_argument('--local-sources', nargs='+', type=str, help='local data files to use in place of internet sources.')
+    if db:
+        parser.add_argument('--db', nargs='?', type=str, choices=['dev','prod'], default='dev', help='dev: use local database; prod: use production database')
+    if local_source:
+        parser.add_argument('--local-sources', nargs='+', type=str, help='local data files to use in place of internet sources.')
     args = parser.parse_args()
     return args

etl/constants.py

@@ -0,0 +1,12 @@
+
+# Declare global variables (callable across files)
+TOTAL_STAGES = 5
+CSV = '.csv'  # comma separated values
+DBF = '.dbf'  # dbase
+MDB = '.mdb'  # microsoft access database (jet, access, etc.)
+PRJ = '.prj'  # .shp support file
+SBN = '.sbn'  # .shp support file
+SBX = '.sbx'  # .shp support file
+SHP = '.shp'  # shapes
+SHX = '.shx'  # .shp support file
+SUPPORTED_FILE_EXT = [CSV, DBF, MDB, PRJ, SBN, SBX, SHP, SHX]