diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml new file mode 100644 index 000000000..74dd68dd9 --- /dev/null +++ b/.github/workflows/publish-docs.yml @@ -0,0 +1,47 @@ +# This workflow builds and publishes the latest docs to +# the `gh-pages` branch. +# For more details: https://github.com/marketplace/actions/deploy-to-github-pages +name: Publish docs + +on: + release: + types: [created] + workflow_dispatch: + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + permissions: + contents: write + pages: write + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v2 + with: + # fetch all tags so `versioneer` can properly determine current version + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.11' + - name: Install pandoc + uses: pandoc/actions/setup@v1 + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install -r requirements-ml.txt + pip install -r requirements-reports.txt + pip install -r requirements-docs.txt + pip install -e . + + - name: Build + run: | + cd _docs/docs + python update_documentation.py + - name: Publish + uses: JamesIves/github-pages-deploy-action@v4 + with: + branch: gh-pages + folder: _docs/docs/LATEST/html diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml new file mode 100644 index 000000000..9a230cd27 --- /dev/null +++ b/.github/workflows/publish-package.yml @@ -0,0 +1,52 @@ +# This workflow publishes the package to pypi. +# For more details: +# https://docs.github.com/en/actions/guides/building-and-testing-python#publishing-to-package-registries +name: Publish to PyPi + +on: + release: + types: [created] + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # fetch all tags so `versioneer` can properly determine current version + with: + fetch-depth: 0 + - name: Check if current commit is tagged + # fails and cancels release if the current commit is not tagged + run: | + git describe --exact-match --tags + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-ml.txt ]; then pip install -r requirements-ml.txt; fi + if [ -f requirements-reports.txt ]; then pip install -r requirements-reports.txt; fi + pip install setuptools wheel twine + - name: Build + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TWINE_REPOSITORY: pypi + run: | + python setup.py sdist bdist_wheel + - name: Test build + # fails and cancels release if the built package fails to import + run: | + pip install dist/*.whl + python -c 'import dataprofiler; print(dataprofiler.__version__)' + - name: Publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TWINE_REPOSITORY: pypi + run: | + twine upload dist/* diff --git a/.github/workflows/publish-python-package.yml b/.github/workflows/publish-python-package.yml deleted file mode 100644 index 75b9a41e2..000000000 --- a/.github/workflows/publish-python-package.yml +++ /dev/null @@ -1,38 +0,0 @@ - -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -name: Publish Python Package - -on: - release: - types: [created] - branches: - - 'release/*' - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f requirements-ml.txt ]; then pip install -r requirements-ml.txt; fi - if [ -f requirements-reports.txt ]; then pip install -r requirements-reports.txt; fi - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - TWINE_REPOSITORY: pypi - run: | - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/.github/workflows/test-python-package.yml b/.github/workflows/test-package.yml similarity index 79% rename from .github/workflows/test-python-package.yml rename to .github/workflows/test-package.yml index fa84b3d3a..752cb232e 100644 --- a/.github/workflows/test-python-package.yml +++ b/.github/workflows/test-package.yml @@ -7,8 +7,6 @@ on: pull_request: branches: - 'main' - - 'feature/**' - - 'dev' jobs: build: @@ -16,12 +14,12 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, "3.10"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v5.6.0 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -38,4 +36,4 @@ jobs: pre-commit run --all-files - name: Test with pytest run: | - DATAPROFILER_SEED=0 pytest --forked --cov=dataprofiler --cov-fail-under=80 + DATAPROFILER_SEED=0 pytest --cov=dataprofiler --cov-fail-under=80 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7baeb59ec..8de323b60 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,9 +2,10 @@ repos: # Black: format Python code # https://github.com/psf/black/blob/master/.pre-commit-hooks.yaml - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 24.3.0 hooks: - id: black + exclude: (versioneer.py|dataprofiler/_version.py|_docs/) types: [file, python] language_version: python3 # Isort: sort import statements @@ -15,15 +16,16 @@ repos: rev: 5.12.0 hooks: - id: isort + exclude: _docs/ language_version: python3 # Flake8: complexity and style checking # https://flake8.pycqa.org/en/latest/user/using-hooks.html - repo: https://github.com/pycqa/flake8 - rev: 4.0.1 + rev: 7.3.0 hooks: - id: flake8 additional_dependencies: [flake8-docstrings] - exclude: (^docs/|^dataprofiler/tests/|^.*/__init__.py) + exclude: (^docs/|^dataprofiler/tests/|^.*/__init__.py|_docs/) language_version: python3 # General fixers: format files for white spaces and trailing new lines, warn on debug statements # https://github.com/pre-commit/pre-commit-hooks#hooks-available @@ -31,46 +33,50 @@ repos: rev: v4.0.1 hooks: - id: trailing-whitespace - exclude: (^dataprofiler/tests/data/|^dataprofiler/tests/speed_tests/data/) + exclude: (^dataprofiler/tests/data/|^dataprofiler/tests/speed_tests/data/|_docs/) - id: debug-statements - id: end-of-file-fixer - exclude: (^dataprofiler/tests/data/) + exclude: (^dataprofiler/tests/data/|_docs/) # Mypy: Optional static type checking # https://github.com/pre-commit/mirrors-mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.982 + rev: v1.11.2 hooks: - id: mypy - exclude: (^dataprofiler/tests/|^resources/|^examples|venv*/) + # Let mypy own target selection and exclusions via setup.cfg. + args: [--config-file=setup.cfg, dataprofiler] + pass_filenames: false language_version: python3 additional_dependencies: # Keep up-to-date with the respective requirement files [ # requirements.txt h5py>=2.10.0, wheel>=0.33.1, - numpy>=1.22.0, - pandas>=1.1.2, + 'numpy>=1.22.0,<3.0.0', + 'pandas>=1.1.2,<3.0.0', python-dateutil>=2.7.5, pytz>=2020.1, - pyarrow>=1.0.1, - chardet>=3.0.4, - fastavro>=1.0.0.post1, - python-snappy>=0.5.4, + 'pyarrow>=1.0.1,<24.0.0', + 'chardet>=3.0.4,<7.0.0', + fastavro>=1.1.0, + python-snappy>=0.7.1, charset-normalizer>=1.3.6, psutil>=4.0.0, - scipy>=1.4.1, - requests>=2.28.1, + scipy>=1.10.0, + requests>=2.32.4, networkx>=2.5.1, typing-extensions>=3.10.0.2, HLL>=2.0.3, datasketches>=4.1.0, - boto3>=1.28.61, + packaging>=23.0, + boto3>=1.37.15, + urllib3>=2.5.0, # requirements-dev.txt - check-manifest>=0.48, - black==22.3.0, + check-manifest>=0.50, + black>=24.3.0, isort==5.12.0, - pre-commit==2.19.0, + pre-commit==4.3.0, tox==3.25.1, types-setuptools==67.7.0.1, types-python-dateutil==2.8.19.12, @@ -80,11 +86,9 @@ repos: # requirements-ml.txt scikit-learn>=0.23.2, - 'keras>=2.4.3,<3.0.0', + 'keras>3.4.0,<4.0.0', rapidfuzz>=2.6.1, - "tensorflow>=2.6.4,<2.15.0; sys.platform != 'darwin'", - "tensorflow>=2.6.4,<2.15.0; sys_platform == 'darwin' and platform_machine != 'arm64'", - "tensorflow-macos>=2.6.4,<2.15.0; sys_platform == 'darwin' and platform_machine == 'arm64'", + "tensorflow>=2.16.0", tqdm>=4.0.0, # requirements-reports.txt @@ -93,36 +97,39 @@ repos: # requirements-test.txt coverage>=5.0.1, - dask>=2.29.0, fsspec>=0.3.3, pytest>=6.0.1, pytest-cov>=2.8.1, pytest-xdist>=2.1.0, pytest-forked>=1.3.0, toolz>=0.10.0, - 'memray>=1.7.0,<1.12.0', + 'memray>=1.18.0', ] # Check-manifest: ensures required non-Python files are included in MANIFEST.in # https://github.com/mgedmin/check-manifest/blob/master/.pre-commit-hooks.yaml - repo: https://github.com/mgedmin/check-manifest - rev: "0.48" + rev: "0.50" hooks: - id: check-manifest - additional_dependencies: ['h5py', 'wheel', 'future', 'numpy', 'pandas', - 'python-dateutil', 'pytz', 'pyarrow', 'chardet', 'fastavro', - 'python-snappy', 'charset-normalizer', 'psutil', 'scipy', 'requests', - 'networkx','typing-extensions', 'HLL', 'datasketches', 'boto3'] + additional_dependencies: ['h5py', 'wheel', 'future', 'numpy>=1.22.0,<3.0.0', + 'pandas', 'python-dateutil', 'pytz', 'pyarrow<24.0.0', 'chardet', + 'fastavro>=1.1.0', 'python-snappy', 'charset-normalizer', 'psutil', + 'scipy>=1.10.0', 'requests>=2.32.4', 'networkx', 'typing-extensions', + 'HLL', 'datasketches', 'packaging>=23.0', 'boto3>=1.37.15', + 'urllib3>=2.5.0'] # Pyupgrade - standardize and modernize Python syntax for newer versions of the language - repo: https://github.com/asottile/pyupgrade rev: v3.3.0 hooks: - id: pyupgrade args: ["--py38-plus"] + exclude: (versioneer.py|dataprofiler/_version.py| _docs/) # Autoflake - cleanup unused variables and imports - repo: https://github.com/PyCQA/autoflake rev: v2.0.0 hooks: - id: autoflake + exclude: _docs/ args: - "--in-place" - "--ignore-pass-statements" diff --git a/.whitesource b/.whitesource new file mode 100644 index 000000000..37dfa8e25 --- /dev/null +++ b/.whitesource @@ -0,0 +1,3 @@ +{ + "settingsInheritedFrom": "capitalone/whitesource-config" +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..69954d2db --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,46 @@ +# Changelog + +## 0.14.0 - 2026-09-08 + +### Breaking Changes + +- Dropped support for Python 3.9. The minimum supported version is now Python + 3.10 (`python_requires>=3.10`). +- Raised the `keras` requirement to `>3.4.0,<4.0.0` (previously `<=3.4.0`) and + migrated the bundled labeler models from the TensorFlow SavedModel format to + the single-file `.keras` format. Environments pinned to `keras<=3.4.0` are no + longer supported, and previously saved models must be re-saved. +- Moved the bundled labeler resources from the top-level `resources/` directory + into the package at `dataprofiler/resources/`. Code that referenced the old + top-level path must be updated; use + `dataprofiler.labelers.utils.find_resources_dir()` to resolve resource paths. + +### Added + +- Added compatibility support for NumPy 2.0 while constraining `numpy` to + `>=1.22.0,<3.0.0` to avoid future breakage from NumPy 3. +- Added compatibility support for Keras versions newer than 3.4.0 while + constraining `keras` to `>3.4.0,<4.0.0` to avoid future breakage from Keras 4. +- Added Python 3.12 and 3.13 to the test matrix. +- Added `find_resources_dir()` for resolving packaged resources via + `importlib.resources`. + +### Changed + +- Replaced the deprecated `pkg_resources` usage with `importlib.resources`, and + switched packaging from `data_files` to `package_data`. +- Constrained `chardet` to `<7.0.0` and `pandas` to `<3.0.0` to avoid breakage + from upcoming major releases. +- Constrained `pyarrow` to `<24.0.0`. +- Simplified the `tensorflow` requirement to `tensorflow>=2.16.0`, removing the + platform-specific `tensorflow-macos` marker. +- Updated the pre-commit configuration to align hook versions and hook + dependencies with the current project requirements. +- Enabled `mypy` checking across the `dataprofiler` package (excluding tests). +- Removed `--forked` from the `tox` pytest invocation and raised `memray` to + `>=1.18.0`. + +### Security + +- Resolved reported vulnerabilities by raising `boto3>=1.37.15`, + `requests>=2.32.4`, and adding an explicit `urllib3>=2.5.0` floor. diff --git a/MANIFEST.in b/MANIFEST.in index 12480abd8..f5ba88190 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,8 @@ global-exclude .DS_Store +global-exclude */__pycache__/* include *.txt +include CHANGELOG.md include CODEOWNERS recursive-include dataprofiler *.avro recursive-include dataprofiler *.csv @@ -12,8 +14,22 @@ recursive-include dataprofiler *.parquet recursive-include dataprofiler *.py recursive-include dataprofiler *.txt -recursive-include resources *.json -recursive-include resources *.pb -recursive-include resources *.py +recursive-include dataprofiler/resources *.json +recursive-include dataprofiler/resources *.pb +recursive-include dataprofiler/resources *.py +recursive-include dataprofiler/resources *.keras -recursive-include dataprofiler/labelers/embeddings/ *.txt +recursive-include dataprofiler/labelers/embeddings *.txt +include versioneer.py +include dataprofiler/_version.py +include .whitesource + +recursive-exclude _docs *.html +recursive-exclude _docs *.cfg +exclude _docs/LICENSE +recursive-exclude _docs *.md +recursive-exclude _docs *.nojekyll +recursive-exclude _docs *.png +recursive-exclude _docs *.py +recursive-exclude _docs *.rst +recursive-exclude _docs Makefile diff --git a/_docs/README.md b/_docs/README.md new file mode 100644 index 000000000..0f925ac58 --- /dev/null +++ b/_docs/README.md @@ -0,0 +1,59 @@ +Visit our [documentation page.](https://capitalone.github.io/DataProfiler) + +### How to properly write documentation: + +#### Packages +In any package directory, overall package comments can be made in the +\_\_init\_\_.py of the directory. At the top of the \_\_init\_\_.py, +include your comments in between triple quotations. + +#### Classes +In any class file, include overall class comments at the top of the file +in between triple quotes and/or in the init function. + +#### Functions +reStructuredText Docstring Format is the standard. Here is an example: + + def format_data(self, predictions, verbose=False): + """ + Formats word level labeling of the Unstructured Data Labeler as you want + + :param predictions: A 2D list of word level predictions/labeling + :type predictions: Dict + :param verbose: A flag to determine verbosity + :type verbose: Bool + :return: JSON structure containing specified formatted output + :rtype: JSON + + :Example: + Look at this test. Don't forget the double colons to make a code block:: + This is a codeblock + Type example code here + """ + +### How to update the documentation: + + +1. Set up your local environment +```bash +# install sphinx requirements +# install the requirements from the feature branch +pip install pandoc && +pip install -r requirements.txt && +pip install -r requirements-ml.txt && +pip install -r requirements-reports.txt && +pip install -r requirements-docs.txt && +pip install -e . + +``` +2. And finally, from the root of `DataProfiler`, run the following commands to generate the sphinx documentation: +```bash +cd _docs/docs +python update_documentation.py + +``` + +3. View new docs +```bash +open index.html +``` diff --git a/_docs/docs/Makefile b/_docs/docs/Makefile new file mode 100644 index 000000000..81ca02cf5 --- /dev/null +++ b/_docs/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = buildcode + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/_docs/docs/make.bat b/_docs/docs/make.bat new file mode 100644 index 000000000..6247f7e23 --- /dev/null +++ b/_docs/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/_docs/docs/source/API.rst b/_docs/docs/source/API.rst new file mode 100644 index 000000000..fdbf2242b --- /dev/null +++ b/_docs/docs/source/API.rst @@ -0,0 +1,16 @@ +.. _API: + +API +*** + +The API is split into 4 main components: Profilers, Labelers, Data Readers, and +Validators. + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + dataprofiler.data_readers + dataprofiler.profilers + dataprofiler.labelers + dataprofiler.validators \ No newline at end of file diff --git a/_docs/docs/source/DL-Flowchart.png b/_docs/docs/source/DL-Flowchart.png new file mode 100644 index 000000000..696eeb5dc Binary files /dev/null and b/_docs/docs/source/DL-Flowchart.png differ diff --git a/_docs/docs/source/_static/custom.css b/_docs/docs/source/_static/custom.css new file mode 100644 index 000000000..8a7c7cb54 --- /dev/null +++ b/_docs/docs/source/_static/custom.css @@ -0,0 +1,50 @@ +/* + the ipython3 code blocks coming from the notebooks + were not getting the dark theme styles applied, so + manually overriding them +*/ +@media (prefers-color-scheme: dark) { + .highlight-ipython3 { + border: none !important; + border-radius: 2px !important; + background: #202020 !important; + color: #d0d0d0 !important; + } +} + +@media (prefers-color-scheme: dark) { + tr:nth-child(odd) { + background-color: #202020 !important; + } +} + +@media (prefers-color-scheme: dark) { + .dataframe { + color: white !important; + } +} + +.hidden { + display: none; +} + +.version { + text-align: right; + font-size: 24px; + margin-top: -47px; + margin-right: 3px; +} + +.sidebar-brand { + margin-bottom: -10px; + margin-top: 10px; +} + +/* unknown warning was showing, manually hiding */ +#Visualizing-Logged-Dataframes .admonition.warning { + display: none; +} + +div.output_area.stderr { + display: none; +} diff --git a/_docs/docs/source/_static/images/DataProfilerDarkLogoLong.png b/_docs/docs/source/_static/images/DataProfilerDarkLogoLong.png new file mode 100644 index 000000000..a339e0f6a Binary files /dev/null and b/_docs/docs/source/_static/images/DataProfilerDarkLogoLong.png differ diff --git a/_docs/docs/source/_static/images/DataProfilerLogoLightTheme.png b/_docs/docs/source/_static/images/DataProfilerLogoLightTheme.png new file mode 100644 index 000000000..35e59c349 Binary files /dev/null and b/_docs/docs/source/_static/images/DataProfilerLogoLightTheme.png differ diff --git a/_docs/docs/source/_static/images/DataProfilerLogoLightThemeLong.png b/_docs/docs/source/_static/images/DataProfilerLogoLightThemeLong.png new file mode 100644 index 000000000..ca86fe167 Binary files /dev/null and b/_docs/docs/source/_static/images/DataProfilerLogoLightThemeLong.png differ diff --git a/_docs/docs/source/_static/images/branching_workflow_diagram.png b/_docs/docs/source/_static/images/branching_workflow_diagram.png new file mode 100644 index 000000000..60a9515d0 Binary files /dev/null and b/_docs/docs/source/_static/images/branching_workflow_diagram.png differ diff --git a/_docs/docs/source/_static/images/histogram_example_0.png b/_docs/docs/source/_static/images/histogram_example_0.png new file mode 100644 index 000000000..9b8301363 Binary files /dev/null and b/_docs/docs/source/_static/images/histogram_example_0.png differ diff --git a/_docs/docs/source/_static/images/histogram_example_1.png b/_docs/docs/source/_static/images/histogram_example_1.png new file mode 100644 index 000000000..062dfdbb9 Binary files /dev/null and b/_docs/docs/source/_static/images/histogram_example_1.png differ diff --git a/_docs/docs/source/_static/images/histogram_example_2.png b/_docs/docs/source/_static/images/histogram_example_2.png new file mode 100644 index 000000000..1aedf7549 Binary files /dev/null and b/_docs/docs/source/_static/images/histogram_example_2.png differ diff --git a/_docs/docs/source/_static/images/missing_value_barchart_example_0.png b/_docs/docs/source/_static/images/missing_value_barchart_example_0.png new file mode 100644 index 000000000..33cb7afd2 Binary files /dev/null and b/_docs/docs/source/_static/images/missing_value_barchart_example_0.png differ diff --git a/_docs/docs/source/_static/images/missing_value_matrix_example_0.png b/_docs/docs/source/_static/images/missing_value_matrix_example_0.png new file mode 100644 index 000000000..21799cddf Binary files /dev/null and b/_docs/docs/source/_static/images/missing_value_matrix_example_0.png differ diff --git a/_docs/docs/source/add_new_model_to_data_labeler.nblink b/_docs/docs/source/add_new_model_to_data_labeler.nblink new file mode 100644 index 000000000..4c5fe646a --- /dev/null +++ b/_docs/docs/source/add_new_model_to_data_labeler.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/add_new_model_to_data_labeler.ipynb" +} \ No newline at end of file diff --git a/_docs/docs/source/architecture.rst b/_docs/docs/source/architecture.rst new file mode 100644 index 000000000..469308993 --- /dev/null +++ b/_docs/docs/source/architecture.rst @@ -0,0 +1,48 @@ +.. _architecture: + +Architecture & Design Overview +****************************** + +This section describes the design rationale, algorithmic choices, assumptions, testing strategy, and contribution process used in the DataProfiler library. + +Overview +-------- + +DataProfiler computes numeric statistics (e.g., mean, variance, skewness, kurtosis) using **streaming algorithms** that allow efficient, incremental updates without recomputing from raw data. Approximate quantile metrics like the median are calculated using histogram-based estimation, making the system scalable for large or streaming datasets. + +Additionally, DataProfiler uses a **Convolutional Neural Network (CNN)** to detect and label entities (e.g., names, emails, credit cards) in unstructured text. This supports critical tasks such as **PII detection**, **schema inference**, and **data quality analysis** across structured and unstructured data. + +Algorithm Rationale +------------------- + +The algorithms used are designed for **speed, scalability, and flexibility**: + +- **Streaming numeric methods** (e.g., Welford's algorithm, moment-based metrics, histogram binning) efficiently summarize data without full recomputation. +- **CNNs for entity detection** are fast, high-throughput, and well-suited for sequence labeling tasks in production environments. + +These choices align with the tool's goal of delivering fast, accurate data profiling with minimal configuration. + +Assumptions & Limitations +------------------------- + +- **Consistent formatting** of sensitive entities is assumed (e.g., standardized credit card or SSN formats). +- **Overlapping entity types** (e.g., phone vs. SSN) may lead to misclassification without context. +- **Synthetic training data** may not fully capture real-world diversity, reducing model accuracy on natural or unstructured text. +- **Quantile estimation** (e.g., median) is approximate and based on binning rather than exact sorting. + +Testing & Validation +-------------------- + +- Comprehensive **unit testing** is performed across Python 3.9, 3.10, and 3.11. +- Tests are executed on every pull request targeting `dev` or `main` branches. +- All pull requests require **two code reviewer approvals** before merging. +- Testing includes correctness, performance, and compatibility checks to ensure production readiness. + +Versioning & Contributions +-------------------------- + +- Versioning and development are managed via **GitHub**. +- Future changes must follow the guidelines in `CONTRIBUTING.md`, including: + - Forking the repo and branching from `dev` or an active feature branch. + - Ensuring **80%+ unit test coverage** for all new functionality. + - Opening a PR and securing **two approvals** prior to merging. diff --git a/_docs/docs/source/column_name_labeler_example.nblink b/_docs/docs/source/column_name_labeler_example.nblink new file mode 100644 index 000000000..c39e674fb --- /dev/null +++ b/_docs/docs/source/column_name_labeler_example.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/column_name_labeler.ipynb" +} \ No newline at end of file diff --git a/_docs/docs/source/conf.py b/_docs/docs/source/conf.py new file mode 100644 index 000000000..80168effd --- /dev/null +++ b/_docs/docs/source/conf.py @@ -0,0 +1,84 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +import re + +sys.path.insert(0, os.path.abspath(f'../../../')) + +# -- Project information ----------------------------------------------------- + +project = 'Data Profiler' +copyright = '2024, Jeremy Goodsitt, Austin Walters, Anh Truong, Grant Eden, and Chris Wallace' +author = 'Jeremy Goodsitt, Austin Walters, Anh Truong, Grant Eden, and Chris Wallace' + +# The full version, including alpha/beta/rc tags +# release = '21.01.20' +from dataprofiler import __version__ as version # noqa F401 + + +version_clip = re.search(r'\s*([\d.]+)', version).group(1) +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.intersphinx', + 'nbsphinx', + 'nbsphinx_link', +] + +# Don't execute the notebook cells when generating the documentation +# This can be configured on a per notebook basis as well +# See: https://nbsphinx.readthedocs.io/en/0.2.15/never-execute.html#Explicitly-Dis-/ +nbsphinx_execute = "never" +nbsphinx_prolog = """ +`View this notebook on GitHub `_ +""" + +autoclass_content = 'both' +autodoc_default_options = { + 'members': True, + 'member-order': 'bysource', + 'undoc-members': True, + 'exclude-members': '__weakref__', + 'inherited-members': True, +} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "furo" +html_title = f"
v{version_clip}
" +html_static_path = ["_static"] +html_css_files = ["custom.css"] +html_favicon = "_static/images/DataProfilerLogoLightTheme.png" +html_theme_options = { + "light_logo": "images/DataProfilerLogoLightThemeLong.png", + "dark_logo": "images/DataProfilerDarkLogoLong.png", +} diff --git a/_docs/docs/source/data_labeling.rst b/_docs/docs/source/data_labeling.rst new file mode 100644 index 000000000..db76fe791 --- /dev/null +++ b/_docs/docs/source/data_labeling.rst @@ -0,0 +1,365 @@ +.. _data_labeling: + +Labeler (Sensitive Data) +************************ + +In this library, the term *data labeling* refers to entity recognition. + +Builtin to the data profiler is a classifier which evaluates the complex data types of the dataset. +For structured data, it determines the complex data type of each column. When +running the data profile, it uses the default data labeling model builtin to the +library. However, the data labeler allows users to train their own data labeler +as well. + +*Data Labels* are determined per cell for structured data (column/row when +the *profiler* is used) or at the character level for unstructured data. This +is a list of the default labels. + +* UNKNOWN +* ADDRESS +* BAN (bank account number, 10-18 digits) +* CREDIT_CARD +* EMAIL_ADDRESS +* UUID +* HASH_OR_KEY (md5, sha1, sha256, random hash, etc.) +* IPV4 +* IPV6 +* MAC_ADDRESS +* PERSON +* PHONE_NUMBER +* SSN +* URL +* US_STATE +* DRIVERS_LICENSE +* DATE +* TIME +* DATETIME +* INTEGER +* FLOAT +* QUANTITY +* ORDINAL + + +Identify Entities in Structured Data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Makes predictions and identifying labels: + +.. code-block:: python + + import dataprofiler as dp + + # load data and data labeler + data = dp.Data("your_data.csv") + data_labeler = dp.DataLabeler(labeler_type='structured') + + # make predictions and get labels per cell + predictions = data_labeler.predict(data) + +Identify Entities in Unstructured Data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Predict which class characters belong to in unstructured text: + +.. code-block:: python + + import dataprofiler as dp + + data_labeler = dp.DataLabeler(labeler_type='unstructured') + + # Example sample string, must be in an array (multiple arrays can be passed) + sample = ["Help\tJohn Macklemore\tneeds\tfood.\tPlease\tCall\t555-301-1234." + "\tHis\tssn\tis\tnot\t334-97-1234. I'm a BAN: 000043219499392912.\n"] + + # Prediction what class each character belongs to + model_predictions = data_labeler.predict( + sample, predict_options=dict(show_confidences=True)) + + # Predictions / confidences are at the character level + final_results = model_predictions["pred"] + final_confidences = model_predictions["conf"] + +It's also possible to change output formats, output similar to a **SpaCy** format: + +.. code-block:: python + + import dataprofiler as dp + + data_labeler = dp.DataLabeler(labeler_type='unstructured', trainable=True) + + # Example sample string, must be in an array (multiple arrays can be passed) + sample = ["Help\tJohn Macklemore\tneeds\tfood.\tPlease\tCall\t555-301-1234." + "\tHis\tssn\tis\tnot\t334-97-1234. I'm a BAN: 000043219499392912.\n"] + + # Set the output to the NER format (start position, end position, label) + data_labeler.set_params( + { 'postprocessor': { 'output_format':'ner', 'use_word_level_argmax':True } } + ) + + results = data_labeler.predict(sample) + + print(results) + +Train a New Data Labeler +~~~~~~~~~~~~~~~~~~~~~~~~ + +Mechanism for training your own data labeler on their own set of structured data +(tabular): + +.. code-block:: python + + import dataprofiler as dp + + # Will need one column with a default label of UNKNOWN + data = dp.Data("your_file.csv") + + data_labeler = dp.train_structured_labeler( + data=data, + save_dirpath="/path/to/save/labeler", + epochs=2 + ) + + data_labeler.save_to_disk("my/save/path") # Saves the data labeler for reuse + +Load an Existing Data Labeler +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Mechanism for loading an existing data_labeler: + +.. code-block:: python + + import dataprofiler as dp + + data_labeler = dp.DataLabeler( + labeler_type='structured', dirpath="/path/to/my/labeler") + + # get information about the parameters/inputs/output formats for the DataLabeler + data_labeler.help() + +Extending a Data Labeler with Transfer Learning +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Extending or changing labels of a data labeler w/ transfer learning: +Note: By default, **a labeler loaded will not be trainable**. In order to load a +trainable DataLabeler, the user must set `trainable=True` or load a labeler +using the `TrainableDataLabeler` class. + +The following illustrates how to change the labels: + +.. code-block:: python + + import dataprofiler as dp + + labels = ['label1', 'label2', ...] # new label set can also be an encoding dict + data = dp.Data("your_file.csv") # contains data with new labels + + # load default structured Data Labeler w/ trainable set to True + data_labeler = dp.DataLabeler(labeler_type='structured', trainable=True) + + # this will use transfer learning to retrain the data labeler on your new + # dataset and labels. + # NOTE: data must be in an acceptable format for the preprocessor to interpret. + # please refer to the preprocessor/model for the expected data format. + # Currently, the DataLabeler cannot take in Tabular data, but requires + # data to be ingested with two columns [X, y] where X is the samples and + # y is the labels. + model_results = data_labeler.fit(x=data['samples'], y=data['labels'], + validation_split=0.2, epochs=2, labels=labels) + + # final_results, final_confidences are a list of results for each epoch + epoch_id = 0 + final_results = model_results[epoch_id]["pred"] + final_confidences = model_results[epoch_id]["conf"] + +The following illustrates how to extend the labels: + +.. code-block:: python + + import dataprofiler as dp + + new_labels = ['label1', 'label2', ...] + data = dp.Data("your_file.csv") # contains data with new labels + + # load default structured Data Labeler w/ trainable set to True + data_labeler = dp.DataLabeler(labeler_type='structured', trainable=True) + + # this will maintain current labels and model weights, but extend the model's + # labels + for label in new_labels: + data_labeler.add_label(label) + + # NOTE: a user can also add a label which maps to the same index as an existing + # label + # data_labeler.add_label(label, same_as='') + + # For a trainable model, the user must then train the model to be able to + # continue using the labeler since the model's graph has likely changed + # NOTE: data must be in an acceptable format for the preprocessor to interpret. + # please refer to the preprocessor/model for the expected data format. + # Currently, the DataLabeler cannot take in Tabular data, but requires + # data to be ingested with two columns [X, y] where X is the samples and + # y is the labels. + model_results = data_labeler.fit(x=data['samples'], y=data['labels'], + validation_split=0.2, epochs=2) + + # final_results, final_confidences are a list of results for each epoch + epoch_id = 0 + final_results = model_results[epoch_id]["pred"] + final_confidences = model_results[epoch_id]["conf"] + + +Changing pipeline parameters: + +.. code-block:: python + + import dataprofiler as dp + + # load default Data Labeler + data_labeler = dp.DataLabeler(labeler_type='structured') + + # change parameters of specific component + data_labeler.preprocessor.set_params({'param1': 'value1'}) + + # change multiple simultaneously. + data_labeler.set_params({ + 'preprocessor': {'param1': 'value1'}, + 'model': {'param2': 'value2'}, + 'postprocessor': {'param3': 'value3'} + }) + + +Build Your Own Data Labeler +=========================== + +The DataLabeler has 3 main components: preprocessor, model, and postprocessor. +To create your own DataLabeler, each one would have to be created or an +existing component can be reused. + +Given a set of the 3 components, you can construct your own DataLabeler: + +.. code-block:: python + from dataprofiler.labelers.base_data_labeler import BaseDataLabeler, \ + TrainableDataLabeler + from dataprofiler.labelers.character_level_cnn_model import CharacterLevelCnnModel + from dataprofiler.labelers.data_processing import \ + StructCharPreprocessor, StructCharPostprocessor + + # load a non-trainable data labeler + model = CharacterLevelCnnModel(...) + preprocessor = StructCharPreprocessor(...) + postprocessor = StructCharPostprocessor(...) + + data_labeler = BaseDataLabeler.load_with_components( + preprocessor=preprocessor, model=model, postprocessor=postprocessor) + + # check for basic compatibility between the processors and the model + data_labeler.check_pipeline() + + + # load trainable data labeler + data_labeler = TrainableDataLabeler.load_with_components( + preprocessor=preprocessor, model=model, postprocessor=postprocessor) + + # check for basic compatibility between the processors and the model + data_labeler.check_pipeline() + +Option for swapping out specific components of an existing labeler. + +.. code-block:: python + + import dataprofiler as dp + from dataprofiler.labelers.character_level_cnn_model import \ + CharacterLevelCnnModel + from dataprofiler.labelers.data_processing import \ + StructCharPreprocessor, StructCharPostprocessor + + model = CharacterLevelCnnModel(...) + preprocessor = StructCharPreprocessor(...) + postprocessor = StructCharPostprocessor(...) + + data_labeler = dp.DataLabeler(labeler_type='structured') + data_labeler.set_preprocessor(preprocessor) + data_labeler.set_model(model) + data_labeler.set_postprocessor(postprocessor) + + # check for basic compatibility between the processors and the model + data_labeler.check_pipeline() + + +Model Component +~~~~~~~~~~~~~~~ + +In order to create your own model component for data labeling, you can utilize +the `BaseModel` class from `dataprofiler.labelers.base_model` and +overriding the abstract class methods. + +Reviewing `CharacterLevelCnnModel` from +`dataprofiler.labelers.character_level_cnn_model` illustrates the functions +which need an override. + +#. `__init__`: specifying default parameters and calling base `__init__` +#. `_validate_parameters`: validating parameters given by user during setting +#. `_need_to_reconstruct_model`: flag for when to reconstruct a model (i.e. + parameters change or labels change require a model reconstruction) +#. `_construct_model`: initial construction of the model given the parameters +#. `_reconstruct_model`: updates model architecture for new label set while + maintaining current model weights +#. `fit`: mechanism for the model to learn given training data +#. `predict`: mechanism for model to make predictions on data +#. `details`: prints a summary of the model construction +#. `save_to_disk`: saves model and model parameters to disk +#. `load_from_disk`: loads model given a path on disk + + +Preprocessor Component +~~~~~~~~~~~~~~~~~~~~~~ + +In order to create your own preprocessor component for data labeling, you can +utilize the `BaseDataPreprocessor` class +from `dataprofiler.labelers.data_processing` and override the abstract class +methods. + +Reviewing `StructCharPreprocessor` from +`dataprofiler.labelers.data_processing` illustrates the functions which +need an override. + +#. `__init__`: passing parameters to the base class and executing any + extraneous calculations to be saved as parameters +#. `_validate_parameters`: validating parameters given by user during + setting +#. `process`: takes in the user data and converts it into an digestible, + iterable format for the model +#. `set_params` (optional): if a parameter requires processing before setting, + a user can override this function to assist with setting the parameter +#. `_save_processor` (optional): if a parameter is not JSON serializable, a + user can override this function to assist in saving the processor and its + parameters +#. `load_from_disk` (optional): if a parameter(s) is not JSON serializable, a + user can override this function to assist in loading the processor + +Postprocessor Component +~~~~~~~~~~~~~~~~~~~~~~~ + +The postprocessor is nearly identical to the preprocessor except it handles +the output of the model for processing. In order to create your own +postprocessor component for data labeling, you can utilize the +`BaseDataPostprocessor` class from `dataprofiler.labelers.data_processing` +and override the abstract class methods. + +Reviewing `StructCharPostprocessor` from +`dataprofiler.labelers.data_processing` illustrates the functions which +need an override. + +#. `__init__`: passing parameters to the base class and executing any + extraneous calculations to be saved as parameters +#. `_validate_parameters`: validating parameters given by user during + setting +#. `process`: takes in the output of the model and processes for output to + the user +#. `set_params` (optional): if a parameter requires processing before setting, + a user can override this function to assist with setting the parameter +#. `_save_processor` (optional): if a parameter is not JSON serializable, a + user can override this function to assist in saving the processor and its + parameters +#. `load_from_disk` (optional): if a parameter(s) is not JSON serializable, a + user can override this function to assist in loading the processor diff --git a/_docs/docs/source/data_reader.nblink b/_docs/docs/source/data_reader.nblink new file mode 100644 index 000000000..8d7215f46 --- /dev/null +++ b/_docs/docs/source/data_reader.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/data_readers.ipynb" +} \ No newline at end of file diff --git a/_docs/docs/source/data_readers.rst b/_docs/docs/source/data_readers.rst new file mode 100644 index 000000000..877ea56dd --- /dev/null +++ b/_docs/docs/source/data_readers.rst @@ -0,0 +1,184 @@ +.. _data_readers: + +Data Readers +************ + +The `Data` class itself will identify then output one of the following `Data` class types. +Using the data reader is easy, just pass it through the Data object. + +.. code-block:: python + + import dataprofiler as dp + data = dp.Data("your_file.csv") + +The supported file types are: + +* CSV file (or any delimited file) +* JSON object +* Avro file +* Parquet file +* Graph data file +* Text file +* Pandas DataFrame +* A URL that points to one of the supported file types above + +It's also possible to specifically call one of the data classes such as the following command: + +.. code-block:: python + + from dataprofiler.data_readers.csv_data import CSVData + data = CSVData("your_file.csv", options={"delimiter": ","}) + +Additionally any of the data classes can be loaded using a URL: + +.. code-block:: python + + import dataprofiler as dp + data = dp.Data("https://you_website.com/your_file.file", options={"verify_ssl": "True"}) + +Below are descriptions of the various `Data` classes and the available options. + +CSVData +======= + +Data class for loading datasets of type CSV. Can be specified by passing +in memory data or via a file path. Options pertaining the CSV may also +be specified using the options dict parameter. + +`CSVData(input_file_path=None, data=None, options=None)` + +Possible `options`: + +* delimiter - Must be a string, for example `"delimiter": ","` +* data_format - Must be a string, possible choices: "dataframe", "records" +* selected_columns - Columns being selected from the entire dataset, must be a + list `["column 1", "ssn"]` +* sample_nrows - Reservoir sampling to sample `"n"` rows out of a total of `"M"` rows. + Specified for how many rows to sample, default None. +* header - Define the header, for example + + * `"header": 'auto'` for auto detection + * `"header": None` for no header + * `"header": ` to specify the header row (0 based index) + +JSONData +======== + +Data class for loading datasets of type JSON. Can be specified by +passing in memory data or via a file path. Options pertaining the JSON +may also be specified using the options dict parameter. JSON data can be +accessed via the "data" property, the "metadata" property, and the +"data_and_metadata" property. + +`JSONData(input_file_path=None, data=None, options=None)` + +Possible `options`: + +* data_format - must be a string, choices: "dataframe", "records", "json", "flattened_dataframe" + + * "flattened_dataframe" is best used for JSON structure typically found in data streams that contain + nested lists of dictionaries and a payload. For example: `{"data": [ columns ], "response": 200}` +* selected_keys - columns being selected from the entire dataset, must be a list `["column 1", "ssn"]` +* payload_keys - The dictionary keys for the payload of the JSON, typically called "data" + or "payload". Defaults to ["data", "payload", "response"]. + + +AVROData +======== + +Data class for loading datasets of type AVRO. Can be specified by +passing in memory data or via a file path. Options pertaining the AVRO +may also be specified using the options dict parameter. + +`AVROData(input_file_path=None, data=None, options=None)` + +Possible `options`: + +* data_format - must be a string, choices: "dataframe", "records", "avro", "json", "flattened_dataframe" + + * "flattened_dataframe" is best used for AVROs with a JSON structure typically found in data streams that contain + nested lists of dictionaries and a payload. For example: `{"data": [ columns ], "response": 200}` +* selected_keys - columns being selected from the entire dataset, must be a list `["column 1", "ssn"]` + +ParquetData +=========== + +Data class for loading datasets of type PARQUET. Can be specified by +passing in memory data or via a file path. Options pertaining the +PARQUET may also be specified using the options dict parameter. + +`ParquetData(input_file_path=None, data=None, options=None)` + +Possible `options`: + +* data_format - must be a string, choices: "dataframe", "records", "json" +* selected_keys - columns being selected from the entire dataset, must be a list `["column 1", "ssn"]` +* sample_nrows - Random sampling to sample `"n"` rows out of a total of `"M"` rows. + Specified for how many rows to sample, default None. + +GraphData +========= + +Data Class for loading datasets of graph data. Currently takes CSV format, +further type formats will be supported. Can be specified by passing +in memory data (NetworkX Graph) or via a file path. Options pertaining the CSV file may also +be specified using the options dict parameter. Loads data from CSV into memory +as a NetworkX Graph. + +`GraphData(input_file_path=None, data=None, options=None)` + +Possible `options`: + +* delimiter - must be a string, for example `"delimiter": ","` +* data_format - must be a string, possible choices: "graph", "dataframe", "records" +* header - Define the header, for example + + * `"header": 'auto'` for auto detection + * `"header": None` for no header + * `"header": ` to specify the header row (0 based index) + +TextData +======== + +Data class for loading datasets of type TEXT. Can be specified by +passing in memory data or via a file path. Options pertaining the TEXT +may also be specified using the options dict parameter. + +`TextData(input_file_path=None, data=None, options=None)` + +Possible `options`: + +* data_format: user selected format in which to return data. Currently only supports "text". +* samples_per_line - chunks by which to read in the specified dataset + + +Data Using a URL +================ + +Data class for loading datasets of any type using a URL. Specified by passing in +any valid URL that points to one of the valid data types. Options pertaining the +URL may also be specified using the options dict parameter. + +`Data(input_file_path=None, data=None, options=None)` + +Possible `options`: + +* verify_ssl: must be a boolean string, choices: "True", "False". Set to "True" by default. + +Data Using an AWS S3 URI +======================== + +Data class for loading datasets from AWS S3 URI. Specified by passing in +any valid bucket path that points to one of the valid data types. + +`Data('s3a://my-bucket/file_name.txt')` + +Possible `options`: + +* `storage_options`: must be a dictionary where the keys for boto3 initialization are set + If `storage_options` is provided in `options`, the below variables are retrieved from the dictionary provided. Otherwise, will retrieve from `environment variables `_. + + * `AWS_ACCESS_KEY_ID` + * `AWS_SECRET_ACCESS_KEY` + * `AWS_SESSION_TOKEN` + * `AWS_REGION` (default `us-east-1`) diff --git a/_docs/docs/source/examples.rst b/_docs/docs/source/examples.rst new file mode 100644 index 000000000..3637da6ac --- /dev/null +++ b/_docs/docs/source/examples.rst @@ -0,0 +1,24 @@ +.. _examples: + +Examples +******** + +These examples provide a more in-depth look into the details of the ``Data Profiler`` library. + +Basics +------ + +.. toctree:: + :maxdepth: 0 + + Overview of Data Profiler + Data Reader + Structured Profiler + Unstructured Profiler + Graph Profiler + Labeler + Adding Models to a Labeler Pipeline + Creating a Regex Labeler + Creating a ColumnName Labeler + Merge Profile List + Dataloader with Popmon Reports diff --git a/_docs/docs/source/graph_data_demo.nblink b/_docs/docs/source/graph_data_demo.nblink new file mode 100644 index 000000000..40408c3ae --- /dev/null +++ b/_docs/docs/source/graph_data_demo.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/graph_data_demo.ipynb" +} diff --git a/_docs/docs/source/graphs.rst b/_docs/docs/source/graphs.rst new file mode 100644 index 000000000..23c2d316b --- /dev/null +++ b/_docs/docs/source/graphs.rst @@ -0,0 +1,196 @@ +.. _reports: + +Graphs +****** + +Graph Your Data +=============== + +We can plot some of our data as seaborn histogram plots. Below will demonstrate how to do so and provide examples. + +The following plots are currently available to work directly with your profilers: + + * histogram (numeric columns only) + * missing values matrix + +Below shows how to do so with examples. + +What we need to import +~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: python + + from dataprofiler.reports import graphs + +The main functions that is used to plot histograms are in graphs. **You will also need the `dataprofiler[reports]` requirement to be installed**: + +.. code-block:: console + + pip install 'dataprofiler[reports]' + +Plotting from a StructuredProfiler class +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +With a StructuredProfiler class variable, we can specify what columns we want to be plotted, and plot them into histograms. + +.. code-block:: python + + graphs.plot_histograms(profiler, column_names, column_inds) + +These are what the variables mean: + + * **profiler** - StructuredProfiler class variable that contains the data we want + * **columns** - (Optional) The list of IntColumn or FloatColumn *names* we want to specifically plot. If specified, `column_inds` cannot be specified. + * **column_inds** - (Optional) The list of IntColumn or FloatColumn *indexes* we want to specifically plot. If specified, `column_names` cannot be specified. + + +Additionally, we can also plot the missing values matrix for a StructuredProfiler: + +.. code-block:: python + + graphs.plot_missing_values_matrix(profiler, ax, title) + +These are what the variables mean: + + * **profiler** - StructuredProfiler class variable that contains the data we want + * **ax** - (Optional) MatPlotLib Axes to plot the matrix within. + * **title** - (Optional) The title of the axes we want to define. + + +Plotting an individual IntColumn or FloatColumn +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +With a column's Int or Float profile, we can plot their respective histograms. + +.. code-block:: python + + graphs.plot_col_histogram(column, axes, title) + +These are what the variables mean: + + * **column** - The IntColumn or FloatColumn we want to plot + * **axes** - (Optional) The MatPlotLib Axes to plot the histogram within. + * **title** - (Optional) The title of the axes we want to define. + + +Additionally, we can also plot the missing values bargraph for any column profile: + +.. code-block:: python + + graphs.plot_col_missing_values(profiler, ax, title) + +These are what the variables mean: + + * **profiler** - The StructuredColProfiler we want to plot + * **ax** - (Optional) MatPlotLib Axes to plot the matrix within. + * **title** - (Optional) The title of the axes we want to define. + +Examples +~~~~~~~~ + +Histograms +---------- + +1. This example demonstrates how we can take a StructuredProfiler class and plot histograms of the specified columns. + +.. code-block:: python + + import dataprofiler as dp + from dataprofiler.reports import graphs + + + data = [[1, 'a', 1.0], + [2, 'b', 2.2], + [3, 'c', 3.5], + [None, 'd', 10.0]] + profiler = dp.StructuredProfiler(data) + + # This will plot all IntColumn and FloatColumn as histograms (The first and last column). + fig = graphs.plot_histograms(profiler) + fig.show() + + # This will only plot the specified column, 0. + columns_names = [0] + fig = graphs.plot_histograms(profiler, columns_names) + fig.show() + +.. image:: _static/images/histogram_example_0.png + :alt: First Histogram Example Image + +.. image:: _static/images/histogram_example_1.png + :alt: Second Histogram Example Image + +2. This example demonstrates how we can plot a low level profiler. + +.. code-block:: python + + import pandas as pd + + from dataprofiler.profilers import IntColumn + from dataprofiler.reports import graphs + + + data = pd.Series([1, 2, 3], dtype=str) + profiler = IntColumn('example') + profiler.update(data) + + # Plot the axes + ax = graphs.plot_col_histogram(profiler) + + # get and show the figure of the plotted histogram + fig = ax.get_figure() + fig.show() + +.. image:: _static/images/histogram_example_2.png + :alt: Histogram Column Only Example Image + + +Missing Values Matrix +--------------------- + +1. This example demonstrates how we can take a StructuredProfiler class and plot a missing values matrix. + +.. code-block:: python + + import dataprofiler as dp + from dataprofiler.reports import graphs + + + data = pd.DataFrame( + [[None, '', 1.0, '1/2/2021'], + [3, None, 3.5, ''], + [1, None, 1.0, '2/5/2020'], + [None, 1, 10.0, '3/5/2020']], + columns=['integer', 'str', 'float', 'datetime'], + dtype=object + ) + profiler = dp.StructuredProfiler(data) + + # This will plot the missing values matrix for all columns. + fig = graphs.plot_missing_values_matrix(profiler) + fig.show() + +.. image:: _static/images/missing_value_matrix_example_0.png + :alt: Missing Values Matrix Example Image + +2. This example demonstrates how we can plot barchart of a column's missing values. + +.. code-block:: python + + import pandas as pd + + from dataprofiler.profilers.profile_builder import StructuredColProfiler + from dataprofiler.reports import graphs + + + data = pd.Series([1, 2, 3, None, None, 4], name='example', dtype=str) + profiler = StructuredColProfiler(data) + + # Plot the axes, can be a list of multiple columns + ax = graphs.plot_col_missing_values([profiler]) + + # get and show the figure of the plotted histogram + fig = ax.get_figure() + fig.show() + +.. image:: _static/images/missing_value_barchart_example_0.png + :alt: Missing Values Column Only Example Image \ No newline at end of file diff --git a/_docs/docs/source/index.rst b/_docs/docs/source/index.rst new file mode 100644 index 000000000..8225be28f --- /dev/null +++ b/_docs/docs/source/index.rst @@ -0,0 +1,479 @@ +.. _Data Profiler: + +==================================== +Data Profiler | What's in your data? +==================================== + +Purpose +======= + +The DataProfiler is a Python library designed to make data analysis, monitoring and **sensitive data detection** easy. + +Loading **Data** with a single command, the library automatically formats & loads files into a DataFrame. **Profiling** the Data, the library identifies the schema, statistics, entities and more. Data Profiles can then be used in downstream applications or reports. + +The Data Profiler comes with a cutting edge pre-trained deep learning model, used to efficiently identify **sensitive data** (or **PII**). If customization is needed, it's easy to add new entities to the existing pre-trained model or insert a new pipeline for entity recognition. + +The best part? Getting started only takes a few lines of code (`Example CSV`_): + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + data = Data("your_file.csv") # Auto-Detect & Load: CSV, AVRO, Parquet, JSON, Text + print(data.data.head(5)) # Access data directly via a compatible Pandas DataFrame + + profile = Profiler(data) # Calculate Statistics, Entity Recognition, etc + readable_report = profile.report(report_options={"output_format":"pretty"}) + print(json.dumps(readable_report, indent=4)) + + +To install the full package from pypi: + +.. code-block:: console + + pip install DataProfiler[ml] + +If the ML requirements are too strict (say, you don't want to install tensorflow), you can install a slimmer package. The slimmer package disables the default sensitive data detection / entity recognition (labler) + +Install from pypi: + +.. code-block:: console + + pip install DataProfiler + +If you have suggestions or find a bug, please open an `issue`_. + +Visit the :ref:`API` to explore Data Profiler's terminology. + + +What is a Data Profile? +======================= + +In the case of this library, a data profile is a dictionary containing statistics and predictions about the underlying dataset. There are "global statistics" or `global_stats`, which contain dataset level data and there are "column/row level statistics" or `data_stats` (each column is a new key-value entry). + +The format for a structured profile is below: + +.. code-block:: python + + "global_stats": { + "samples_used": int, + "column_count": int, + "row_count": int, + "row_has_null_ratio": float, + "row_is_null_ratio": float, + "unique_row_ratio": float, + "duplicate_row_count": int, + "file_type": string, + "encoding": string, + "correlation_matrix": list[list[int]], (*) + "chi2_matrix": list[list[float]], + "profile_schema": dict[string, list[int]] + }, + "data_stats": [ + { + "column_name": string, + "data_type": string, + "data_label": string, + "categorical": bool, + "order": string, + "samples": list[str], + "statistics": { + "sample_size": int, + "null_count": int, + "null_types": list[string], + "null_types_index": dict[string, list[int]], + "data_type_representation": dict[string, list[string]], + "min": [null, float], + "max": [null, float], + "sum": float, + "mode": list[float], + "median": float, + "median_absolute_deviation": float, + "mean": float, + "variance": float, + "stddev": float, + "skewness": float, + "kurtosis": float, + "num_zeros": int, + "num_negatives": int, + "histogram": { + "bin_counts": list[int], + "bin_edges": list[float], + }, + "quantiles": { + int: float + }, + "vocab": list[char], + "avg_predictions": dict[string, float], + "data_label_representation": dict[string, float], + "categories": list[str], + "unique_count": int, + "unique_ratio": float, + "categorical_count": dict[string, int], + "gini_impurity": float, + "unalikeability": float, + "precision": { + 'min': int, + 'max': int, + 'mean': float, + 'var': float, + 'std': float, + 'sample_size': int, + 'margin_of_error': float, + 'confidence_level': float + }, + "times": dict[string, float], + "format": string + }, + "null_replication_metrics": { + "class_prior": list[int], + "class_sum": list[list[int]], + "class_mean": list[list[int]] + } + } + ] + +(*) Currently the correlation matrix update is toggled off. It will be reset in a later update. Users can still use it as desired with the is_enable option set to True. + +The format for an unstructured profile is below: + +.. code-block:: python + + "global_stats": { + "samples_used": int, + "empty_line_count": int, + "file_type": string, + "encoding": string, + "memory_size": float, # in MB + }, + "data_stats": { + "data_label": { + "entity_counts": { + "word_level": dict[string, int], + "true_char_level": dict[string, int], + "postprocess_char_level": dict[string, int] + }, + "entity_percentages": { + "word_level": dict[string, float], + "true_char_level": dict[string, float], + "postprocess_char_level": dict[string, float] + }, + "times": dict[string, float] + }, + "statistics": { + "vocab": list[char], + "vocab_count": dict[string, int], + "words": list[string], + "word_count": dict[string, int], + "times": dict[string, float] + } + } + +The format for a graph profile is below: + +.. code-block:: python + + "num_nodes": int, + "num_edges": int, + "categorical_attributes": list[string], + "continuous_attributes": list[string], + "avg_node_degree": float, + "global_max_component_size": int, + "continuous_distribution": { + "": { + "name": string, + "scale": float, + "properties": list[float, np.array] + }, + "": None, + }, + "categorical_distribution": { + "": None, + "": { + "bin_counts": list[int], + "bin_edges": list[float] + }, + }, + "times": dict[string, float] + +Supported Data Formats +~~~~~~~~~~~~~~~~~~~~~~ + +* Any delimited file (CSV, TSV, etc.) +* JSON object +* Avro file +* Parquet file +* Text file +* Pandas DataFrame +* A URL that points to one of the supported file types above + + +Data Labels +~~~~~~~~~~~ + +*Data Labels* are determined per cell for structured data (column/row when the *profiler* is used) or at the character level for unstructured data. + +* UNKNOWN +* ADDRESS +* BAN (bank account number, 10-18 digits) +* CREDIT_CARD +* EMAIL_ADDRESS +* UUID +* HASH_OR_KEY (md5, sha1, sha256, random hash, etc.) +* IPV4 +* IPV6 +* MAC_ADDRESS +* PERSON +* PHONE_NUMBER +* SSN +* URL +* US_STATE +* DRIVERS_LICENSE +* DATE +* TIME +* DATETIME +* INTEGER +* FLOAT +* QUANTITY +* ORDINAL + + +Get Started +=========== + +Load a File +~~~~~~~~~~~ + +The profiler should automatically identify the file type and load the data into a `Data Class`. + +Along with other attributtes the `Data class` enables structured data to be accessed via a valid Pandas DataFrame. + +.. code-block:: python + + # Load a csv file, return a CSVData object + csv_data = Data('your_file.csv') + + # Print the first 10 rows of the csv file + print(csv_data.data.head(10)) + + # Load a parquet file, return a ParquetData object + parquet_data = Data('your_file.parquet') + + # Sort the data by the name column + parquet_data.data.sort_values(by='name', inplace=True) + + # Print the sorted first 10 rows of the parquet data + print(parquet_data.data.head(10)) + + +If the file type is not automatically identified (rare), you can specify them +specifically, see section Data Readers. + +Profile a File +~~~~~~~~~~~~~~ + +Example uses a CSV file for example, but CSV, JSON, Avro, Parquet or Text should also work. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + # Load file (CSV should be automatically identified) + data = Data("your_file.csv") + + # Profile the dataset + profile = Profiler(data) + + # Generate a report and use json to prettify. + report = profile.report(report_options={"output_format":"pretty"}) + + # Print the report + print(json.dumps(report, indent=4)) + +Updating Profiles +~~~~~~~~~~~~~~~~~ + +Currently, the data profiler is equipped to update its profile in batches. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + # Load and profile a CSV file + data = Data("your_file.csv") + profile = Profiler(data) + + # Update the profile with new data: + new_data = Data("new_data.csv") + profile.update_profile(new_data) + + # Print the report using json to prettify. + report = profile.report(report_options={"output_format":"pretty"}) + print(json.dumps(report, indent=4)) + + +Merging Profiles +~~~~~~~~~~~~~~~~ + +If you have two files with the same schema (but different data), it is possible to merge the two profiles together via an addition operator. + +This also enables profiles to be determined in a distributed manner. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + # Load a CSV file with a schema + data1 = Data("file_a.csv") + profile1 = Profiler(data) + + # Load another CSV file with the same schema + data2 = Data("file_b.csv") + profile2 = Profiler(data) + + profile3 = profile1 + profile2 + + # Print the report using json to prettify. + report = profile3.report(report_options={"output_format":"pretty"}) + print(json.dumps(report, indent=4)) + +Profile a Pandas DataFrame +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import pandas as pd + import dataprofiler as dp + import json + + my_dataframe = pd.DataFrame([[1, 2.0],[1, 2.2],[-1, 3]]) + profile = dp.Profiler(my_dataframe) + + # print the report using json to prettify. + report = profile.report(report_options={"output_format":"pretty"}) + print(json.dumps(report, indent=4)) + + # read a specified column, in this case it is labeled 0: + print(json.dumps(report["data stats"][0], indent=4)) + + +Unstructured Profiler +~~~~~~~~~~~~~~~~~~~~~ + +In addition to the structured profiler, the Data Profiler provides unstructured +profiling for the TextData object or string. Unstructured profiling also works +with list(string), pd.Series(string) or pd.DataFrame(string) given profiler_type +option specified as `unstructured`. Below is an example of unstructured profile +with a text file. + +.. code-block:: python + + import dataprofiler as dp + import json + my_text = dp.Data('text_file.txt') + profile = dp.Profiler(my_text) + + # print the report using json to prettify. + report = profile.report(report_options={"output_format":"pretty"}) + print(json.dumps(report, indent=4)) + +Another example of unstructured profile with pd.Series of string is given as below + +.. code-block:: python + + import dataprofiler as dp + import pandas as pd + import json + + text_data = pd.Series(['first string', 'second string']) + profile = dp.Profiler(text_data, profiler_type="unstructured") + + # print the report using json to prettify. + report = profile.report(report_options={"output_format":"pretty"}) + print(json.dumps(report, indent=4)) + + +Graph Profiler +~~~~~~~~~~~~~~ + +DataProfiler also provides the ability to profile graph data from a csv file. Below is an example of the graph profiler with a graph data csv file: + +.. code-block:: python + + import dataprofiler as dp + import pprint + + my_graph = dp.Data('graph_file.csv') + profile = dp.Profiler(my_graph) + + # print the report using pretty print (json dump does not work on numpy array values inside dict) + report = profile.report() + printer = pprint.PrettyPrinter(sort_dicts=False, compact=True) + printer.pprint(report) + + +Specifying a Filetype or Delimiter +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Example of specifying a CSV data type, with a `,` delimiter. +In addition, it utilizes only the first 10,000 rows. + +.. code-block:: python + + import json + import os + from dataprofiler import Data, Profiler + from dataprofiler.data_readers.csv_data import CSVData + + # Load a CSV file, with "," as the delimiter + data = CSVData("your_file.csv", options={"delimiter": ","}) + + # Split the data, such that only the first 10,000 rows are used + data = data.data[0:10000] + + # Read in profile and print results + profile = Profiler(data) + print(json.dumps(profile.report(report_options={"output_format":"pretty"}), indent=4)) + + +.. toctree:: + :maxdepth: 2 + :hidden: + :caption: Getting Started: + + Intro + install.rst + data_readers.rst + profiler.rst + data_labeling.rst + graphs.rst + architecture.rst + +.. toctree:: + :maxdepth: 2 + :hidden: + :caption: User Guide: + + examples.rst + API.rst + +.. toctree:: + :maxdepth: 2 + :hidden: + :caption: Community: + + roadmap.rst + Changelog + Feedback + GitHub + Contributing + +.. _Example CSV: https://raw.githubusercontent.com/capitalone/DataProfiler/main/dataprofiler/tests/data/csv/aws_honeypot_marx_geo.csv +.. _issue: https://github.com/capitalone/DataProfiler/issues/new/choose + + + diff --git a/_docs/docs/source/install.rst b/_docs/docs/source/install.rst new file mode 100644 index 000000000..bdf4c3bb4 --- /dev/null +++ b/_docs/docs/source/install.rst @@ -0,0 +1,145 @@ +.. _install: + +Install +******* + +To install the full package from pypi: + +.. code-block:: console + + pip install DataProfiler[ml] + +If the ML requirements are too strict (say, you don't want to install +tensorflow), you can install a slimmer package. The slimmer package disables +the default sensitive data detection / entity recognition (labler) + +Install from pypi: + +.. code-block:: console + + pip install DataProfiler + +Snappy Installation +=================== + +This is required to profile parquet/avro datasets + +MacOS (intel chip) with homebrew: + +.. code-block:: console + + brew install snappy && CPPFLAGS="-I/usr/local/include -L/usr/local/lib" pip install python-snappy + + +MacOS (apple chip) with homebrew: + +.. code-block:: console + + brew install snappy && CPPFLAGS="-I/opt/homebrew/include -L/opt/homebrew/lib" pip install python-snappy + + +Linux install: + +.. code-block:: console + + sudo apt-get -y install libsnappy-dev + + +Build From Scratch +================== + +NOTE: Installation for python3 + +virtualenv install: + +.. code-block:: console + + python3 -m pip install virtualenv + + +Setup virtual env: + +.. code-block:: console + + python3 -m virtualenv --python=python3 venv3 + source venv3/bin/activate + + +Install requirements: + +.. code-block:: console + + pip3 install -r requirements.txt + +Install labeler dependencies: + +.. code-block:: console + + pip3 install -r requirements-ml.txt + + +Install via the repo -- Build setup.py and install locally: + +.. code-block:: console + + python3 setup.py sdist bdist bdist_wheel + pip3 install dist/DataProfiler*-py3-none-any.whl + + +If you see: + +.. code-block:: console + + ERROR: Double requirement given:dataprofiler==X.Y.Z from dataprofiler/dist/DataProfiler-X.Y.Z-py3-none-any.whl (already in dataprofiler==X2.Y2.Z2 from dataprofiler/dist/DataProfiler-X2.Y2.Z2-py3-none-any.whl, name='dataprofiler') + +This means that you have multiple versions of the DataProfiler distribution +in the dist folder. +To resolve, either remove the older one or delete the folder and rerun the steps +above. + +Install via github: + +.. code-block:: console + + pip3 install git+https://github.com/capitalone/dataprofiler.git#egg=dataprofiler + + + +Testing +======= + +For testing, install test requirements: + +.. code-block:: console + + pip3 install -r requirements-test.txt + + +To run all unit tests, use: + +.. code-block:: console + + DATAPROFILER_SEED=0 python3 -m unittest discover -p "test*.py" + + +To run file of unit tests, use form: + +.. code-block:: console + + DATAPROFILER_SEED=0 python3 -m unittest discover -p test_profile_builder.py + + +To run a file with Pytest use: + +.. code-block:: console + + DATAPROFILER_SEED=0 pytest dataprofiler/tests/data_readers/test_csv_data.py -v + + +To run individual of unit test, use form: + +.. code-block:: console + + DATAPROFILER_SEED=0 python3 -m unittest dataprofiler.tests.profilers.test_profile_builder.TestProfiler + + diff --git a/_docs/docs/source/labeler.nblink b/_docs/docs/source/labeler.nblink new file mode 100644 index 000000000..f862443fd --- /dev/null +++ b/_docs/docs/source/labeler.nblink @@ -0,0 +1,6 @@ +{ + "path": "../../../examples/labeler.ipynb", + "extra-media": [ + "../../../examples/DL-Flowchart.png" + ] +} \ No newline at end of file diff --git a/_docs/docs/source/merge_profile_list.nblink b/_docs/docs/source/merge_profile_list.nblink new file mode 100644 index 000000000..39102658b --- /dev/null +++ b/_docs/docs/source/merge_profile_list.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/merge_profile_list.ipynb" +} \ No newline at end of file diff --git a/_docs/docs/source/modules.rst b/_docs/docs/source/modules.rst new file mode 100644 index 000000000..0593459df --- /dev/null +++ b/_docs/docs/source/modules.rst @@ -0,0 +1,7 @@ +dataprofiler +============ + +.. toctree:: + :maxdepth: 4 + + dataprofiler diff --git a/_docs/docs/source/overview.nblink b/_docs/docs/source/overview.nblink new file mode 100644 index 000000000..4c118878e --- /dev/null +++ b/_docs/docs/source/overview.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/intro_data_profiler.ipynb" +} \ No newline at end of file diff --git a/_docs/docs/source/popmon_dp_loader_example.nblink b/_docs/docs/source/popmon_dp_loader_example.nblink new file mode 100644 index 000000000..1a288a318 --- /dev/null +++ b/_docs/docs/source/popmon_dp_loader_example.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/popmon_dp_loader_example.ipynb" +} \ No newline at end of file diff --git a/_docs/docs/source/profiler.rst b/_docs/docs/source/profiler.rst new file mode 100644 index 000000000..56d16a274 --- /dev/null +++ b/_docs/docs/source/profiler.rst @@ -0,0 +1,965 @@ +.. _profiler: + +Profiler +******** + +Profile Your Data +================= + +Profiling your data is easy. Just use the data reader, send the data to the +profiler, and print out the report. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + data = Data("your_file.csv") # Auto-Detect & Load: CSV, AVRO, Parquet, JSON, Text + + profile = Profiler(data) # Calculate Statistics, Entity Recognition, etc + + readable_report = profile.report(report_options={"output_format": "pretty"}) + print(json.dumps(readable_report, indent=4)) + +If the data is structured, the profile will return global statistics as well as +column by column statistics. The vast amount of statistics are listed on the +intro page. + +Load a File +~~~~~~~~~~~ + +The profiler should automatically identify the file type and load the data into a `Data Class`. + +Along with other attributtes the `Data class` enables structured data to be accessed via a valid Pandas DataFrame. + +.. code-block:: python + + # Load a csv file, return a CSVData object + csv_data = Data('your_file.csv') + + # Print the first 10 rows of the csv file + print(csv_data.data.head(10)) + + # Load a parquet file, return a ParquetData object + parquet_data = Data('your_file.parquet') + + # Sort the data by the name column + parquet_data.data.sort_values(by='name', inplace=True) + + # Print the sorted first 10 rows of the parquet data + print(parquet_data.data.head(10)) + + +If the file type is not automatically identified (rare), you can specify them +specifically, see section Data Readers. + +Profile a File +~~~~~~~~~~~~~~ + +Example uses a CSV file for example, but CSV, JSON, Avro or Parquet should also work. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + # Load file (CSV should be automatically identified) + data = Data("your_file.csv") + + # Profile the dataset + profile = Profiler(data) + + # Generate a report and use json to prettify. + report = profile.report(report_options={"output_format": "pretty"}) + + # Print the report + print(json.dumps(report, indent=4)) + +Updating Profiles +~~~~~~~~~~~~~~~~~ + +Currently, the data profiler is equipped to update its profile in batches. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + # Load and profile a CSV file + data = Data("your_file.csv") + profile = Profiler(data) + + # Update the profile with new data: + new_data = Data("new_data.csv") + profile.update_profile(new_data) + + # Print the report using json to prettify. + report = profile.report(report_options={"output_format": "pretty"}) + print(json.dumps(report, indent=4)) + + +Merging Profiles +~~~~~~~~~~~~~~~~ + +If you have two files with the same schema (but different data), it is possible to merge the two profiles together via an addition operator. + +This also enables profiles to be determined in a distributed manner. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + # Load a CSV file with a schema + data1 = Data("file_a.csv") + profile1 = Profiler(data) + + # Load another CSV file with the same schema + data2 = Data("file_b.csv") + profile2 = Profiler(data) + + profile3 = profile1 + profile2 + + # Print the report using json to prettify. + report = profile3.report(report_options={"output_format": "pretty"}) + print(json.dumps(report, indent=4)) + + +Profile Differences +~~~~~~~~~~~~~~~~~~~ + +Profile differences take two profiles and find the differences +between them. Create the difference report like this: + +.. code-block:: python + + from dataprofiler import Data, Profiler + + # Load a CSV file + data1 = Data("file_a.csv") + profile1 = Profiler(data) + + # Load another CSV file + data2 = Data("file_b.csv") + profile2 = Profiler(data) + + diff_report = profile1.diff(profile2) + print(diff_report) + +The `.diff()` operation is available between two profiles, although there are different +outputs depending on the type of profile being differenced. For example, for numerical +column profiles (e.g. integers and floats), two valuable calculations that +`.diff()` returns are `t-test`, `chi2-test`, and `psi` (Popoulation Stability Index) +for understanding distributional changes. + +The difference report contains a dictionary that mirrors the profile report. +Each data type has its own difference: + +* **Int/Float** - One profile subtracts the value from the other. + +* **String** - The strings will be shown in a list: + + - [profile1 str, profile2 str] +* **List** - A list of 3 will be returned showing the unique values of + each profile and the shared values: + + - [profile 1 unique values, shared values, profile 2 unique values] +* **Dict** - Some dictionaries with varied keys will also return a list + of three in the format: + + - [profile 1 unique key-values, shared key differences, profile 2 unique key-values] + +Otherwise, when no differences occur: + +* **Any Type No Differences** - A string will report: "unchanged". + +Below is the structured difference report: + +.. code-block:: python + + { + 'global_stats': { + 'file_type': [str, str], + 'encoding': [str, str], + 'samples_used': int, + 'column_count': int, + 'row_count': int, + 'row_has_null_ratio': float, + 'row_is_null_ratio': float, + 'unique_row_ratio': float, + 'duplicate_row_count': int, + 'correlation_matrix': list[list[float]], + 'chi2_matrix': list[list[float]], + 'profile_schema': list[dict[str, int]] + }, + 'data_stats': [{ + 'column_name': str, + 'data_type': [str, str], + 'data_label': [list[str], list[str], list[str]], + 'categorical': [str, str], + 'order': [str, str], + 'statistics': { + 'min': float, + 'max': float, + 'sum': float, + 'mean': float, + 'median': float, + 'mode': [list[float], list[float], list[float]], + 'median_absolute_deviation': float, + 'variance': float, + 'stddev': float, + 't-test': { + 't-statistic': float, + 'conservative': {'deg_of_free': int, + 'p-value': float}, + 'welch': {'deg_of_free': float, + 'p-value': float}}, + 'psi': float, + "chi2-test": { + "chi2-statistic": float, + "deg_of_free": int, + "p-value": float + }, + 'unique_count': int, + 'unique_ratio': float, + 'categories': [list[str], list[str], list[str]], + 'gini_impurity': float, + 'unalikeability': float, + 'categorical_count': [dict[str, int], dict[str, int], dict[str, int]], + 'avg_predictions': [dict[str, float]], + 'label_representation': [dict[str, float]], + 'sample_size': int, + 'null_count': int, + 'null_types': [list[str], list[str], list[str]], + 'null_types_index': [dict[str, int], dict[str, int], dict[str, int]], + 'data_type_representation': [dict[str, float]] + }, + "null_replication_metrics": { + "class_prior": list[int], + "class_sum": list[list[int]], + "class_mean": list[list[int]] + } + } + +Below is the unstructured difference report: + +.. code-block:: python + + { + 'global_stats': { + 'file_type': [str, str], + 'encoding': [str, str], + 'samples_used': int, + 'empty_line_count': int, + 'memory_size': float + }, + 'data_stats': { + 'data_label': { + 'entity_counts': { + 'word_level': dict[str, int], + 'true_char_level': dict[str, int], + 'postprocess_char_level': dict[str, int] + }, + 'entity_percentages': { + 'word_level': dict[str, float], + 'true_char_level': dict[str, float], + 'postprocess_char_level': dict[str, float] + } + }, + 'statistics': { + 'vocab': [list[str], list[str], list[str]], + 'vocab_count': [dict[str, int], dict[str, int], dict[str, int]], + 'words': [list[str], list[str], list[str]], + 'word_count': [dict[str, int], dict[str, int], dict[str, int]] + } + } + } + + +Saving and Loading a Profile +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The profiles can easily be saved and loaded as shown below: + +**NOTE: Json saving and loading only supports Structured Profiles currently.** + +There are two save/load methods: + +* **Pickle save/load** + + * Save a profile as a `.pkl` file. + * Load a `.pkl` file as a profile object. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + # Load a CSV file, with "," as the delimiter + data = Data("your_file.csv") + + # Read data into profile + profile = Profiler(data) + + # save structured profile to pkl file + profile.save(filepath="my_profile.pkl") + + # load pkl file to structured profile + loaded_pkl_profile = dp.Profiler.load(filepath="my_profile.pkl") + + print(json.dumps(loaded_pkl_profile.report(report_options={"output_format": "compact"}), + indent=4)) + +* **Json save/load** + + * Save a profile as a human-readable `.json` file. + * Load a `.json` file as a profile object. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + # Load a CSV file, with "," as the delimiter + data = Data("your_file.csv") + + # Read data into profile + profile = Profiler(data) + + # save structured profile to json file + profile.save(filepath="my_profile.json", save_method="json") + + # load json file to structured profile + loaded_json_profile = dp.Profiler.load(filepath="my_profile.json", load_method="json") + + print(json.dumps(loaded_json_profile.report(report_options={"output_format": "compact"}), + indent=4)) + + +Structured vs Unstructured Profiles +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using the profiler, the data profiler will automatically infer whether to +create the structured profile or the unstructured profile. However, you can be +explicit as shown below: + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + + # Creating a structured profile + data1 = Data("normal_csv_file.csv") + structured_profile = Profiler(data1, profiler_type="structured") + + structured_report = structured_profile.report(report_options={"output_format": "pretty"}) + print(json.dumps(structured_report, indent=4)) + + # Creating an unstructured profile + data2 = Data("normal_text_file.txt") + unstructured_profile = Profiler(data2, profiler_type="unstructured") + + unstructured_report = unstructured_profile.report(report_options={"output_format": "pretty"}) + print(json.dumps(unstructured_report, indent=4)) + + +Setting the Sample Size +~~~~~~~~~~~~~~~~~~~~~~~ + +There are two ways to set sample size in a profile: samples_per_update and +min_true_samples. Samples_per_update takes an integer as the exact amount that +will be sampled. Min_true_samples will set the minimum amount of samples that +are not null. For example: + +.. code-block:: python + + from dataprofiler import Profiler + + sample_array = [1.0, NULL, 2.0] + profile = dp.Profiler(sample_array, samples_per_update=2) + +The first two samples (1.0 and NULL) are used for the statistical analysis. + +In contrast, if we also set min_true_samples to 2 then the Data Reader will +continue to read until the minimum true samples were found for the given column. +For example: + +.. code-block:: python + + from dataprofiler import Profiler + + sample_array = [1.0, NULL, 2.0] + profile = dp.Profiler(sample_array, samples_per_update=2, min_true_samples=2) + +This will use all samples in the statistical analysis until the number of "true" +(non-NULL) values are reached. Both min_true_samples and +samples_per_update conditions must be met. In this case, the profile will grab +the first two samples (1.0 and NULL) to satisfy the samples_per_update, and then +it will grab the first two VALID samples (1.0 and 2.0) to satisfy the +min_true_samples. + +Profile a Pandas DataFrame +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import pandas as pd + import dataprofiler as dp + import json + + my_dataframe = pd.DataFrame([[1, 2.0],[1, 2.2],[-1, 3]]) + profile = dp.Profiler(my_dataframe) + + # print the report using json to prettify. + report = profile.report(report_options={"output_format": "pretty"}) + print(json.dumps(report, indent=4)) + + # read a specified column, in this case it is labeled 0: + print(json.dumps(report["data stats"][0], indent=4)) + + +Specifying a Filetype or Delimiter +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Example of specifying a CSV data type, with a `,` delimiter. +In addition, it utilizes only the first 10,000 rows. + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler + from dataprofiler.data_readers.csv_data import CSVData + + # Load a CSV file, with "," as the delimiter + data = CSVData("your_file.csv", options={"delimiter": ","}) + + # Split the data, such that only the first 10,000 rows are used + data = data.data[0:10000] + + # Read in profile and print results + profile = Profiler(data) + print(json.dumps(profile.report(report_options={"output_format": "pretty"}), indent=4)) + +Setting Profiler Seed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Example of specifying a seed for reproducibility. + +.. code-block:: python + + import dataprofiler as dp + + # Set seed to non-negative integer value or None + dp.set_seed(0) + + +Profile Statistic Descriptions +============================== + +Structured Profile +~~~~~~~~~~~~~~~~~~ + +**global_stats**: + +* samples_used - number of input data samples used to generate this profile +* column_count - the number of columns contained in the input dataset +* row_count - the number of rows contained in the input dataset +* row_has_null_ratio - the proportion of rows that contain at least one null value to the total number of rows +* row_is_null_ratio - the proportion of rows that are fully comprised of null values (null rows) to the total number of rows +* unique_row_ratio - the proportion of distinct rows in the input dataset to the total number of rows +* duplicate_row_count - the number of rows that occur more than once in the input dataset +* file_type - the format of the file containing the input dataset (ex: .csv) +* encoding - the encoding of the file containing the input dataset (ex: UTF-8) +* correlation_matrix - matrix of shape `column_count` x `column_count` containing the correlation coefficients between each column in the dataset +* chi2_matrix - matrix of shape `column_count` x `column_count` containing the chi-square statistics between each column in the dataset +* profile_schema - a description of the format of the input dataset labeling each column and its index in the dataset + * string - the label of the column in question and its index in the profile schema +* times - the duration of time it took to generate the global statistics for this dataset in milliseconds + +**data_stats**: + +* column_name - the label/title of this column in the input dataset +* data_type - the primitive python data type that is contained within this column +* data_label - the label/entity of the data in this column as determined by the Labeler component +* categorical - 'true' if this column contains categorical data +* order - the way in which the data in this column is ordered, if any, otherwise “random†+* samples - a small subset of data entries from this column +* statistics - statistical information on the column + * sample_size - number of input data samples used to generate this profile + * null_count - the number of null entries in the sample + * null_types - a list of the different null types present within this sample + * null_types_index - a dict containing each null type and a respective list of the indicies that it is present within this sample + * data_type_representation - the percentage of samples used identifying as each data_type + * min - minimum value in the sample + * max - maximum value in the sample + * mode - mode of the entries in the sample + * median - median of the entries in the sample + * median_absolute_deviation - the median absolute deviation of the entries in the sample + * sum - the total of all sampled values from the column + * mean - the average of all entries in the sample + * variance - the variance of all entries in the sample + * stddev - the standard deviation of all entries in the sample + * skewness - the statistical skewness of all entries in the sample + * kurtosis - the statistical kurtosis of all entries in the sample + * num_zeros - the number of entries in this sample that have the value 0 + * num_negatives - the number of entries in this sample that have a value less than 0 + * histogram - contains histogram relevant information + * bin_counts - the number of entries within each bin + * bin_edges - the thresholds of each bin + * quantiles - the value at each percentile in the order they are listed based on the entries in the sample + * vocab - a list of the characters used within the entries in this sample + * avg_predictions - average of the data label prediction confidences across all data points sampled + * categories - a list of each distinct category within the sample if `categorial` = 'true' + * unique_count - the number of distinct entries in the sample + * unique_ratio - the proportion of the number of distinct entries in the sample to the total number of entries in the sample + * categorical_count - number of entries sampled for each category if `categorical` = 'true' + * gini_impurity - measure of how often a randomly chosen element from the set would be incorrectly labeled if it was randomly labeled according to the distribution of labels in the subset + * unalikeability - a value denoting how frequently entries differ from one another within the sample + * precision - a dict of statistics with respect to the number of digits in a number for each sample + * times - the duration of time it took to generate this sample's statistics in milliseconds + * format - list of possible datetime formats +* null_replication_metrics - statistics of data partitioned based on whether column value is null (index 1 of lists referenced by dict keys) or not (index 0) + * class_prior - a list containing probability of a column value being null and not null + * class_sum - a list containing sum of all other rows based on whether column value is null or not + * class_mean - a list containing mean of all other rows based on whether column value is null or not + +Unstructured Profile +~~~~~~~~~~~~~~~~~~~~ + +**global_stats**: + +* samples_used - number of input data samples used to generate this profile +* empty_line_count - the number of empty lines in the input data +* file_type - the file type of the input data (ex: .txt) +* encoding - file encoding of the input data file (ex: UTF-8) +* memory_size - size of the input data in MB +* times - duration of time it took to generate this profile in milliseconds + +**data_stats**: + +* data_label - labels and statistics on the labels of the input data + * entity_counts - the number of times a specific label or entity appears inside the input data + * word_level - the number of words counted within each label or entity + * true_char_level - the number of characters counted within each label or entity as determined by the model + * postprocess_char_level - the number of characters counted within each label or entity as determined by the postprocessor + * entity_percentages - the percentages of each label or entity within the input data + * word_level - the percentage of words in the input data that are contained within each label or entity + * true_char_level - the percentage of characters in the input data that are contained within each label or entity as determined by the model + * postprocess_char_level - the percentage of characters in the input data that are contained within each label or entity as determined by the postprocessor + * times - the duration of time it took for the data labeler to predict on the data +* statistics - statistics of the input data + * vocab - a list of each character in the input data + * vocab_count - the number of occurrences of each distinct character in the input data + * words - a list of each word in the input data + * word_count - the number of occurrences of each distinct word in the input data + * times - the duration of time it took to generate the vocab and words statistics in milliseconds + +Graph Profile +~~~~~~~~~~~~~~~~~~ + +* num_nodes - number of nodes in the graph +* num_edges - number of edges in the graph +* categorical_attributes - list of categorical edge attributes +* continuous_attributes - list of continuous edge attributes +* avg_node_degree - average degree of nodes in the graph +* global_max_component_size: size of the global max component + +**continuous_distribution**: + +* : name of N-th edge attribute in list of attributes + * name - name of distribution for attribute + * scale - negative log likelihood used to scale and compare distributions + * properties - list of statistical properties describing the distribution + * [shape (optional), loc, scale, mean, variance, skew, kurtosis] + +**categorical_distribution**: + +* : name of N-th edge attribute in list of attributes + * bin_counts: counts in each bin of the distribution histogram + * bin_edges: edges of each bin of the distribution histogram + +* times - duration of time it took to generate this profile in milliseconds + +Profile Options +=============== + +The data profiler accepts several options to toggle on and off +features. The 8 columns (int options, float options, datetime options, +text options, order options, category options, data labeler options) can be +enabled or disabled. By default, all options are toggled on. Below is an example +of how to alter these options. Options shared by structured and unstructured options +must be specified as structured or unstructured when setting (ie. datalabeler options). + + +.. code-block:: python + + import json + from dataprofiler import Data, Profiler, ProfilerOptions + + # Load and profile a CSV file + data = Data("your_file.csv") + profile_options = ProfilerOptions() + + #All of these are different examples of adjusting the profile options + + # Options can be toggled directly like this: + profile_options.structured_options.text.is_enabled = False + profile_options.structured_options.text.vocab.is_enabled = True + profile_options.structured_options.int.variance.is_enabled = True + profile_options.structured_options.data_labeler.data_labeler_dirpath = \ + "Wheres/My/Datalabeler" + profile_options.structured_options.data_labeler.is_enabled = False + + # A dictionary can be sent in to set the properties for all the options + profile_options.set({"structured_options.data_labeler.is_enabled": False, "min.is_enabled": False}) + + # Specific columns can be set/disabled/enabled in the same way + profile_options.structured_options.text.set({"max.is_enabled":True, + "variance.is_enabled": True}) + + # numeric stats can be turned off/on entirely + profile_options.set({"is_numeric_stats_enabled": False}) + profile_options.set({"int.is_numeric_stats_enabled": False}) + + profile = Profiler(data, options=profile_options) + + # Print the report using json to prettify. + report = profile.report(report_options={"output_format": "pretty"}) + print(json.dumps(report, indent=4)) + + +Below is an breakdown of all the options. + +* **ProfilerOptions** - The top-level options class that contains options for the Profiler class + + * **presets** - A pre-configured mapping of a string name to group of options: + + * **default is None** + + * **"complete"** + + .. code-block:: python + + options = ProfilerOptions(presets="complete") + + * **"data_types"** + + .. code-block:: python + + options = ProfilerOptions(presets="data_types") + + * **"numeric_stats_disabled"** + + .. code-block:: python + + options = ProfilerOptions(presets="numeric_stats_disabled") + + * **"lower_memory_sketching"** + + .. code-block:: python + + options = ProfilerOptions(presets="lower_memory_sketching") + + * **structured_options** - Options responsible for all structured data + + * **multiprocess** - Option to enable multiprocessing. If on, multiprocessing is toggled on if the dataset contains more than 750,000 rows or more than 20 columns. + Automatically selects the optimal number of pooling processes to utilize based on system constraints when toggled on. + + * is_enabled - (Boolean) Enables or disables multiprocessing + + * **sampling_ratio** - A percentage, as a decimal, ranging from greater than 0 to less than or equal to 1 indicating how much input data to sample. Default value set to 0.2. + + * **int** - Options for the integer columns + + * is_enabled - (Boolean) Enables or disables the integer operations + + * min - Finds minimum value in a column + + * is_enabled - (Boolean) Enables or disables min + + * max - Finds maximum value in a column + + * is_enabled - (Boolean) Enables or disables max + + * mode - Finds mode(s) in a column + + * is_enabled - (Boolean) Enables or disables mode + * top_k_modes - (Int) Sets the number of modes to return if multiple exist. Default returns max 5 modes. + * median - Finds median value in a column + + * is_enabled - (Boolean) Enables or disables median + * sum - Finds sum of all values in a column + + * is_enabled - (Boolean) Enables or disables sum + + * variance - Finds variance of all values in a column + + * is_enabled - (Boolean) Enables or disables variance + * skewness - Finds skewness of all values in a column + + * is_enabled - (Boolean) Enables or disables skewness + * kurtosis - Finds kurtosis of all values in a column + + * is_enabled - (Boolean) Enables or disables kurtosis + * median_abs_deviation - Finds median absolute deviation of all values in a column + + * is_enabled - (Boolean) Enables or disables median absolute deviation + * num_zeros - Finds the count of zeros in a column + + * is_enabled - (Boolean) Enables or disables num_zeros + * num_negatives - Finds the count of negative numbers in a column + + * is_enabled - (Boolean) Enables or disables num_negatives + * bias_correction - Applies bias correction to variance, skewness, and kurtosis calculations + + * is_enabled - (Boolean) Enables or disables bias correction + * histogram_and_quantiles - Generates a histogram and quantiles + from the column values + + * bin_count_or_method - (String/List[String]) Designates preferred method for calculating histogram bins or the number of bins to use. + If left unspecified (None) the optimal method will be chosen by attempting all methods. + If multiple specified (list) the optimal method will be chosen by attempting the provided ones. + methods: 'auto', 'fd', 'doane', 'scott', 'rice', 'sturges', 'sqrt' + Note: 'auto' is used to choose optimally between 'fd' and 'sturges' + * num_quantiles - (Int) Number of quantiles to bin the data. + Default value is set to 1,000 quantiles. + * is_enabled - (Boolean) Enables or disables histogram and quantiles + * **float** - Options for the float columns + + * is_enabled - (Boolean) Enables or disables the float operations + * precision - Finds the precision (significant figures) within the column + + * is_enabled - (Boolean) Enables or disables precision + * sample_ratio - (Float) The ratio of 0 to 1 how much data (identified as floats) to utilize as samples in determining precision + + * min - Finds minimum value in a column + + * is_enabled - (Boolean) Enables or disables min + * max - Finds maximum value in a column + + * is_enabled - (Boolean) Enables or disables max + * mode - Finds mode(s) in a column + + * is_enabled - (Boolean) Enables or disables mode + * top_k_modes - (Int) Sets the number of modes to return if multiple exist. Default returns max 5 modes. + * median - Finds median value in a column + + * is_enabled - (Boolean) Enables or disables median + * sum - Finds sum of all values in a column + + * is_enabled - (Boolean) Enables or disables sum + * variance - Finds variance of all values in a column + + * is_enabled - (Boolean) Enables or disables variance + * skewness - Finds skewness of all values in a column + + * is_enabled - (Boolean) Enables or disables skewness + * kurtosis - Finds kurtosis of all values in a column + + * is_enabled - (Boolean) Enables or disables kurtosis + * median_abs_deviation - Finds median absolute deviation of all values in a column + + * is_enabled - (Boolean) Enables or disables median absolute deviation + * is_numeric_stats_enabled - (Boolean) enable or disable all numeric stats + * num_zeros - Finds the count of zeros in a column + + * is_enabled - (Boolean) Enables or disables num_zeros + * num_negatives - Finds the count of negative numbers in a column + + * is_enabled - (Boolean) Enables or disables num_negatives + * bias_correction - Applies bias correction to variance, skewness, and kurtosis calculations + + * is_enabled - (Boolean) Enables or disables bias correction + * histogram_and_quantiles - Generates a histogram and quantiles + from the column values + + * bin_count_or_method - (String/List[String]) Designates preferred method for calculating histogram bins or the number of bins to use. + If left unspecified (None) the optimal method will be chosen by attempting all methods. + If multiple specified (list) the optimal method will be chosen by attempting the provided ones. + methods: 'auto', 'fd', 'doane', 'scott', 'rice', 'sturges', 'sqrt' + Note: 'auto' is used to choose optimally between 'fd' and 'sturges' + * num_quantiles - (Int) Number of quantiles to bin the data. + Default value is set to 1,000 quantiles. + * is_enabled - (Boolean) Enables or disables histogram and quantiles + * **text** - Options for the text columns + + * is_enabled - (Boolean) Enables or disables the text operations + * vocab - Finds all the unique characters used in a column + + * is_enabled - (Boolean) Enables or disables vocab + * min - Finds minimum value in a column + + * is_enabled - (Boolean) Enables or disables min + * max - Finds maximum value in a column + + * is_enabled - (Boolean) Enables or disables max + * mode - Finds mode(s) in a column + + * is_enabled - (Boolean) Enables or disables mode + * top_k_modes - (Int) Sets the number of modes to return if multiple exist. Default returns max 5 modes. + * median - Finds median value in a column + + * is_enabled - (Boolean) Enables or disables median + * sum - Finds sum of all values in a column + + * is_enabled - (Boolean) Enables or disables sum + * variance - Finds variance of all values in a column + + * is_enabled - (Boolean) Enables or disables variance + * skewness - Finds skewness of all values in a column + + * is_enabled - (Boolean) Enables or disables skewness + * kurtosis - Finds kurtosis of all values in a column + + * is_enabled - (Boolean) Enables or disables kurtosis + * median_abs_deviation - Finds median absolute deviation of all values in a column + + * is_enabled - (Boolean) Enables or disables median absolute deviation + * bias_correction - Applies bias correction to variance, skewness, and kurtosis calculations + + * is_enabled - (Boolean) Enables or disables bias correction + * is_numeric_stats_enabled - (Boolean) enable or disable all numeric stats + * num_zeros - Finds the count of zeros in a column + + * is_enabled - (Boolean) Enables or disables num_zeros + * num_negatives - Finds the count of negative numbers in a column + + * is_enabled - (Boolean) Enables or disables num_negatives + * histogram_and_quantiles - Generates a histogram and quantiles + from the column values + + * bin_count_or_method - (String/List[String]) Designates preferred method for calculating histogram bins or the number of bins to use. + If left unspecified (None) the optimal method will be chosen by attempting all methods. + If multiple specified (list) the optimal method will be chosen by attempting the provided ones. + methods: 'auto', 'fd', 'doane', 'scott', 'rice', 'sturges', 'sqrt' + Note: 'auto' is used to choose optimally between 'fd' and 'sturges' + * num_quantiles - (Int) Number of quantiles to bin the data. + Default value is set to 1,000 quantiles. + * is_enabled - (Boolean) Enables or disables histogram and quantiles + * **datetime** - Options for the datetime columns + + * is_enabled - (Boolean) Enables or disables the datetime operations + * **order** - Options for the order columns + + * is_enabled - (Boolean) Enables or disables the order operations + * **category** - Options for the category columns + + * is_enabled - (Boolean) Enables or disables the category operations + * top_k_categories - (int) Number of categories to be displayed when reporting + * max_sample_size_to_check_stop_condition - (int) The maximum sample size before categorical stop conditions are checked + * stop_condition_unique_value_ratio - (float) The highest ratio of unique values to dataset size that is to be considered a categorical type + * cms - (Boolean) Enables or Disables the use of count min sketch / heavy hitters for approximate frequency counts + * cms_confidence - (float) Defines the number of hashes used in CMS, default 0.95 + * cms_relative_error - (float) Defines the number of buckets used in CMS, default 0.01 + * cms_max_num_heavy_hitters - (int) The value used to define the threshold for minimum frequency required by a category to be counted + * **data_labeler** - Options for the data labeler columns + + * is_enabled - (Boolean) Enables or disables the data labeler operations + * data_labeler_dirpath - (String) Directory path to data labeler + * data_labeler_object - (BaseDataLabeler) Datalabeler to replace + the default labeler + * max_sample_size - (Int) The max number of samples for the data + labeler + * **correlation** - Option set for correlation profiling + * is_enabled - (Boolean) Enables or disables performing correlation profiling + * columns - Columns considered to calculate correlation + * **row_statistics** - (Boolean) Option to enable/disable row statistics calculations + + * unique_count - (UniqueCountOptions) Option to enable/disable unique row count calculations + + * is_enabled - (Bool) Enables or disables options for unique row count + * hashing_method - (String) Property to specify row hashing method ("full" | "hll") + * hll - (HyperLogLogOptions) Options for alternative method of estimating unique row count (activated when `hll` is the selected hashing_method) + + * seed - (Int) Used to set HLL hashing function seed + * register_count - (Int) Number of registers is equal to 2^register_count + + * null_count - (Boolean) Option to enable/disable functionalities for row_has_null_ratio and row_is_null_ratio + * **chi2_homogeneity** - Options for the chi-squared test matrix + + * is_enabled - (Boolean) Enables or disables performing chi-squared tests for homogeneity between the categorical columns of the dataset. + * **null_replication_metrics** - Options for calculating null replication metrics + + * is_enabled - (Boolean) Enables or disables calculation of null replication metrics + * **unstructured_options** - Options responsible for all unstructured data + + * **text** - Options for the text profile + + * is_case_sensitive - (Boolean) Specify whether the profile is case sensitive + * stop_words - (List of Strings) List of stop words to be removed when profiling + * top_k_chars - (Int) Number of top characters to be retrieved when profiling + * top_k_words - (Int) Number of top words to be retrieved when profiling + * vocab - Options for vocab count + + * is_enabled - (Boolean) Enables or disables the vocab stats + * words - Options for word count + + * is_enabled - (Boolean) Enables or disables the word stats + * **data_labeler** - Options for the data labeler + + * is_enabled - (Boolean) Enables or disables the data labeler operations + * data_labeler_dirpath - (String) Directory path to data labeler + * data_labeler_object - (BaseDataLabeler) Datalabeler to replace + the default labeler + * max_sample_size - (Int) The max number of samples for the data + labeler + + + +Statistical Dependency on Order of Updates +========================================== + +Some profile features/statistics are dependent on the order in which the profiler +is updated with new data. + +Order Profile +~~~~~~~~~~~~~ + +The order profiler utilizes the last value in the previous data batch to ensure +the subsequent dataset is above/below/equal to that value when predicting +non-random order. + +For instance, a dataset to be predicted as ascending would require the following +batch data update to be ascending and its first value `>=` than that of the +previous batch of data. + +Ex. of ascending: + +.. code-block:: python + + batch_1 = [0, 1, 2] + batch_2 = [3, 4, 5] + +Ex. of random: + +.. code-block:: python + + batch_1 = [0, 1, 2] + batch_2 = [1, 2, 3] # notice how the first value is less than the last value in the previous batch + + +Reporting Structure +=================== + +For every profile, we can provide a report and customize it with a couple optional parameters: + +* output_format (string) + + * This will allow the user to decide the output format for report. + + * Options are one of [pretty, compact, serializable, flat]: + + * Pretty: floats are rounded to four decimal places, and lists are shortened. + * Compact: Similar to pretty, but removes detailed statistics such as runtimes, label probabilities, index locations of null types, etc. + * Serializable: Output is json serializable and not prettified + * Flat: Nested output is returned as a flattened dictionary +* num_quantile_groups (int) + + * You can sample your data as you like! With a minimum of one and a maximum of 1000, you can decide the number of quantile groups! + +.. code-block:: python + + report = profile.report(report_options={"output_format": "pretty"}) + report = profile.report(report_options={"output_format": "compact"}) + report = profile.report(report_options={"output_format": "serializable"}) + report = profile.report(report_options={"output_format": "flat"}) diff --git a/_docs/docs/source/profiler_example.nblink b/_docs/docs/source/profiler_example.nblink new file mode 100644 index 000000000..142ebd97f --- /dev/null +++ b/_docs/docs/source/profiler_example.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/structured_profilers.ipynb" +} \ No newline at end of file diff --git a/_docs/docs/source/regex_labeler_from_scratch.nblink b/_docs/docs/source/regex_labeler_from_scratch.nblink new file mode 100644 index 000000000..3d98c5f1e --- /dev/null +++ b/_docs/docs/source/regex_labeler_from_scratch.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/regex_labeler_from_scratch/DataLabeler_from_scratch.ipynb" +} \ No newline at end of file diff --git a/_docs/docs/source/roadmap.rst b/_docs/docs/source/roadmap.rst new file mode 100644 index 000000000..93886690b --- /dev/null +++ b/_docs/docs/source/roadmap.rst @@ -0,0 +1,58 @@ +.. _roadmap: + +Roadmap +******* + +For more detailed tasks, checkout the repo's github issues page here: +`Github Issues `_. + + +Data Reader Updates +=================== +- Read data from S3 bucket + - All in the current `dp.Data()` API paradigm, we want to enable passing an S3 bucket file path to read in data from AWS s3. +- Pass list of data file paths to data reader +- Pass in linst of data frames to data reader + +New Model +========= +- Transformer model from sensitive data detection + +Historical Profiles +=================== +- Some questions about Historical Profiles / need to step back and rething design to start: + - Meta profile on top? + - Stored windowed info inside? Etc... +- Branch with current state of Historical Profiles +- Two example notebooks of current state: + - Notebook example `one `_. + - Notebook example `two `_. + + +Conditional Report Metric +========================= +- Based on what is populated on other metrics in the report, have "secondary" / "derivatives" of that number (or that number in conjunction with another number) populate in thie report as well. +- For example, if null_count is not None, then populate a null_percent key with a value of the dividence of (null_count / sample_count). + +Space / Time Testing +==================== +- Automatic comparison testing for space and time analysis on PR’s + - Standardize a report for space time analysis for future comparisons (create baseline numbers) + - Include those in integration tests that will automatically run on code when it is changed in PRs +- Could be an optional test, if the user thinks there is concern around the change driving an issue in the library performance + +Testing Suite Upgrades +====================== +- Add mocking to unit tests where mocking is not utilized +- Integration testing separated out from the unit testing suite. Determine how to only run remotely during PRs +- Backward compatibility testing along with informative warnings and errors when a user is utilizing incompatible versions of the library and saved profile object + +Historical Versions +=================== +- Legacy version upgrades to enable patches to prior versions of the Data Profiler + +Miscellaneous +============== +- Refact/or Pandas to Polars DataFrames +- Spearman correlation calculation +- Workflow Profiles diff --git a/_docs/docs/source/unstructured_profiler_example.nblink b/_docs/docs/source/unstructured_profiler_example.nblink new file mode 100644 index 000000000..5b6829754 --- /dev/null +++ b/_docs/docs/source/unstructured_profiler_example.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/unstructured_profilers.ipynb" +} \ No newline at end of file diff --git a/_docs/docs/update_documentation.py b/_docs/docs/update_documentation.py new file mode 100644 index 000000000..7be79612d --- /dev/null +++ b/_docs/docs/update_documentation.py @@ -0,0 +1,87 @@ +#!/usr/bin/python +"""Script which auto updates the github pages documentation.""" +import os +import subprocess +import sys + +sys.path.insert(0, os.path.abspath(f'../../')) + +from dataprofiler import __version__ as version # noqa F401 + +# Make the rst files from the current repo +subprocess.run( + [ + "sphinx-apidoc", + "--templatedir=./source/_templates/", + "-f", + "-e", + "-o", + "../docs/source", + f"../../dataprofiler", + f"../../dataprofiler/tests/", + ] +) + +update_index_rst = True + +if not version: + Exception("There must be a valid version argument.") + +# Check if the source index file has already been updated +source_index = open("source/index.rst", "r+") +source_index_lines = source_index.readlines() +source_index.close() +for sentence in source_index_lines: + if sentence.startswith("* `" + version): + update_index_rst = False + +# Update the index file if needed +version_reference = "" +if update_index_rst: + buffer = 0 + source_index = open("source/index.rst", "w") + for sentence in source_index_lines: + if sentence.startswith("Documentation for"): + doc_version = "Documentation for " + version + "\n" + source_index.write(doc_version) + elif sentence.startswith("Versions"): + source_index.write("Versions\n") + source_index.write("========\n") + version_tag = "* `" + version + "`_\n" + source_index.write(version_tag) + version_reference = ( + ".. _" + version + ": ../../" + version + "/html/index.html\n\n" + ) + buffer = 1 + else: + if buffer == 0: + source_index.write(sentence) + else: + buffer = buffer - 1 + source_index.write(version_reference) +source_index.close() + +# Make the html files + +build_directory = "BUILDDIR= LATEST" +subprocess.run(["make", "html", build_directory]) + +# update the index file to redirect to the most current version of documentation +index_file = open("../index.html", "w") +redirect_link = ( + '' +) +index_file.write(redirect_link) +index_file.close() + +# update the profiler_options.html file to redirect to detailed options docs +index_file = open("../profiler_options.html", "w") +redirect_link = ( + '' +) +index_file.write(redirect_link) +index_file.close() diff --git a/_docs/index.html b/_docs/index.html new file mode 100644 index 000000000..fb51eaca9 --- /dev/null +++ b/_docs/index.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/_docs/profiler_options.html b/_docs/profiler_options.html new file mode 100644 index 000000000..831f653ff --- /dev/null +++ b/_docs/profiler_options.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/_docs/setup.cfg b/_docs/setup.cfg new file mode 100644 index 000000000..c9c21e52f --- /dev/null +++ b/_docs/setup.cfg @@ -0,0 +1,7 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203 + +[isort] +multi_line_output=3 +profile=black diff --git a/dataprofiler/__init__.py b/dataprofiler/__init__.py index 2e89d3e2b..6b29c5b7a 100644 --- a/dataprofiler/__init__.py +++ b/dataprofiler/__init__.py @@ -1,5 +1,7 @@ """Package for dataprofiler.""" + from . import settings +from ._version import get_versions from .data_readers.data import Data from .dp_logging import get_logger, set_verbosity from .labelers.data_labelers import ( @@ -18,23 +20,8 @@ from .profilers.profiler_options import ProfilerOptions from .reports import graphs from .validators.base_validators import Validator -from .version import __version__ - -try: - import snappy -except ImportError: - import warnings - warnings.warn( - "Snappy must be installed to use parquet/avro datasets." - "\n\n" - "For macOS use Homebrew:\n" - "\t`brew install snappy`" - "\n\n" - "For linux use apt-get:\n`" - "\tsudo apt-get -y install libsnappy-dev`\n", - ImportWarning, - ) +__version__ = get_versions()["version"] def set_seed(seed=None): diff --git a/dataprofiler/_typing.py b/dataprofiler/_typing.py index fa362d1b1..b7a623881 100644 --- a/dataprofiler/_typing.py +++ b/dataprofiler/_typing.py @@ -1,4 +1,5 @@ """Contains typing aliases.""" + from typing import Dict, List, NewType, Union import numpy as np diff --git a/dataprofiler/_version.py b/dataprofiler/_version.py new file mode 100644 index 000000000..e26962f44 --- /dev/null +++ b/dataprofiler/_version.py @@ -0,0 +1,525 @@ +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.19 (https://github.com/python-versioneer/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys +from typing import Any, Callable + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + git_date = "$Format:%ci$" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "dataprofiler-" + cfg.versionfile_source = "dataprofiler/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} # type: ignore +HANDLERS: dict[str, dict[str, Callable[..., Any]]] = {} + + +def register_vcs_handler(vcs, method): # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except OSError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print(f"unable to find command, tried {commands}") + return None, None + stdout = p.communicate()[0].strip().decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs) + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except OSError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = {r.strip() for r in refnames.strip("()").split(",")} + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = {r for r in refs if re.search(r'\d', r)} + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post0.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post0.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post0.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} diff --git a/dataprofiler/data_readers/avro_data.py b/dataprofiler/data_readers/avro_data.py index 720b9d1f7..7f15bdec4 100644 --- a/dataprofiler/data_readers/avro_data.py +++ b/dataprofiler/data_readers/avro_data.py @@ -1,4 +1,5 @@ """Contains class for saving and loading spreadsheet data.""" + from io import BytesIO, StringIO from typing import Any, Dict, List, Optional, Union diff --git a/dataprofiler/data_readers/base_data.py b/dataprofiler/data_readers/base_data.py index 27d8d5deb..3dd32b219 100644 --- a/dataprofiler/data_readers/base_data.py +++ b/dataprofiler/data_readers/base_data.py @@ -1,4 +1,5 @@ """Contains abstract class for data loading and saving.""" + import locale import sys from collections import OrderedDict @@ -154,7 +155,7 @@ def get_batch_generator( ) -> Generator[Union[pd.DataFrame, List], None, None]: """Get batch generator.""" data_length = len(self.data) - indices = np.random.permutation(data_length) + indices: np.ndarray = np.random.permutation(data_length) for i in range(0, data_length, batch_size): if isinstance(self.data, pd.DataFrame): yield self.data.iloc[indices[i : i + batch_size], :] diff --git a/dataprofiler/data_readers/csv_data.py b/dataprofiler/data_readers/csv_data.py index 7e13d4075..49d1c6a78 100644 --- a/dataprofiler/data_readers/csv_data.py +++ b/dataprofiler/data_readers/csv_data.py @@ -1,4 +1,5 @@ """Contains class that saves and loads spreadsheet data.""" + import csv import random import re @@ -673,8 +674,10 @@ def is_match(cls, file_path: str, options: Optional[Dict] = None) -> bool: empty_line_count = 0 delimiter_count = dict() - delimiter_regex = data_utils.get_delimiter_regex(delimiter, quotechar) - space_regex = data_utils.get_delimiter_regex(" ", quotechar) + delimiter_regex = data_utils.get_delimiter_regex( + delimiter or ",", quotechar or '"' + ) + space_regex = data_utils.get_delimiter_regex(" ", quotechar or '"') # Count the possible delimiters for line in data_as_str.split("\n")[header:]: @@ -721,15 +724,19 @@ def is_match(cls, file_path: str, options: Optional[Dict] = None) -> bool: active_line_count - max_deviation_count ) / active_line_count - delimiter_count_values = np.array(list(delimiter_count.values())) - count_percent = delimiter_count_values / np.sum(delimiter_count_values) + delimiter_count_values = cast( + np.ndarray, np.array(list(delimiter_count.values())) + ) + count_percent = cast( + np.ndarray, delimiter_count_values / np.sum(delimiter_count_values) + ) if not count_percent.size: return False - max_count_index = count_percent.argmax() + max_count_index = int(count_percent.argmax()) max_count_value = list(delimiter_count.keys())[max_count_index] - max_count_percent = count_percent[max_count_index] + max_count_percent = float(count_percent[max_count_index]) # Inferred the file was a CSV if (max_count_value > 0 or delimiter is None) and ( diff --git a/dataprofiler/data_readers/data_utils.py b/dataprofiler/data_readers/data_utils.py index 611d25dc3..99aae4122 100644 --- a/dataprofiler/data_readers/data_utils.py +++ b/dataprofiler/data_readers/data_utils.py @@ -1,10 +1,12 @@ """Contains functions for data readers.""" + import json import logging import os import random import re import urllib +import urllib.parse from collections import OrderedDict from io import BytesIO, StringIO, TextIOWrapper from itertools import islice @@ -24,12 +26,12 @@ import boto3 import botocore -import dateutil import numpy as np import pandas as pd import pyarrow.parquet as pq import requests from chardet.universaldetector import UniversalDetector +from dateutil import parser as dateutil_parser # type: ignore[import-untyped] from typing_extensions import TypeGuard from .. import dp_logging, rng_utils @@ -276,7 +278,7 @@ def read_json( return lines -def reservoir(file: TextIOWrapper, sample_nrows: int) -> list: +def reservoir(file: TextIOWrapper | StringIO, sample_nrows: int) -> list: """ Implement the mathematical logic of Reservoir sampling. @@ -334,7 +336,7 @@ def reservoir(file: TextIOWrapper, sample_nrows: int) -> list: except StopIteration: break # Append new, replace old with dummy, and keep track of order - remove_index = rng.integers(0, sample_nrows) + remove_index = int(rng.integers(0, sample_nrows)) values[indices[remove_index]] = str(None) indices[remove_index] = len(values) values.append(newval) @@ -343,7 +345,9 @@ def reservoir(file: TextIOWrapper, sample_nrows: int) -> list: return values -def rsample(file_path: TextIOWrapper, sample_nrows: int, args: dict) -> StringIO: +def rsample( + file_path: TextIOWrapper | StringIO, sample_nrows: int, args: dict +) -> StringIO: """ Implement Reservoir Sampling to sample n rows out of a total of M rows. @@ -368,7 +372,7 @@ def rsample(file_path: TextIOWrapper, sample_nrows: int, args: dict) -> StringIO def read_csv_df( - file_path: Union[str, BytesIO, TextIOWrapper], + file_path: Union[str, StringIO, BytesIO, TextIOWrapper], delimiter: Optional[str], header: Optional[int], sample_nrows: Optional[int] = None, @@ -424,7 +428,7 @@ def read_csv_df( file_path = open(file_path, encoding=encoding) is_file_open = True - file_data = file_path + file_data: TextIOWrapper | StringIO = file_path if sample_nrows: file_data = rsample(file_path, sample_nrows, args) fo = pd.read_csv(file_data, **args) @@ -502,10 +506,10 @@ def sample_parquet( # sample n_rows = parquet_table.num_rows if n_rows > sample_nrows: - sample_index = np.array([False] * n_rows) + sample_index = cast(np.ndarray, np.array([False] * n_rows)) sample_index[random.sample(range(n_rows), sample_nrows)] = True else: - sample_index = np.array([True] * n_rows) + sample_index = cast(np.ndarray, np.array([True] * n_rows)) sample_df = parquet_table.filter(sample_index).to_pandas() # Convert all the unicode columns to utf-8 @@ -673,7 +677,7 @@ def _decode_is_valid(encoding): # If no encoding is still found, default to utf-8 if not encoding: encoding = "utf-8" - return encoding.lower() + return str(encoding).lower() def detect_cell_type(cell: str) -> str: @@ -690,7 +694,7 @@ def detect_cell_type(cell: str) -> str: try: # need to ingore type bc https://github.com/python/mypy/issues/8878 - if dateutil.parser.parse(cell, fuzzy=False): # type:ignore + if dateutil_parser.parse(cell, fuzzy=False): # type: ignore[attr-defined] cell_type = "date" except (ValueError, OverflowError, TypeError): pass diff --git a/dataprofiler/data_readers/filepath_or_buffer.py b/dataprofiler/data_readers/filepath_or_buffer.py index 56c21e28e..860eb037b 100644 --- a/dataprofiler/data_readers/filepath_or_buffer.py +++ b/dataprofiler/data_readers/filepath_or_buffer.py @@ -1,4 +1,5 @@ """Contains functions and classes for handling filepaths and buffers.""" + from io import BytesIO, StringIO, TextIOWrapper from typing import IO, Any, Optional, Type, Union, cast @@ -92,7 +93,7 @@ def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None: TextIOWrapper, self._filepath_or_buffer ) # guaranteed by self._is_wrapped wrapper = self._filepath_or_buffer - self._filepath_or_buffer = wrapper.buffer + self._filepath_or_buffer = cast(BytesIO, wrapper.buffer) wrapper.detach() if isinstance(self._filepath_or_buffer, (StringIO, BytesIO)): diff --git a/dataprofiler/data_readers/graph_data.py b/dataprofiler/data_readers/graph_data.py index 337408a68..3cc83b040 100644 --- a/dataprofiler/data_readers/graph_data.py +++ b/dataprofiler/data_readers/graph_data.py @@ -1,4 +1,5 @@ """Contains class for identifying, reading, and loading graph data.""" + import csv from typing import Dict, List, Optional, Union, cast diff --git a/dataprofiler/data_readers/json_data.py b/dataprofiler/data_readers/json_data.py index 93e5d7e6f..cc71c57e4 100644 --- a/dataprofiler/data_readers/json_data.py +++ b/dataprofiler/data_readers/json_data.py @@ -1,4 +1,5 @@ """Contains class to save and load json data.""" + import json import re import warnings @@ -71,9 +72,9 @@ def __init__( self._data_formats["records"] = self._get_data_as_records self._data_formats["json"] = self._get_data_as_json - self._data_formats[ - "flattened_dataframe" - ] = self._get_data_as_flattened_dataframe + self._data_formats["flattened_dataframe"] = ( + self._get_data_as_flattened_dataframe + ) self._selected_data_format: str = options.get( "data_format", "flattened_dataframe" ) diff --git a/dataprofiler/data_readers/parquet_data.py b/dataprofiler/data_readers/parquet_data.py index 4fa567b8d..2e66d1a0a 100644 --- a/dataprofiler/data_readers/parquet_data.py +++ b/dataprofiler/data_readers/parquet_data.py @@ -1,4 +1,5 @@ """Contains class to save and load parquet data.""" + from io import BytesIO, StringIO from typing import Any, Dict, List, Optional, Union @@ -67,7 +68,7 @@ def __init__( self._load_data(data) @property - def file_encoding(self) -> None: + def file_encoding(self) -> Optional[str]: """Set file encoding to None since not detected for avro.""" return None diff --git a/dataprofiler/data_readers/structured_mixins.py b/dataprofiler/data_readers/structured_mixins.py index 3587291f1..6b1da157d 100644 --- a/dataprofiler/data_readers/structured_mixins.py +++ b/dataprofiler/data_readers/structured_mixins.py @@ -1,4 +1,5 @@ """Contains mixin data class for loading datasets of tye SpreadSheet.""" + from logging import Logger from typing import Any, Dict, List, Optional, Union, cast diff --git a/dataprofiler/labelers/__init__.py b/dataprofiler/labelers/__init__.py index 1b2302fcf..a355ead23 100644 --- a/dataprofiler/labelers/__init__.py +++ b/dataprofiler/labelers/__init__.py @@ -26,6 +26,7 @@ 2. structured_model 3. regex_model """ + # import data labelers # import models from .base_data_labeler import BaseDataLabeler, TrainableDataLabeler diff --git a/dataprofiler/labelers/base_data_labeler.py b/dataprofiler/labelers/base_data_labeler.py index 201f78998..4547b592a 100644 --- a/dataprofiler/labelers/base_data_labeler.py +++ b/dataprofiler/labelers/base_data_labeler.py @@ -1,4 +1,5 @@ """Contains abstract classes from which labeler classes will inherit.""" + from __future__ import annotations import json @@ -9,15 +10,14 @@ import numpy as np import pandas as pd -import pkg_resources from dataprofiler._typing import DataArray from .. import data_readers -from . import data_processing +from . import data_processing, utils from .base_model import BaseModel -default_labeler_dir = pkg_resources.resource_filename("resources", "labelers") +default_labeler_dir = str(utils.find_resources_dir("labelers")) class BaseDataLabeler: @@ -246,7 +246,8 @@ def set_params(self, params: dict) -> None: self._postprocessor.set_params(**params["postprocessor"]) self.check_pipeline( - skip_postprocessor=self._postprocessor is None, error_on_mismatch=False + skip_postprocessor=self._postprocessor is None, + error_on_mismatch=False, ) def add_label(self, label: str, same_as: str = None) -> None: @@ -438,7 +439,9 @@ def get_parameter_overlap_mismatches( messages.append( "Preprocessor and postprocessor value for `{}` do not " "match. {} != {}".format( - param, preprocessor_params[param], postprocessor_params[param] + param, + preprocessor_params[param], + postprocessor_params[param], ) ) if messages: @@ -490,7 +493,8 @@ def _load_parameters(dirpath: str, load_options: dict = None) -> dict[str, dict] "The load_options preprocessor class does not " "match the required DataLabeler preprocessor." "\n {} != {}".format( - processor_class.__class__.__name__, param_processor_class + processor_class.__class__.__name__, + param_processor_class, ) ) params["preprocessor"]["class"] = load_options.get("preprocessor_class") @@ -505,7 +509,8 @@ def _load_parameters(dirpath: str, load_options: dict = None) -> dict[str, dict] raise ValueError( "The load_options postprocessor class does not match " "the required DataLabeler postprocessor.\n {} != {}".format( - processor_class.__class__.__name__, param_processor_class + processor_class.__class__.__name__, + param_processor_class, ) ) params["postprocessor"]["class"] = load_options.get("postprocessor_class") diff --git a/dataprofiler/labelers/base_model.py b/dataprofiler/labelers/base_model.py index 032c2ea38..c5d7aef50 100644 --- a/dataprofiler/labelers/base_model.py +++ b/dataprofiler/labelers/base_model.py @@ -1,4 +1,5 @@ """Contains abstract classes for labeling data.""" + from __future__ import annotations import abc @@ -78,7 +79,7 @@ def __eq__(self, other: object) -> bool: :rtype: bool """ if ( - type(self) != type(other) + type(self) is not type(other) or not isinstance(other, BaseModel) or self._parameters != other._parameters or self._label_mapping != other._label_mapping diff --git a/dataprofiler/labelers/char_load_tf_model.py b/dataprofiler/labelers/char_load_tf_model.py index b168e9234..8adbfa0bd 100644 --- a/dataprofiler/labelers/char_load_tf_model.py +++ b/dataprofiler/labelers/char_load_tf_model.py @@ -1,4 +1,5 @@ """Contains class for training data labeler model.""" + from __future__ import annotations import copy @@ -17,6 +18,7 @@ from .. import dp_logging from . import labeler_utils from .base_model import AutoSubRegistrationMeta, BaseModel, BaseTrainableModel +from .character_level_cnn_model import ArgMaxLayer _file_dir = os.path.dirname(os.path.abspath(__file__)) @@ -29,6 +31,8 @@ class CharLoadTFModel(BaseTrainableModel, metaclass=AutoSubRegistrationMeta): # boolean if the label mapping requires the mapping for index 0 reserved requires_zero_mapping = False + _SOFTMAX_OUTPUT = "softmax_output" + _ARGMAX_OUTPUT = "argmax_output" def __init__( self, model_path: str, label_mapping: dict[str, int], parameters: dict = None @@ -61,6 +65,35 @@ def __init__( BaseModel.__init__(self, label_mapping, parameters) + @classmethod + def _create_model_outputs( + cls, softmax_output: tf.Tensor, argmax_output: tf.Tensor | None = None + ) -> dict[str, tf.Tensor]: + """Return normalized dict outputs for training and inference.""" + if argmax_output is None: + argmax_output = ArgMaxLayer(name=cls._ARGMAX_OUTPUT)(softmax_output) + return { + cls._SOFTMAX_OUTPUT: softmax_output, + cls._ARGMAX_OUTPUT: argmax_output, + } + + @classmethod + def _normalize_model_outputs(cls, model: tf.keras.Model) -> tf.keras.Model: + """Convert list-style outputs to the normalized dict structure.""" + return labeler_utils.normalize_tf_model_outputs( + model, + [cls._SOFTMAX_OUTPUT, cls._ARGMAX_OUTPUT], + lambda softmax_output, extra_outputs: cls._create_model_outputs( + softmax_output, extra_outputs[0] + ), + ) + + def _new_softmax_head_name(self) -> str: + """Return a layer name unique within the current model graph.""" + return labeler_utils.get_tf_rebuild_layer_name( + self._model, f"{self._SOFTMAX_OUTPUT}_rebuild" + ) + def __eq__(self, other: object) -> bool: """ Check if two models are equal with one another. @@ -215,15 +248,34 @@ def load_from_disk(cls, dirpath: str) -> CharLoadTFModel: tf_model = tf.keras.models.load_model(dirpath) loaded_model = cls(dirpath, label_mapping, parameters) - loaded_model._model = tf_model + loaded_model._model = cls._normalize_model_outputs(tf_model) # load self loaded_model._model_num_labels = loaded_model.num_labels loaded_model._model_default_ind = loaded_model.label_mapping[ loaded_model._parameters["default_label"] ] + loaded_model._compile_model(loaded_model.num_labels) return loaded_model + def _compile_model(self, num_labels: int) -> None: + """Compile the model with dict-based losses and metrics.""" + losses = { + self._SOFTMAX_OUTPUT: "categorical_crossentropy", + self._ARGMAX_OUTPUT: None, + } + f1_score_training = labeler_utils.F1Score( + num_classes=num_labels, average="micro" + ) + metrics = { + self._SOFTMAX_OUTPUT: [ + "categorical_crossentropy", + "acc", + f1_score_training, + ] + } + self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + def _construct_model(self) -> None: """ Model constructor for the data labeler. @@ -237,38 +289,28 @@ def _construct_model(self) -> None: model_loc = self._parameters["model_path"] self._model: tf.keras.Model = tf.keras.models.load_model(model_loc) - softmax_output_layer_name = self._model.outputs[0].name.split("/")[0] + self._model = self._normalize_model_outputs(self._model) + softmax_output = self._model.output[self._SOFTMAX_OUTPUT] + softmax_layer = softmax_output._keras_history[0] + softmax_output_layer_name = softmax_layer.name softmax_layer_ind = cast( int, labeler_utils.get_tf_layer_index_from_name( self._model, softmax_output_layer_name ), ) - softmax_layer = self._model.get_layer(softmax_output_layer_name) - new_softmax_layer = softmax_layer.output + new_softmax_layer = softmax_output if softmax_layer.weights[0].shape[-1] != num_labels: new_softmax_layer = tf.keras.layers.Dense( - num_labels, activation="softmax", name="softmax_output" + num_labels, + activation="softmax", + name=self._new_softmax_head_name(), )(self._model.layers[softmax_layer_ind - 1].output) - # Output the model into a .pb file for TensorFlow - argmax_layer = tf.keras.backend.argmax(new_softmax_layer) - - argmax_outputs = [new_softmax_layer, argmax_layer] - self._model = tf.keras.Model(self._model.inputs, argmax_outputs) - - # Compile the model w/ metrics - softmax_output_layer_name = self._model.outputs[0].name.split("/")[0] - losses = {softmax_output_layer_name: "categorical_crossentropy"} - - # use f1 score metric - f1_score_training = labeler_utils.F1Score( - num_classes=num_labels, average="micro" - ) - metrics = {softmax_output_layer_name: ["acc", f1_score_training]} - - self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + output_dict = self._create_model_outputs(new_softmax_layer) + self._model = tf.keras.Model(self._model.inputs, output_dict) + self._compile_model(num_labels) self._epoch_id = 0 self._model_num_labels = num_labels @@ -294,32 +336,17 @@ def _reconstruct_model(self) -> None: num_labels = self.num_labels default_ind = self.label_mapping[self._parameters["default_label"]] - # Remove the 2 output layers ('softmax', 'tf_op_layer_ArgMax') - for _ in range(2): - self._model.layers.pop() - # Add the final Softmax layer to the previous spot + # self._model.layers[-2] to skip: original softmax final_softmax_layer = tf.keras.layers.Dense( - num_labels, activation="softmax", name="softmax_output" - )(self._model.layers[-4].output) - - # Output the model into a .pb file for TensorFlow - argmax_layer = tf.keras.backend.argmax(final_softmax_layer) + num_labels, + activation="softmax", + name=self._new_softmax_head_name(), + )(self._model.layers[-2].output) - argmax_outputs = [final_softmax_layer, argmax_layer] - self._model = tf.keras.Model(self._model.inputs, argmax_outputs) - - # Compile the model - softmax_output_layer_name = self._model.outputs[0].name.split("/")[0] - losses = {softmax_output_layer_name: "categorical_crossentropy"} - - # use f1 score metric - f1_score_training = labeler_utils.F1Score( - num_classes=num_labels, average="micro" - ) - metrics = {softmax_output_layer_name: ["acc", f1_score_training]} - - self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + output_dict = self._create_model_outputs(final_softmax_layer) + self._model = tf.keras.Model(self._model.inputs, output_dict) + self._compile_model(num_labels) self._epoch_id = 0 self._model_num_labels = num_labels @@ -370,34 +397,53 @@ def fit( f1_report: dict = {} self._model.reset_metrics() - softmax_output_layer_name = self._model.outputs[0].name.split("/")[0] start_time = time.time() batch_id = 0 for x_train, y_train in train_data: + y_train_dict = { + self._SOFTMAX_OUTPUT: y_train, + self._ARGMAX_OUTPUT: None, + } model_results = self._model.train_on_batch( - x_train, {softmax_output_layer_name: y_train} + x_train, + y_train_dict, + return_dict=True, + ) + acc_value = next( + (value for key, value in model_results.items() if key.endswith("acc")), + np.nan, + ) + f1_value = next( + (value for key, value in model_results.items() if "f1" in key.lower()), + np.nan, ) sys.stdout.flush() if verbose: sys.stdout.write( "\rEPOCH %d, batch_id %d: loss: %f - acc: %f - " - "f1_score %f" % (self._epoch_id, batch_id, *model_results[1:]) + "f1_score %f" + % ( + self._epoch_id, + batch_id, + model_results.get("loss", np.nan), + acc_value, + f1_value, + ) ) batch_id += 1 - for i, metric_label in enumerate(self._model.metrics_names): - history[metric_label] = model_results[i] + history.update(model_results) if val_data: f1, f1_report = self._validate_training(val_data) # type: ignore history["f1_report"] = f1_report - val_f1 = f1_report["weighted avg"]["f1-score"] if f1_report else np.NAN + val_f1 = f1_report["weighted avg"]["f1-score"] if f1_report else np.nan val_precision = ( - f1_report["weighted avg"]["precision"] if f1_report else np.NAN + f1_report["weighted avg"]["precision"] if f1_report else np.nan ) - val_recall = f1_report["weighted avg"]["recall"] if f1_report else np.NAN + val_recall = f1_report["weighted avg"]["recall"] if f1_report else np.nan epoch_time = time.time() - start_time logger.info( "\rEPOCH %d (%ds), loss: %f - acc: %f - f1_score %f -- " @@ -405,7 +451,9 @@ def fit( % ( self._epoch_id, epoch_time, - *model_results[1:], + model_results.get("loss", np.nan), + acc_value, + f1_value, val_f1, val_precision, val_recall, @@ -452,7 +500,7 @@ def _validate_training( y_val_pred.append( self._model.predict( x_val, batch_size=batch_size_test, verbose=verbose_keras - )[1] + )[self._ARGMAX_OUTPUT] ) y_val_test.append(np.argmax(y_val, axis=-1)) batch_id += 1 @@ -525,10 +573,10 @@ def predict( if show_confidences: confidences[ allocation_index : allocation_index + num_samples_in_batch - ] = model_output[0].numpy() - predictions[ - allocation_index : allocation_index + num_samples_in_batch - ] = model_output[1].numpy() + ] = model_output[self._SOFTMAX_OUTPUT].numpy() + predictions[allocation_index : allocation_index + num_samples_in_batch] = ( + model_output[self._ARGMAX_OUTPUT].numpy() + ) allocation_index += num_samples_in_batch diff --git a/dataprofiler/labelers/character_level_cnn_model.py b/dataprofiler/labelers/character_level_cnn_model.py index 3194a2616..bc30dd035 100644 --- a/dataprofiler/labelers/character_level_cnn_model.py +++ b/dataprofiler/labelers/character_level_cnn_model.py @@ -1,4 +1,5 @@ """Contains classes for char data labeling.""" + from __future__ import annotations import copy @@ -7,6 +8,7 @@ import sys import time from collections import defaultdict +from typing import cast import numpy as np import tensorflow as tf @@ -74,11 +76,155 @@ def create_glove_char(n_dims: int, source_file: str = None) -> None: file.write(word + " " + " ".join(str(num) for num in embd) + "\n") +@tf.keras.utils.register_keras_serializable(package="CharacterLevelCnnModel") +class ThreshArgMaxLayer(tf.keras.layers.Layer): + """Keras layer applying a thresholded argmax.""" + + def __init__( + self, threshold_: float, num_labels_: int, default_ind: int = 1, *args, **kwargs + ) -> None: + """Apply a minimum threshold to the argmax value. + + When below this threshold the index will be the default. + + :param num_labels: number of entities + :type num_labels: int + :param threshold: default set to 0 so all confidences pass. + :type threshold: float + :param default_ind: default index + :type default_ind: int + :return: final argmax threshold layer for the model + :return : tensor containing argmax thresholded integers, labels out + :rtype: tf.Tensor + """ + super().__init__(*args, **kwargs) + self._threshold_ = threshold_ + self._num_labels_ = num_labels_ + self._default_ind = default_ind + thresh_init = tf.constant_initializer(threshold_) + self.thresh_vec = tf.Variable( + name="ThreshVec", + initial_value=thresh_init(shape=[num_labels_]), + trainable=False, + ) + + def get_config(self): + """Return a serializable config for saving the layer.""" + config = super().get_config().copy() + config.update( + { + "threshold_": self._threshold_, + "num_labels_": self._num_labels_, + "default_ind": self._default_ind, + } + ) + return config + + def call(self, argmax_layer: tf.Tensor, confidence_layer: tf.Tensor) -> tf.Tensor: + """Apply the threshold argmax to the input tensor.""" + argmax_layer = tf.cast(argmax_layer, tf.int64) + threshold_at_argmax = tf.gather(self.thresh_vec, argmax_layer) + + confidence_max_layer = tf.keras.backend.max(confidence_layer, axis=2) + + # Check if the confidences meet the threshold minimum. + argmax_mask = tf.keras.backend.cast( + tf.keras.backend.greater_equal(confidence_max_layer, threshold_at_argmax), + dtype=argmax_layer.dtype, + ) + + # Create a vector the same size as the batch_size which + # represents the background label + bg_label_tf = tf.keras.backend.constant( + self._default_ind, dtype=argmax_layer.dtype + ) + + # Generate the final predicted output using the function: + final_predicted_layer = tf.add( + bg_label_tf, + tf.multiply(tf.subtract(argmax_layer, bg_label_tf), argmax_mask), + name="ThreshArgMax", + ) + # final_predicted_layer.set_shape(argmax_layer.shape) + return final_predicted_layer + + +@tf.keras.utils.register_keras_serializable(package="CharacterLevelCnnModel") +class ArgMaxLayer(tf.keras.layers.Layer): + """Keras layer returning integer argmax indices.""" + + def call(self, confidence_layer: tf.Tensor) -> tf.Tensor: + """Return argmax indices as int64.""" + return tf.cast(tf.keras.ops.argmax(confidence_layer, axis=2), tf.int64) + + def compute_output_shape(self, input_shape): + """Return the confidence tensor shape without the class dimension.""" + return input_shape[:-1] + + +@tf.keras.utils.register_keras_serializable(package="CharacterLevelCnnModel") +class EncodingLayer(tf.keras.layers.Layer): + """Encodes strings to integers.""" + + def __init__( + self, max_char_encoding_id: int, max_len: int, *args, **kwargs + ) -> None: + """ + Encode characters for the list of sentences. + + :param max_char_encoding_id: Maximum integer value for encoding the + input + :type max_char_encoding_id: int + :param max_len: Maximum char length in a sample + :type max_len: int + """ + super().__init__(*args, **kwargs) + self.max_char_encoding_id = max_char_encoding_id + self.max_len = max_len + + def get_config(self): + """Return a serializable config for saving the layer.""" + config = super().get_config().copy() + config.update( + { + "max_char_encoding_id": self.max_char_encoding_id, + "max_len": self.max_len, + } + ) + return config + + def call(self, input_str_tensor: tf.Tensor) -> tf.Tensor: + """ + Encode characters for the list of sentences. + + :param input_str_tensor: input list of sentences converted to tensor + :type input_str_tensor: tf.tensor + :return : tensor containing encoded list of input sentences + :rtype: tf.Tensor + """ + # convert characters to indices + input_str_flatten = tf.reshape(input_str_tensor, [-1]) + sentences_encode = tf.strings.unicode_decode( + input_str_flatten, input_encoding="UTF-8" + ) + sentences_encode = tf.add(tf.cast(1, tf.int32), sentences_encode) + sentences_encode = tf.math.minimum( + sentences_encode, self.max_char_encoding_id + 1 + ) + + # padding + sentences_encode_pad = sentences_encode.to_tensor(shape=[None, self.max_len]) + return sentences_encode_pad + + class CharacterLevelCnnModel(BaseTrainableModel, metaclass=AutoSubRegistrationMeta): """Class for training char data labeler.""" # boolean if the label mapping requires the mapping for index 0 reserved requires_zero_mapping: bool = True + _SOFTMAX_OUTPUT = "softmax_output" + _ARGMAX_OUTPUT = "argmax_output" + _THRESH_OUTPUT = "thresh_argmax_output" def __init__(self, label_mapping: dict[str, int], parameters: dict = None) -> None: """ @@ -115,6 +261,54 @@ def __init__(self, label_mapping: dict[str, int], parameters: dict = None) -> No BaseModel.__init__(self, label_mapping, parameters) + @classmethod + def _create_model_outputs( + cls, + softmax_output: tf.Tensor, + default_ind: int, + num_labels: int, + argmax_output: tf.Tensor | None = None, + threshold_output: tf.Tensor | None = None, + ) -> dict[str, tf.Tensor]: + """Return normalized dict outputs for training and inference.""" + if argmax_output is None: + argmax_output = ArgMaxLayer(name=cls._ARGMAX_OUTPUT)(softmax_output) + if threshold_output is None: + threshold_output = ThreshArgMaxLayer( + threshold_=0.0, + num_labels_=num_labels, + default_ind=default_ind, + name=cls._THRESH_OUTPUT, + )(argmax_output, softmax_output) + return { + cls._SOFTMAX_OUTPUT: softmax_output, + cls._ARGMAX_OUTPUT: argmax_output, + cls._THRESH_OUTPUT: threshold_output, + } + + @classmethod + def _normalize_model_outputs( + cls, model: tf.keras.Model, default_ind: int, num_labels: int + ) -> tf.keras.Model: + """Convert list-style outputs to the normalized dict structure.""" + return labeler_utils.normalize_tf_model_outputs( + model, + [cls._SOFTMAX_OUTPUT, cls._ARGMAX_OUTPUT, cls._THRESH_OUTPUT], + lambda softmax_output, extra_outputs: cls._create_model_outputs( + softmax_output, + default_ind, + num_labels, + extra_outputs[0], + extra_outputs[1], + ), + ) + + def _new_softmax_head_name(self) -> str: + """Return a layer name unique within the current model graph.""" + return labeler_utils.get_tf_rebuild_layer_name( + self._model, f"{self._SOFTMAX_OUTPUT}_rebuild" + ) + def __eq__(self, other: object) -> bool: """ Check if two models are equal with one another. @@ -280,7 +474,7 @@ def save_to_disk(self, dirpath: str) -> None: labels_dirpath = os.path.join(dirpath, "label_mapping.json") with open(labels_dirpath, "w") as fp: json.dump(self.label_mapping, fp) - self._model.save(os.path.join(dirpath)) + self._model.save(os.path.join(dirpath, "model.keras")) @classmethod def load_from_disk(cls, dirpath: str) -> CharacterLevelCnnModel: @@ -301,15 +495,7 @@ def load_from_disk(cls, dirpath: str) -> CharacterLevelCnnModel: with open(labels_dirpath) as fp: label_mapping = json.load(fp) - # use f1 score metric - custom_objects = { - "F1Score": labeler_utils.F1Score( - num_classes=max(label_mapping.values()) + 1, average="micro" - ), - "CharacterLevelCnnModel": cls, - } - with tf.keras.utils.custom_object_scope(custom_objects): - tf_model = tf.keras.models.load_model(dirpath) + tf_model = tf.keras.models.load_model(os.path.join(dirpath, "model.keras")) loaded_model = cls(label_mapping, parameters) loaded_model._model = tf_model @@ -325,43 +511,21 @@ def load_from_disk(cls, dirpath: str) -> CharacterLevelCnnModel: loaded_model._construct_model() tf1_weights.append(loaded_model._model.weights[-1].value()) loaded_model._model.set_weights(tf1_weights) + else: + loaded_model._model = cls._normalize_model_outputs( + tf_model, + loaded_model.label_mapping[loaded_model._parameters["default_label"]], + loaded_model.num_labels, + ) # load self loaded_model._model_num_labels = loaded_model.num_labels loaded_model._model_default_ind = loaded_model.label_mapping[ loaded_model._parameters["default_label"] ] + loaded_model._compile_loss(loaded_model._model, loaded_model.num_labels) return loaded_model - @staticmethod - def _char_encoding_layer( - input_str_tensor: tf.Tensor, max_char_encoding_id: int, max_len: int - ) -> tf.Tensor: - """ - Encode characters for the list of sentences. - - :param input_str_tensor: input list of sentences converted to tensor - :type input_str_tensor: tf.tensor - :param max_char_encoding_id: Maximum integer value for encoding the - input - :type max_char_encoding_id: int - :param max_len: Maximum char length in a sample - :type max_len: int - :return : tensor containing encoded list of input sentences - :rtype: tf.Tensor - """ - # convert characters to indices - input_str_flatten = tf.reshape(input_str_tensor, [-1]) - sentences_encode = tf.strings.unicode_decode( - input_str_flatten, input_encoding="UTF-8" - ) - sentences_encode = tf.add(tf.cast(1, tf.int32), sentences_encode) - sentences_encode = tf.math.minimum(sentences_encode, max_char_encoding_id + 1) - - # padding - sentences_encode_pad = sentences_encode.to_tensor(shape=[None, max_len]) - return sentences_encode_pad - @staticmethod def _argmax_threshold_layer( num_labels: int, threshold: float = 0.0, default_ind: int = 1 @@ -383,47 +547,27 @@ def _argmax_threshold_layer( """ # Initialize the thresholds vector variable and create the threshold # matrix. - class ThreshArgMaxLayer(tf.keras.layers.Layer): - def __init__(self, threshold_: float, num_labels_: int) -> None: - super().__init__() - thresh_init = tf.constant_initializer(threshold_) - self.thresh_vec = tf.Variable( - name="ThreshVec", - initial_value=thresh_init(shape=[num_labels_]), - trainable=False, - ) - - def call( - self, argmax_layer: tf.Tensor, confidence_layer: tf.Tensor - ) -> tf.Tensor: - threshold_at_argmax = tf.gather(self.thresh_vec, argmax_layer) - - confidence_max_layer = tf.keras.backend.max(confidence_layer, axis=2) - - # Check if the confidences meet the threshold minimum. - argmax_mask = tf.keras.backend.cast( - tf.keras.backend.greater_equal( - confidence_max_layer, threshold_at_argmax - ), - dtype=argmax_layer.dtype, - ) - - # Create a vector the same size as the batch_size which - # represents the background label - bg_label_tf = tf.keras.backend.constant( - default_ind, dtype=argmax_layer.dtype - ) - - # Generate the final predicted output using the function: - final_predicted_layer = tf.add( - bg_label_tf, - tf.multiply(tf.subtract(argmax_layer, bg_label_tf), argmax_mask), - name="ThreshArgMax", - ) + return ThreshArgMaxLayer(threshold, num_labels, default_ind) - return final_predicted_layer - - return ThreshArgMaxLayer(threshold, num_labels) + @staticmethod + def _compile_loss(model: tf.keras.Model, num_labels: int) -> None: + """Compiles the loss for the given model and number of labels.""" + losses = { + CharacterLevelCnnModel._SOFTMAX_OUTPUT: "categorical_crossentropy", + CharacterLevelCnnModel._ARGMAX_OUTPUT: None, + CharacterLevelCnnModel._THRESH_OUTPUT: None, + } + f1_score_training = labeler_utils.F1Score( + num_classes=num_labels, average="micro" + ) + metrics = { + CharacterLevelCnnModel._SOFTMAX_OUTPUT: [ + "categorical_crossentropy", + "acc", + f1_score_training, + ] + } + model.compile(loss=losses, optimizer="adam", metrics=metrics) def _construct_model(self) -> None: """ @@ -449,17 +593,13 @@ def _construct_model(self) -> None: max_length = self._parameters["max_length"] max_char_encoding_id = self._parameters["max_char_encoding_id"] - # Encoding layer - def encoding_function(input_str: tf.Tensor) -> tf.Tensor: - char_in_vector = CharacterLevelCnnModel._char_encoding_layer( - input_str, max_char_encoding_id, max_length - ) - return char_in_vector - self._model.add(tf.keras.layers.Input(shape=(None,), dtype=tf.string)) self._model.add( - tf.keras.layers.Lambda(encoding_function, output_shape=tuple([max_length])) + EncodingLayer( + max_char_encoding_id=max_char_encoding_id, + max_len=max_length, + ), ) # Create a pre-trained weight matrix @@ -469,12 +609,12 @@ def encoding_function(input_str: tf.Tensor) -> tf.Tensor: _file_dir, "embeddings/glove-reduced-{}D.txt".format(self._parameters["dim_embed"]), ) - embedding_matrix = np.zeros( - (max_char_encoding_id + 2, self._parameters["dim_embed"]) + embedding_matrix = cast( + np.ndarray, + np.zeros((max_char_encoding_id + 2, self._parameters["dim_embed"])), ) embedding_dict = build_embd_dictionary(embed_file) - input_shape = tuple([max_length]) # Fill in the weight matrix: let pad and space be 0s for ascii_num in range(max_char_encoding_id): if chr(ascii_num) in embedding_dict: @@ -485,7 +625,6 @@ def encoding_function(input_str: tf.Tensor) -> tf.Tensor: max_char_encoding_id + 2, self._parameters["dim_embed"], weights=[embedding_matrix], - input_length=input_shape[0], trainable=True, ) ) @@ -502,8 +641,7 @@ def encoding_function(input_str: tf.Tensor) -> tf.Tensor: ) if self._parameters["dropout"]: self._model.add(tf.keras.layers.Dropout(self._parameters["dropout"])) - # Add batch normalization, set fused = True for compactness - self._model.add(tf.keras.layers.BatchNormalization(fused=False, scale=True)) + self._model.add(tf.keras.layers.BatchNormalization(scale=True)) # Add the fully connected layers for size in self._parameters["size_fc"]: @@ -512,33 +650,19 @@ def encoding_function(input_str: tf.Tensor) -> tf.Tensor: self._model.add(tf.keras.layers.Dropout(self._parameters["dropout"])) # Add the final Softmax layer - self._model.add(tf.keras.layers.Dense(num_labels, activation="softmax")) - - # Output the model into a .pb file for TensorFlow - argmax_layer = tf.keras.backend.argmax(self._model.output) - - # Create confidence layers - final_predicted_layer = CharacterLevelCnnModel._argmax_threshold_layer( - num_labels, threshold=0.0, default_ind=default_ind + self._model.add( + tf.keras.layers.Dense( + num_labels, + activation="softmax", + name=self._SOFTMAX_OUTPUT, + ) ) - argmax_outputs = self._model.outputs + [ - argmax_layer, - final_predicted_layer(argmax_layer, self._model.output), - ] - self._model = tf.keras.Model(self._model.inputs, argmax_outputs) - - # Compile the model - softmax_output_layer_name = self._model.outputs[0].name.split("/")[0] - losses = {softmax_output_layer_name: "categorical_crossentropy"} - - # use f1 score metric - f1_score_training = labeler_utils.F1Score( - num_classes=num_labels, average="micro" + output_dict = self._create_model_outputs( + self._model.outputs[0], default_ind, num_labels ) - metrics = {softmax_output_layer_name: ["acc", f1_score_training]} - - self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + self._model = tf.keras.Model(self._model.inputs, output_dict) + self._compile_loss(self._model, num_labels) self._epoch_id = 0 self._model_num_labels = num_labels @@ -564,41 +688,19 @@ def _reconstruct_model(self) -> None: num_labels = self.num_labels default_ind = self.label_mapping[self._parameters["default_label"]] - # Remove the 3 output layers (dense_2', 'tf_op_layer_ArgMax', - # 'thresh_arg_max_layer') - for _ in range(3): - self._model.layers.pop() - # Add the final Softmax layer to the previous spot + # self._model.layers[-3] to skip: thresh and original softmax final_softmax_layer = tf.keras.layers.Dense( - num_labels, activation="softmax", name="dense_2" - )(self._model.layers[-4].output) + num_labels, + activation="softmax", + name=self._new_softmax_head_name(), + )(self._model.layers[-3].output) - # Output the model into a .pb file for TensorFlow - argmax_layer = tf.keras.backend.argmax(final_softmax_layer) - - # Create confidence layers - final_predicted_layer = CharacterLevelCnnModel._argmax_threshold_layer( - num_labels, threshold=0.0, default_ind=default_ind + output_dict = self._create_model_outputs( + final_softmax_layer, default_ind, num_labels ) - - argmax_outputs = [final_softmax_layer] + [ - argmax_layer, - final_predicted_layer(argmax_layer, final_softmax_layer), - ] - self._model = tf.keras.Model(self._model.inputs, argmax_outputs) - - # Compile the model - softmax_output_layer_name = self._model.outputs[0].name.split("/")[0] - losses = {softmax_output_layer_name: "categorical_crossentropy"} - - # use f1 score metric - f1_score_training = labeler_utils.F1Score( - num_classes=num_labels, average="micro" - ) - metrics = {softmax_output_layer_name: ["acc", f1_score_training]} - - self._model.compile(loss=losses, optimizer="adam", metrics=metrics) + self._model = tf.keras.Model(self._model.inputs, output_dict) + self._compile_loss(self._model, num_labels) self._epoch_id = 0 self._model_num_labels = num_labels self._model_default_ind = default_ind @@ -648,34 +750,54 @@ def fit( f1_report: dict = {} self._model.reset_metrics() - softmax_output_layer_name = self._model.outputs[0].name.split("/")[0] start_time = time.time() batch_id = 0 for x_train, y_train in train_data: + y_train_dict = { + self._SOFTMAX_OUTPUT: y_train, + self._ARGMAX_OUTPUT: None, + self._THRESH_OUTPUT: None, + } model_results = self._model.train_on_batch( - x_train, {softmax_output_layer_name: y_train} + x_train, + y_train_dict, + return_dict=True, + ) + acc_value = next( + (value for key, value in model_results.items() if key.endswith("acc")), + np.nan, + ) + f1_value = next( + (value for key, value in model_results.items() if "f1" in key.lower()), + np.nan, ) sys.stdout.flush() if verbose: sys.stdout.write( "\rEPOCH %d, batch_id %d: loss: %f - acc: %f - " - "f1_score %f" % (self._epoch_id, batch_id, *model_results[1:]) + "f1_score %f" + % ( + self._epoch_id, + batch_id, + model_results.get("loss", np.nan), + acc_value, + f1_value, + ) ) batch_id += 1 - for i, metric_label in enumerate(self._model.metrics_names): - history[metric_label] = model_results[i] + history.update(model_results) if val_data: f1, f1_report = self._validate_training(val_data) # type: ignore history["f1_report"] = f1_report - val_f1 = f1_report["weighted avg"]["f1-score"] if f1_report else np.NAN + val_f1 = f1_report["weighted avg"]["f1-score"] if f1_report else np.nan val_precision = ( - f1_report["weighted avg"]["precision"] if f1_report else np.NAN + f1_report["weighted avg"]["precision"] if f1_report else np.nan ) - val_recall = f1_report["weighted avg"]["recall"] if f1_report else np.NAN + val_recall = f1_report["weighted avg"]["recall"] if f1_report else np.nan epoch_time = time.time() - start_time logger.info( "\rEPOCH %d (%ds), loss: %f - acc: %f - f1_score %f -- " @@ -683,7 +805,9 @@ def fit( % ( self._epoch_id, epoch_time, - *model_results[1:], + model_results.get("loss", np.nan), + acc_value, + f1_value, val_f1, val_precision, val_recall, @@ -729,8 +853,10 @@ def _validate_training( for x_val, y_val in val_data: y_val_pred.append( self._model.predict( - x_val, batch_size=batch_size_test, verbose=verbose_keras - )[1] + tf.convert_to_tensor(x_val), + batch_size=batch_size_test, + verbose=verbose_keras, + )[self._ARGMAX_OUTPUT] ) y_val_test.append(np.argmax(y_val, axis=-1)) batch_id += 1 @@ -787,8 +913,8 @@ def predict( ) # Pre-allocate space for predictions confidences: list | np.ndarray = [] - sentence_lengths = np.zeros((batch_size,), dtype=int) - predictions = np.zeros((batch_size, self._parameters["max_length"])) + sentence_lengths: np.ndarray = np.zeros((batch_size,), dtype=int) + predictions: np.ndarray = np.zeros((batch_size, self._parameters["max_length"])) if show_confidences: confidences = np.zeros( (batch_size, self._parameters["max_length"], self.num_labels) @@ -823,10 +949,10 @@ def predict( if show_confidences: confidences[ allocation_index : allocation_index + num_samples_in_batch - ] = model_output[0].numpy() - predictions[ - allocation_index : allocation_index + num_samples_in_batch - ] = model_output[1].numpy() + ] = model_output[self._SOFTMAX_OUTPUT].numpy() + predictions[allocation_index : allocation_index + num_samples_in_batch] = ( + model_output[self._ARGMAX_OUTPUT].numpy() + ) sentence_lengths[ allocation_index : allocation_index + num_samples_in_batch ] = list(map(lambda x: len(x[0]), batch_data)) diff --git a/dataprofiler/labelers/classification_report_utils.py b/dataprofiler/labelers/classification_report_utils.py index 28e742e32..66ee2c65b 100644 --- a/dataprofiler/labelers/classification_report_utils.py +++ b/dataprofiler/labelers/classification_report_utils.py @@ -1,4 +1,5 @@ """Contains functions for classification.""" + from __future__ import annotations import warnings @@ -31,25 +32,27 @@ def convert_confusion_matrix_to_MCM(conf_matrix: list | np.ndarray) -> np.ndarra """ if not isinstance(conf_matrix, np.ndarray): conf_matrix = np.array(conf_matrix) - num_labels = conf_matrix.shape[0] - num_samples = np.sum(conf_matrix) - MCM = np.zeros((num_labels, 2, 2), dtype=np.int64) + conf_matrix = cast(np.ndarray, conf_matrix) + num_labels = len(conf_matrix) + num_samples: int = int(np.sum(conf_matrix)) + MCM = cast(np.ndarray, np.zeros((num_labels, 2, 2), dtype=np.int64)) # True Positives MCM[:, 1, 1] = np.sum(conf_matrix * np.eye(num_labels), axis=1) # False Negatives - MCM[:, 1, 0] = np.sum( - conf_matrix * (np.ones(num_labels) - np.eye(num_labels)), axis=1 + non_diagonal_mask = cast( + np.ndarray, np.logical_not(np.eye(num_labels, dtype=bool)).astype(np.int64) ) + MCM[:, 1, 0] = np.sum(conf_matrix * non_diagonal_mask, axis=1) # False Positives - MCM[:, 0, 1] = np.sum( - conf_matrix.T * (np.ones(num_labels) - np.eye(num_labels)), axis=1 - ) + MCM[:, 0, 1] = np.sum(conf_matrix.T * non_diagonal_mask, axis=1) # True Negatives - MCM[:, 0, 0] = num_samples - MCM[:, 1, 0] - MCM[:, 0, 1] - MCM[:, 1, 1] + MCM[:, 0, 0] = cast( + np.ndarray, num_samples - MCM[:, 1, 0] - MCM[:, 0, 1] - MCM[:, 1, 1] + ) return MCM @@ -205,8 +208,11 @@ def precision_recall_fscore_support( f_score = (1 + beta2) * precision * recall / denom # Average the results + weights: np.ndarray | None + support: np.ndarray | None = true_sum if average == "weighted": weights = true_sum + assert weights is not None if weights.sum() == 0: return np.array([0.0]), np.array([0.0]), np.array([0.0]), None elif average == "samples": @@ -219,9 +225,9 @@ def precision_recall_fscore_support( precision = np.average(precision, weights=weights) recall = np.average(recall, weights=weights) f_score = np.average(f_score, weights=weights) - true_sum = None # return no support + support = None # return no support - return precision, recall, f_score, true_sum + return precision, recall, f_score, support def classification_report( @@ -300,7 +306,7 @@ def classification_report( """ # ALTERATION: replaced the _check_targets with this if statement since # no y_true, y_pred - y_type = "multiclass" if conf_matrix.shape[0] > 2 else "binary" + y_type = "multiclass" if len(conf_matrix) > 2 else "binary" labels_given = True if labels is None: diff --git a/dataprofiler/labelers/column_name_model.py b/dataprofiler/labelers/column_name_model.py index d698cfd6a..15d999dba 100644 --- a/dataprofiler/labelers/column_name_model.py +++ b/dataprofiler/labelers/column_name_model.py @@ -1,9 +1,10 @@ """Contains class for column name data labeling model.""" + from __future__ import annotations import json import os -from typing import Any, Callable +from typing import Any, Callable, cast import numpy as np @@ -175,7 +176,7 @@ def _reconstruct_model(self) -> None: pass def _need_to_reconstruct_model(self) -> bool: - pass + return False def reset_weights(self) -> None: """Reset weights function.""" @@ -190,15 +191,17 @@ def _model( scorer: Callable, include_label: bool = False, ) -> list: - scores = [] + scores: list[list[float | int]] = [] check_values_list = [dict["attribute"] for dict in check_values_dict] model_outputs = rapidfuzz.process.cdist( list_of_column_names, check_values_list, processor=processor, scorer=scorer ) + model_outputs = cast(np.ndarray, model_outputs) - for iter_value, ngram_match_results in enumerate(model_outputs): + for i in range(len(model_outputs)): + ngram_match_results: np.ndarray = cast(np.ndarray, model_outputs[i]) column_result = [np.max(ngram_match_results)] if include_label: index_max_result = ngram_match_results.argmax(axis=0) diff --git a/dataprofiler/labelers/data_labelers.py b/dataprofiler/labelers/data_labelers.py index 7172e7472..94283d5b5 100644 --- a/dataprofiler/labelers/data_labelers.py +++ b/dataprofiler/labelers/data_labelers.py @@ -1,17 +1,18 @@ """Module to train and choose between structured and unstructured data labelers.""" + from __future__ import annotations import os import pandas as pd -import pkg_resources from .. import data_readers +from . import utils from .base_data_labeler import BaseDataLabeler, TrainableDataLabeler from .base_model import BaseModel from .data_processing import BaseDataPostprocessor, BaseDataPreprocessor -default_labeler_dir = pkg_resources.resource_filename("resources", "labelers") +default_labeler_dir = str(utils.find_resources_dir("labelers")) def train_structured_labeler( @@ -141,11 +142,11 @@ def load_from_library(cls, name: str, trainable: bool = False) -> BaseDataLabele :type trainable: bool :return: DataLabeler class """ + for labeler_name, labeler_class_obj in cls.labeler_classes.items(): + if name == labeler_name: + name = labeler_class_obj._default_model_loc if trainable: return TrainableDataLabeler.load_from_library(name) - for _, labeler_class_obj in cls.labeler_classes.items(): - if name in labeler_class_obj._default_model_loc: - return labeler_class_obj() return BaseDataLabeler.load_from_library(name) @classmethod diff --git a/dataprofiler/labelers/data_processing.py b/dataprofiler/labelers/data_processing.py index d53980a35..794e45268 100644 --- a/dataprofiler/labelers/data_processing.py +++ b/dataprofiler/labelers/data_processing.py @@ -1,4 +1,5 @@ """Contains pre-built processors for data labeling/processing.""" + from __future__ import annotations import abc @@ -14,10 +15,11 @@ from typing import Any, Generator, Iterable, TypeVar, cast import numpy as np -import numpy.typing as npt -import pkg_resources -default_labeler_dir = pkg_resources.resource_filename("resources", "labelers") +from . import utils + +default_labeler_dir = str(utils.find_resources_dir("labelers")) + Processor = TypeVar("Processor", bound="BaseDataProcessor") @@ -70,7 +72,7 @@ def __eq__(self, other: object) -> bool: :rtype: bool """ if ( - type(self) != type(other) + type(self) is not type(other) or not isinstance(other, BaseDataProcessor) or self._parameters != other._parameters ): @@ -173,9 +175,11 @@ def process( labels: np.ndarray | None = None, label_mapping: dict[str, int] | None = None, batch_size: int = 32, - ) -> Generator[tuple[np.ndarray, np.ndarray] | np.ndarray, None, None] | tuple[ - np.ndarray, np.ndarray - ] | np.ndarray: + ) -> ( + Generator[tuple[np.ndarray, np.ndarray] | np.ndarray, None, None] + | tuple[np.ndarray, np.ndarray] + | np.ndarray + ): """Preprocess data.""" raise NotImplementedError() @@ -377,7 +381,16 @@ def _find_nearest_sentence_break_before_ind( sentence: str, start_ind: int, min_ind: int = 0, - separators: tuple[str, ...] = (" ", "\n", ",", "\t", "\r", "\x00", "\x01", ";"), + separators: tuple[str, ...] = ( + " ", + "\n", + ",", + "\t", + "\r", + "\x00", + "\x01", + ";", + ), ) -> int: """ Find nearest separator before the start_ind and return the index. @@ -477,16 +490,20 @@ def gen_none() -> Generator[None, None, None]: sample_buffer = str(sample_buffer) # buffer is empty, add sample to the buffer. sample_len = len(sample_buffer) + label_buffer_list: list[int] = [] if label_set is not None: # Create an entity buffer for sample, assign the default entity - label_buffer = np.full(sample_len, label_mapping[default_label]) + label_buffer: np.ndarray = cast( + np.ndarray, + np.full(sample_len, label_mapping[default_label], dtype=int), + ) # Map the entity to the corresponding character for start, end, label in label_set: label_index = label_mapping[label] label_buffer[start:end] = label_index - label_buffer = label_buffer.tolist() + label_buffer_list = cast(list[int], label_buffer.tolist()) # loop until the buffer is empty and placed as requested buffer_ind = 0 @@ -531,14 +548,15 @@ def gen_none() -> Generator[None, None, None]: # pad the data until fits maximum length pad_len = max( - max_length - separate_ind + buffer_ind, max_length - sample_len + max_length - separate_ind + buffer_ind, + max_length - sample_len, ) # Only add the buffer up until maximum length batch_data["samples"].append(sample_buffer[buffer_ind:separate_ind]) if label_set is not None: batch_data["labels"].append( - label_buffer[buffer_ind:separate_ind] + label_buffer_list[buffer_ind:separate_ind] + [label_mapping[pad_label]] * pad_len ) @@ -605,7 +623,7 @@ def gen_none() -> Generator[None, None, None]: if label_set is not None: flattened_entities.extend( - label_buffer[buffer_ind:separate_ind] + label_buffer_list[buffer_ind:separate_ind] ) buffer_ind = separate_ind @@ -689,7 +707,7 @@ def process( :return batch_data: A dict containing samples of size batch_size :rtype batch_data: dicts """ - num_dim = sum([dim > 1 for dim in data.shape]) + num_dim = sum(dim > 1 for dim in np.shape(data)) if num_dim > 1: raise ValueError( "Multidimensional data given to " @@ -891,7 +909,17 @@ def __init__( flatten_separator: str = " ", use_word_level_argmax: bool = False, output_format: str = "character_argmax", - separators: tuple[str, ...] = (" ", ",", ";", "'", '"', ":", "\n", "\t", "."), + separators: tuple[str, ...] = ( + " ", + ",", + ";", + "'", + '"', + ":", + "\n", + "\t", + ".", + ), word_level_min_percent: float = 0.75, ) -> None: """ @@ -1185,7 +1213,11 @@ def convert_to_NER_format( if begin_idx != -1: # Add last sample sample_output.append( - (begin_idx, curr_idx + 1, reverse_label_mapping[(int(curr_label))]) + ( + begin_idx, + curr_idx + 1, + reverse_label_mapping[(int(curr_label))], + ) ) # Add to total output list output_result.append(sample_output) @@ -1194,7 +1226,10 @@ def convert_to_NER_format( @staticmethod def match_sentence_lengths( - data: np.ndarray, results: dict, flatten_separator: str, inplace: bool = True + data: np.ndarray, + results: dict, + flatten_separator: str, + inplace: bool = True, ) -> dict: """ Convert results from model into same ragged data shapes as original data. @@ -1210,10 +1245,13 @@ def match_sentence_lengths( :type inplace: bool :return: dict(pred=...) or dict(pred=..., conf=...) """ - pred_buffer = [] - conf_buffer = [] + pred_buffer: np.ndarray = np.array([]) + conf_buffer: np.ndarray = np.array([]) result_ind = 0 - buffer_add_inds = np.cumsum(list(map(len, results["pred"]))).tolist() + cumulative_lengths = cast( + np.ndarray, np.cumsum(list(map(len, results["pred"]))) + ) + buffer_add_inds = cast(list[int], cumulative_lengths.tolist()) separator_len = len(flatten_separator) if not inplace: @@ -1403,7 +1441,7 @@ def get_parameters(self, param_list: list[str] | None = None) -> dict: return params def convert_to_unstructured_format( - self, data: np.ndarray, labels: list[str] | npt.NDArray[np.str_] | None + self, data: np.ndarray, labels: list[str] | np.ndarray | None ) -> tuple[str, list[tuple[int, int, str]] | None]: """ Convert data samples list to StructCharPreprocessor required input data format. @@ -1466,14 +1504,14 @@ def process( "If `labels` are specified, `label_mapping` " "must also be specified." ) - if data.shape != labels.shape: + if np.shape(data) != np.shape(labels): raise ValueError( f"Data and labels given to " f"StructCharPreprocessor are of different " - f"shapes, {data.shape} != {labels.shape}" + f"shapes, {np.shape(data)} != {np.shape(labels)}" ) - num_dim = sum([dim > 1 for dim in data.shape]) + num_dim = sum(dim > 1 for dim in np.shape(data)) if num_dim > 1: warnings.warn( "Data given to StructCharPreprocessor was " @@ -1495,7 +1533,7 @@ def process( # with rework, can be tuned to be batches > size 1 for ind in range(len(data)): batch_data: np.ndarray = data[ind : ind + 1] - batch_labels: npt.NDArray[np.str_] | list[str] | None = ( + batch_labels: np.ndarray | list[str] | None = ( None if labels is None else labels[ind : ind + 1] ) ( @@ -1516,7 +1554,10 @@ def process( np_unstruct_labels = None return super().process( - np.array(unstructured_data), np_unstruct_labels, label_mapping, batch_size + np.array(unstructured_data), + np_unstruct_labels, + label_mapping, + batch_size, ) @@ -1586,7 +1627,7 @@ def __eq__(self, other: object) -> bool: :rtype: bool """ if ( - type(self) != type(other) + type(self) is not type(other) or not isinstance(other, StructCharPostprocessor) or self._parameters["default_label"] != other._parameters["default_label"] or self._parameters["pad_label"] != other._parameters["pad_label"] @@ -1662,7 +1703,10 @@ def help(cls) -> None: @staticmethod def match_sentence_lengths( - data: np.ndarray, results: dict, flatten_separator: str, inplace: bool = True + data: np.ndarray, + results: dict, + flatten_separator: str, + inplace: bool = True, ) -> dict: """ Convert results from model into same ragged data shapes as original data. @@ -1678,10 +1722,13 @@ def match_sentence_lengths( :type inplace: bool :return: dict(pred=...) or dict(pred=..., conf=...) """ - pred_buffer = [] - conf_buffer = [] + pred_buffer: np.ndarray = np.array([]) + conf_buffer: np.ndarray = np.array([]) result_ind = 0 - buffer_add_inds = np.cumsum(list(map(len, results["pred"]))).tolist() + cumulative_lengths = cast( + np.ndarray, np.cumsum(list(map(len, results["pred"]))) + ) + buffer_add_inds = cast(list[int], cumulative_lengths.tolist()) separator_len = len(flatten_separator) if not inplace: @@ -1769,9 +1816,13 @@ def convert_to_structured_analysis( ignore_value = label_mapping[pad_label] num_labels = max(label_mapping.values()) + 1 - labels_out = np.ones((len(results["pred"]),)) + labels_out: np.ndarray = cast( + np.ndarray, np.full((len(results["pred"]),), None, dtype=object) + ) if "conf" in results: - confs_out = np.zeros((len(results["pred"]), num_labels)) + confs_out: np.ndarray = cast( + np.ndarray, np.zeros((len(results["pred"]), num_labels)) + ) for i, label_samples in enumerate(zip(results["pred"], sentences)): column_labels, sample = label_samples @@ -1947,9 +1998,11 @@ def _validate_parameters(self, parameters: dict) -> None: # being changed and is already set aggregation_func = parameters.get( "aggregation_func", - self._parameters.get("aggregation_func") - if hasattr(self, "_parameters") - else None, + ( + self._parameters.get("aggregation_func") + if hasattr(self, "_parameters") + else None + ), ) if value is None and aggregation_func == "priority": errors.append( diff --git a/dataprofiler/labelers/labeler_utils.py b/dataprofiler/labelers/labeler_utils.py index b6070ff72..04b0abf07 100644 --- a/dataprofiler/labelers/labeler_utils.py +++ b/dataprofiler/labelers/labeler_utils.py @@ -1,4 +1,5 @@ """Contains functions for the data labeler.""" + from __future__ import annotations import logging @@ -78,8 +79,8 @@ class 1 1.00 0.67 0.80 3 def evaluate_accuracy( - predicted_entities_in_index: list[list[int]], - true_entities_in_index: list[list[int]], + predicted_entities_in_index: list[list[int]] | np.ndarray, + true_entities_in_index: list[list[int]] | np.ndarray, num_labels: int, entity_rev_dict: dict[int, str], verbose: bool = True, @@ -119,13 +120,22 @@ def evaluate_accuracy( if x[1] not in omitted_labels ] - max_len = len(predicted_entities_in_index[0]) - true_labels_padded = np.zeros((len(true_entities_in_index), max_len)) - for i, true_labels_row in enumerate(true_entities_in_index): + predicted_entities: list[np.ndarray] = [ + np.asarray(row) for row in predicted_entities_in_index + ] + true_entities: list[np.ndarray] = [ + np.asarray(row) for row in true_entities_in_index + ] + + max_len = len(predicted_entities[0]) + true_labels_padded: np.ndarray = cast( + np.ndarray, np.zeros((len(true_entities), max_len)) + ) + for i, true_labels_row in enumerate(true_entities): true_labels_padded[i][: len(true_labels_row)] = true_labels_row - true_labels_flatten = np.hstack(true_labels_padded) # type: ignore - predicted_labels_flatten = np.hstack(predicted_entities_in_index) + true_labels_flatten: np.ndarray = np.hstack(true_labels_padded) # type: ignore + predicted_labels_flatten: np.ndarray = np.hstack(predicted_entities) all_labels: list[str] = [] if entity_rev_dict: @@ -135,7 +145,9 @@ def evaluate_accuracy( # By definition a confusion matrix :math:`C` is such that :math:`C_{i, j}` # is equal to the number of observations known to be in group :math:`i` but # predicted to be in group :math:`j`. - conf_mat = np.zeros((num_labels, num_labels), dtype=np.int64) + conf_mat: np.ndarray = cast( + np.ndarray, np.zeros((num_labels, num_labels), dtype=np.int64) + ) batch_size = min(2**20, len(true_labels_flatten)) for batch_ind in range(len(true_labels_flatten) // batch_size + 1): true_label_batch = true_labels_flatten[ @@ -228,6 +240,64 @@ def get_tf_layer_index_from_name(model: tf.keras.Model, layer_name: str) -> int return None +def normalize_tf_model_outputs( + model: tf.keras.Model, + output_names: list[str], + create_outputs_fn: Callable[ + [tf.Tensor, list[tf.Tensor | None]], dict[str, tf.Tensor] + ], +) -> tf.keras.Model: + """Convert a model's outputs into a named dict-output structure when possible.""" + try: + model_output = model.output + except (AttributeError, IndexError): + model_output = None + + try: + model_outputs_list = list(model.outputs) + except (AttributeError, IndexError, TypeError): + model_outputs_list = [] + + if isinstance(model_output, dict): + if set(model_output) == set(output_names): + return model + softmax_output = model_output.get( + output_names[0], next(iter(model_output.values())) + ) + extra_outputs = [model_output.get(name) for name in output_names[1:]] + else: + if not model_outputs_list: + try: + last_output = model.layers[-1].output + except (AttributeError, IndexError): + return model + if not hasattr(last_output, "_keras_history"): + return model + model_outputs_list = [last_output] + softmax_output = model_outputs_list[0] + extra_outputs = [ + model_outputs_list[index] if len(model_outputs_list) > index else None + for index in range(1, len(output_names)) + ] + + try: + output_dict = create_outputs_fn(softmax_output, extra_outputs) + return tf.keras.Model(model.inputs, output_dict) + except (AttributeError, TypeError, ValueError): + return model + + +def get_tf_rebuild_layer_name(model: tf.keras.Model, base_name: str) -> str: + """Return a layer name unique within the current model graph.""" + existing_names = {layer.name for layer in getattr(model, "layers", [])} + if base_name not in existing_names: + return base_name + suffix = 1 + while f"{base_name}_{suffix}" in existing_names: + suffix += 1 + return f"{base_name}_{suffix}" + + def hide_tf_logger_warnings() -> None: """Filter out a set of warnings from the tf logger.""" @@ -358,7 +428,7 @@ def __init__( def _zero_wt_init(name: str) -> tf.Variable: return self.add_weight( - name, shape=self.init_shape, initializer="zeros", dtype=self.dtype + name=name, shape=self.init_shape, initializer="zeros", dtype=self.dtype ) self.true_positives = _zero_wt_init("true_positives") @@ -435,11 +505,6 @@ def get_config(self) -> dict: base_config = super().get_config() return {**base_config, **config} - def reset_state(self) -> None: - """Reset state.""" - reset_value = tf.zeros(self.init_shape, dtype=self.dtype) - tf.keras.backend.batch_set_value([(v, reset_value) for v in self.variables]) - @protected_register_keras_serializable() class F1Score(FBetaScore): diff --git a/dataprofiler/labelers/regex_model.py b/dataprofiler/labelers/regex_model.py index c6a690c17..80458acf2 100644 --- a/dataprofiler/labelers/regex_model.py +++ b/dataprofiler/labelers/regex_model.py @@ -1,4 +1,5 @@ """Contains class for regex data labeling model.""" + from __future__ import annotations import copy @@ -6,6 +7,7 @@ import os import re import sys +from typing import Any import numpy as np @@ -166,7 +168,7 @@ def _reconstruct_model(self) -> None: pass def _need_to_reconstruct_model(self) -> bool: - pass + return False def reset_weights(self) -> None: """Reset weights.""" @@ -224,7 +226,7 @@ def predict( # Construct array initial regex predictions where background is # predicted. - predictions = [np.empty((0,))] * 100 + predictions: list[Any] = [np.empty((0,))] * 100 i = 0 for i, input_string in enumerate(data): @@ -246,7 +248,7 @@ def predict( if verbose: sys.stdout.flush() sys.stdout.write(f"\rData Samples Processed: {i + 1:d} ") - predictions[i] = pred + predictions[i] = pred.tolist() if verbose: logger.info(f"\rData Samples Processed: {i + 1:d} ") diff --git a/dataprofiler/labelers/utils.py b/dataprofiler/labelers/utils.py index 2d587f7b4..b6e4a09c6 100644 --- a/dataprofiler/labelers/utils.py +++ b/dataprofiler/labelers/utils.py @@ -1,8 +1,18 @@ """Contains functions for checking for installations/dependencies.""" + +import importlib.resources import sys import warnings +from pathlib import Path from typing import Any, Callable, List +try: + # Newer Pythons / newer typeshed + from importlib.resources.abc import Traversable +except ModuleNotFoundError: + # Older Pythons + from importlib.abc import Traversable + def warn_missing_module(labeler_function: str, module_name: str) -> None: """ @@ -50,3 +60,15 @@ def new_f(*args: Any, **kwds: Any) -> Any: return new_f return check_module + + +def find_resources_dir(resource_path: str | Path | None = None) -> Traversable: + """Return the path to the package resources.""" + resource = importlib.resources.files("dataprofiler") / "resources" + if resource_path: + resource = resource / str(resource_path) + + if not (resource.is_file() or resource.is_dir()): + raise FileNotFoundError(f"Resource not found: {resource_path}") + + return resource diff --git a/dataprofiler/plugins/decorators.py b/dataprofiler/plugins/decorators.py index c781f4300..f099c1aa4 100644 --- a/dataprofiler/plugins/decorators.py +++ b/dataprofiler/plugins/decorators.py @@ -1,4 +1,5 @@ """Contains function for generating plugins data.""" + from collections import defaultdict from typing import Any, DefaultDict, Dict @@ -21,7 +22,6 @@ def __inner_factory_function(fn): :param fn: Plugin function :return: function """ - global plugins_dict plugins_dict[typ][name] = fn return fn diff --git a/dataprofiler/profilers/__init__.py b/dataprofiler/profilers/__init__.py index 4b068fcb0..14834794b 100644 --- a/dataprofiler/profilers/__init__.py +++ b/dataprofiler/profilers/__init__.py @@ -1,4 +1,5 @@ """Package for providing statistics and predictions for a given dataset.""" + from . import json_decoder from .base_column_profilers import BaseColumnProfiler from .categorical_column_profile import CategoricalColumn diff --git a/dataprofiler/profilers/categorical_column_profile.py b/dataprofiler/profilers/categorical_column_profile.py index 1ca630900..d64f5aa5b 100644 --- a/dataprofiler/profilers/categorical_column_profile.py +++ b/dataprofiler/profilers/categorical_column_profile.py @@ -1,4 +1,5 @@ """Contains class for categorical column profiler.""" + from __future__ import annotations import math @@ -277,28 +278,28 @@ def diff(self, other_profile: CategoricalColumn, options: dict = None) -> dict: # These stats are only diffed if both profiles are categorical if self.is_match and other_profile.is_match: - differences["statistics"][ - "chi2-test" - ] = profiler_utils.perform_chi_squared_test_for_homogeneity( - self._categories, - self.sample_size, - other_profile._categories, - other_profile.sample_size, + differences["statistics"]["chi2-test"] = ( + profiler_utils.perform_chi_squared_test_for_homogeneity( + self._categories, + self.sample_size, + other_profile._categories, + other_profile.sample_size, + ) ) - differences["statistics"][ - "categories" - ] = profiler_utils.find_diff_of_lists_and_sets( - self.categories, other_profile.categories + differences["statistics"]["categories"] = ( + profiler_utils.find_diff_of_lists_and_sets( + self.categories, other_profile.categories + ) ) - differences["statistics"][ - "gini_impurity" - ] = profiler_utils.find_diff_of_numbers( - self.gini_impurity, other_profile.gini_impurity + differences["statistics"]["gini_impurity"] = ( + profiler_utils.find_diff_of_numbers( + self.gini_impurity, other_profile.gini_impurity + ) ) - differences["statistics"][ - "unalikeability" - ] = profiler_utils.find_diff_of_numbers( - self.unalikeability, other_profile.unalikeability + differences["statistics"]["unalikeability"] = ( + profiler_utils.find_diff_of_numbers( + self.unalikeability, other_profile.unalikeability + ) ) cat_count1 = dict( sorted(self._categories.items(), key=itemgetter(1), reverse=True) @@ -326,9 +327,9 @@ def diff(self, other_profile: CategoricalColumn, options: dict = None) -> dict: ) differences["statistics"]["psi"] = total_psi - differences["statistics"][ - "categorical_count" - ] = profiler_utils.find_diff_of_dicts(self_cat_count, other_cat_count) + differences["statistics"]["categorical_count"] = ( + profiler_utils.find_diff_of_dicts(self_cat_count, other_cat_count) + ) return differences diff --git a/dataprofiler/profilers/column_profile_compilers.py b/dataprofiler/profilers/column_profile_compilers.py index 07edf13dc..cfeb8c699 100644 --- a/dataprofiler/profilers/column_profile_compilers.py +++ b/dataprofiler/profilers/column_profile_compilers.py @@ -1,4 +1,5 @@ """For generating a report.""" + from __future__ import annotations import abc diff --git a/dataprofiler/profilers/data_labeler_column_profile.py b/dataprofiler/profilers/data_labeler_column_profile.py index d9bfe1ee9..15375c0b3 100644 --- a/dataprofiler/profilers/data_labeler_column_profile.py +++ b/dataprofiler/profilers/data_labeler_column_profile.py @@ -1,4 +1,5 @@ """Contains class for for profiling data labeler col.""" + from __future__ import annotations import operator @@ -250,18 +251,27 @@ def data_label(self) -> str | None: return None ranks_items = self.rank_distribution.items() - ordered_top_k_rank = np.array( - sorted(ranks_items, key=operator.itemgetter(1), reverse=True) + ordered_top_k_rank = sorted( + ranks_items, key=operator.itemgetter(1), reverse=True )[: self._top_k_labels] - top_k_probabilities = np.fromiter( - map(operator.itemgetter(1), ordered_top_k_rank), dtype=float - ) / sum(self.rank_distribution.values()) + top_k_probabilities = cast( + np.ndarray, + np.fromiter(map(operator.itemgetter(1), ordered_top_k_rank), dtype=float) + / sum(self.rank_distribution.values()), + ) is_value_close = ( top_k_probabilities - top_k_probabilities[0] >= -self._min_prob_differential ) data_label = "|".join( - map(operator.itemgetter(0), ordered_top_k_rank[is_value_close]) + map( + operator.itemgetter(0), + [ + rank_item + for rank_item, keep_value in zip(ordered_top_k_rank, is_value_close) + if keep_value + ], + ) ) top_label = ordered_top_k_rank[0][0] if cast(Dict, self.label_representation)[top_label] < self._min_top_label_prob: @@ -427,7 +437,7 @@ def _update_predictions( start_index = 0 if self.data_labeler.model.requires_zero_mapping: start_index = 1 - for i in range(rank_predictions.shape[0]): + for i in range(len(rank_predictions)): sorted_rank = rank_predictions[i][-self._top_k_voting :] sorted_rank = sorted_rank[np.argsort(predictions["conf"][i][sorted_rank])] for rank_position, value in enumerate(sorted_rank): diff --git a/dataprofiler/profilers/datetime_column_profile.py b/dataprofiler/profilers/datetime_column_profile.py index af99283a9..1042ea0c0 100644 --- a/dataprofiler/profilers/datetime_column_profile.py +++ b/dataprofiler/profilers/datetime_column_profile.py @@ -1,4 +1,5 @@ """Contains class for profiling datetime column.""" + from __future__ import annotations import datetime @@ -216,7 +217,7 @@ def _validate_datetime(date: str, date_format: str) -> datetime.datetime | float :return: either the str converted into a date format, or Nan """ try: - converted_date: (datetime.datetime | float) = datetime.datetime.strptime( + converted_date: datetime.datetime | float = datetime.datetime.strptime( date, date_format ) except (ValueError, TypeError): @@ -237,7 +238,7 @@ def _replace_day_suffix(date: str, pattern: re.Pattern) -> str | float: """ try: new_date: str | float = pattern.sub(r"\1", date) - except (TypeError): + except TypeError: new_date = np.nan return new_date diff --git a/dataprofiler/profilers/float_column_profile.py b/dataprofiler/profilers/float_column_profile.py index bc426a447..b75425b7a 100644 --- a/dataprofiler/profilers/float_column_profile.py +++ b/dataprofiler/profilers/float_column_profile.py @@ -1,8 +1,10 @@ """Float profile analysis for individual col within structured profiling.""" + from __future__ import annotations import copy import re +from typing import cast import numpy as np import pandas as pd @@ -200,7 +202,7 @@ def profile(self) -> dict: :return: """ - profile = NumericStatsMixin.profile(self) + profile = cast(dict, NumericStatsMixin.profile.__get__(self, type(self))) profile.update( dict( precision=dict( diff --git a/dataprofiler/profilers/graph_profiler.py b/dataprofiler/profilers/graph_profiler.py index 0680a29a7..fd7e7453e 100644 --- a/dataprofiler/profilers/graph_profiler.py +++ b/dataprofiler/profilers/graph_profiler.py @@ -1,4 +1,5 @@ """Class and functions to calculate and profile properties of graph data.""" + from __future__ import annotations import importlib @@ -455,10 +456,13 @@ def _get_categorical_distribution( for attribute in attributes: if attribute in categorical_attributes: data_as_list = self._attribute_data_as_list(graph, attribute) - hist, edges = np.histogram(data_as_list, bins="auto", density=False) + hist, edges = cast( + tuple[np.ndarray, np.ndarray], + np.histogram(data_as_list, bins="auto", density=False), + ) categorical_distributions[attribute] = { - "bin_counts": list(hist), - "bin_edges": list(edges), + "bin_counts": cast(list[float], hist.tolist()), + "bin_edges": cast(list[float], edges.tolist()), } else: categorical_distributions[attribute] = None @@ -492,9 +496,11 @@ def _get_categorical_and_continuous_attributes( @staticmethod def _find_all_attributes(graph: nx.Graph) -> list[str]: """Compute the number of attributes for each edge.""" - attribute_list = set( - np.array([list(graph.edges[n].keys()) for n in graph.edges()]).flatten() - ) + attribute_list = { + attribute + for edge in graph.edges() + for attribute in graph.edges[edge].keys() + } return list(attribute_list) def _attribute_data_as_list(self, graph: nx.Graph, attribute: str) -> list: diff --git a/dataprofiler/profilers/helpers/__init__.py b/dataprofiler/profilers/helpers/__init__.py index 43393433d..2c72b2f35 100644 --- a/dataprofiler/profilers/helpers/__init__.py +++ b/dataprofiler/profilers/helpers/__init__.py @@ -1,4 +1,5 @@ """This package provides helper functions for generating reports.""" + from .report_helpers import _prepare_report, calculate_quantiles __all__ = [ diff --git a/dataprofiler/profilers/helpers/report_helpers.py b/dataprofiler/profilers/helpers/report_helpers.py index 0588252c9..44ac8fb1a 100644 --- a/dataprofiler/profilers/helpers/report_helpers.py +++ b/dataprofiler/profilers/helpers/report_helpers.py @@ -1,4 +1,5 @@ """Contains helper functions for generating report.""" + from __future__ import annotations import math diff --git a/dataprofiler/profilers/histogram_utils.py b/dataprofiler/profilers/histogram_utils.py index df230c4c7..b7aed9ad1 100644 --- a/dataprofiler/profilers/histogram_utils.py +++ b/dataprofiler/profilers/histogram_utils.py @@ -7,15 +7,33 @@ A copy of the license for numpy is available here: https://github.com/numpy/numpy/blob/main/LICENSE.txt """ + import operator from typing import List, Optional, Tuple, Union import numpy as np -from numpy.lib.histograms import ( # type: ignore[attr-defined] - _get_outer_edges, - _hist_bin_selectors, - _unsigned_subtract, -) + +from . import profiler_utils + +try: + # numpy v2+ + # NOTE: `numpy.lib._histograms_impl` is a private module, so this import may + # need to be revisited if NumPy exposes a public replacement for these + # helpers. NumPy's 2.4.0 release notes discuss public APIs replacing modules + # that moved private in 2.x: https://numpy.org/doc/stable/release/2.4.0-notes.html + from numpy.lib._histograms_impl import ( # type: ignore[attr-defined] + _get_outer_edges, + _hist_bin_selectors, + _unsigned_subtract, + ) + +except ModuleNotFoundError: + # numpy v1+ + from numpy.lib.histograms import ( + _get_outer_edges, + _hist_bin_selectors, + _unsigned_subtract, + ) def _get_maximum_from_profile(profile): @@ -79,7 +97,7 @@ def _ptp(maximum: float, minimum: float): :return: the difference between the maximum and minimum """ - return np.subtract(maximum, minimum) + return profiler_utils.as_float_scalar(np.subtract(maximum, minimum)) def _calc_doane_bin_width_from_profile(profile): @@ -180,7 +198,9 @@ def _calc_fd_bin_width_from_profile(profile): :return: An estimate of the optimal bin width for the given data. """ - iqr = np.subtract(profile._get_percentile([75]), profile._get_percentile([25])) + iqr = profiler_utils.as_float_scalar( + np.subtract(profile._get_percentile([75]), profile._get_percentile([25])) + ) dataset_size = _get_dataset_size_from_profile(profile) return 2.0 * iqr * dataset_size ** (-1.0 / 3.0) @@ -290,6 +310,8 @@ def _get_bin_edges( else: # Do not call selectors on empty arrays width = _hist_bin_selectors[bin_name](a, (first_edge, last_edge)) + if width is not None: + width = profiler_utils.as_float_scalar(width) if width: n_equal_bins = int( np.ceil(_unsigned_subtract(last_edge, first_edge) / width) @@ -341,6 +363,8 @@ def _calculate_bins_from_profile(profile, bin_method): else: # Do not call selectors on empty arrays width = _hist_bin_width_selectors_for_profile[bin_method](profile) + if width is not None: + width = profiler_utils.as_float_scalar(width) if width and not np.isnan(width): n_equal_bins = int(np.ceil(_ptp(maximum, minimum) / width)) else: diff --git a/dataprofiler/profilers/int_column_profile.py b/dataprofiler/profilers/int_column_profile.py index 014465c71..d0194d8f4 100644 --- a/dataprofiler/profilers/int_column_profile.py +++ b/dataprofiler/profilers/int_column_profile.py @@ -1,6 +1,9 @@ """Int profile analysis for individual col within structured profiling.""" + from __future__ import annotations +from typing import cast + import numpy as np import pandas as pd @@ -98,7 +101,7 @@ def profile(self) -> dict: :return: """ - return NumericStatsMixin.profile(self) + return cast(dict, NumericStatsMixin.profile.__get__(self, type(self))) @property def data_type_ratio(self) -> float | None: @@ -163,7 +166,7 @@ def update(self, df_series: pd.Series) -> IntColumn: df_series = df_series.reset_index(drop=True) is_each_row_int = self._is_each_row_int(df_series) sample_size = len(is_each_row_int) - match_int_count = np.sum(is_each_row_int) + match_int_count: int = int(np.sum(is_each_row_int)) profile = dict(match_count=match_int_count, sample_size=sample_size) BaseColumnProfiler._perform_property_calcs( diff --git a/dataprofiler/profilers/json_decoder.py b/dataprofiler/profilers/json_decoder.py index fb4ff8cb9..eb09db0d7 100644 --- a/dataprofiler/profilers/json_decoder.py +++ b/dataprofiler/profilers/json_decoder.py @@ -1,4 +1,5 @@ """Contains methods to decode components of a Profiler.""" + from __future__ import annotations import warnings @@ -116,9 +117,9 @@ def get_structured_col_profiler_class(class_name: str) -> type[StructuredColProf :type class_name: str representing name of class :return: subclass of StructuredColProfiler object """ - struct_col_profiler_class: None | ( - type[StructuredColProfiler] - ) = _structured_col_profiler.get(class_name) + struct_col_profiler_class: None | (type[StructuredColProfiler]) = ( + _structured_col_profiler.get(class_name) + ) if struct_col_profiler_class is None: raise ValueError( f"Invalid structured col profiler class {class_name} " f"failed to load." @@ -153,9 +154,9 @@ def load_column_profile( JSON """ - column_profiler_cls: type[ - BaseColumnProfiler[BaseColumnProfiler] - ] = get_column_profiler_class(serialized_json["class"]) + column_profiler_cls: type[BaseColumnProfiler[BaseColumnProfiler]] = ( + get_column_profiler_class(serialized_json["class"]) + ) return column_profiler_cls.load_from_dict(serialized_json["data"], config) diff --git a/dataprofiler/profilers/numerical_column_stats.py b/dataprofiler/profilers/numerical_column_stats.py index fa0666a66..1283f3992 100644 --- a/dataprofiler/profilers/numerical_column_stats.py +++ b/dataprofiler/profilers/numerical_column_stats.py @@ -6,10 +6,9 @@ import copy import itertools import warnings -from typing import Any, Callable, Dict, List, TypeVar, cast +from typing import Any, Callable, Dict, List, TypeAlias, TypeVar, cast import numpy as np -import numpy.typing as npt import pandas as pd import scipy.stats @@ -32,6 +31,7 @@ def __init__(self, function: Callable) -> None: NumericStatsMixinT = TypeVar("NumericStatsMixinT", bound="NumericStatsMixin") +Numeric: TypeAlias = int | float | np.float64 | np.int64 class NumericStatsMixin(BaseColumnProfiler[NumericStatsMixinT], metaclass=abc.ABCMeta): @@ -56,10 +56,10 @@ def __init__(self, options: NumericalOptions = None) -> None: "NumericalStatsMixin parameter 'options' must be " "of type NumericalOptions." ) - self.min: int | float | np.float64 | np.int64 | None = None - self.max: int | float | np.float64 | np.int64 | None = None + self.min: Numeric | None = None + self.max: Numeric | None = None self._top_k_modes: int = 5 # By default, return at max 5 modes - self.sum: int | float | np.float64 | np.int64 = np.float64(0) + self.sum: Numeric = np.float64(0) self._biased_variance: float | np.float64 = np.nan self._biased_skewness: float | np.float64 = np.nan self._biased_kurtosis: float | np.float64 = np.nan @@ -198,7 +198,7 @@ def _add_helper_merge_profile_histograms( # calculate the min of the first edge and the max of the last edge # between two arrays global_min_of_histogram_edges = ( - float(self.min) + profiler_utils.as_float_scalar(self.min) if self.min is not None else min( other1._stored_histogram["histogram"]["bin_edges"][0], @@ -207,7 +207,7 @@ def _add_helper_merge_profile_histograms( ) global_max_of_histogram_edges = ( - float(self.max) + profiler_utils.as_float_scalar(self.max) if self.max is not None else max( other1._stored_histogram["histogram"]["bin_edges"][-1], @@ -257,9 +257,9 @@ def _add_helper_merge_profile_histograms( if self.user_set_histogram_bin is None: for method in self.histogram_bin_method_names: - self.histogram_methods[method][ - "suggested_bin_count" - ] = histogram_utils._calculate_bins_from_profile(self, method) + self.histogram_methods[method]["suggested_bin_count"] = ( + histogram_utils._calculate_bins_from_profile(self, method) + ) self._get_quantiles() @@ -298,14 +298,14 @@ def _add_helper( ) if "min" in self.__calculations.keys(): if other1.min is not None and other2.min is not None: - self.min = min(other1.min, other2.min) + self.min = min(other1.min, other2.min) # type: ignore[type-var] elif other2.min is None: self.min = other1.min else: self.min = other2.min if "max" in self.__calculations.keys(): if other1.max is not None and other2.max is not None: - self.max = max(other1.max, other2.max) + self.max = max(other1.max, other2.max) # type: ignore[type-var] elif other2.max is None: self.max = other1.max else: @@ -365,6 +365,7 @@ def _add_helper( other1._median_abs_dev_is_enabled and other2._median_abs_dev_is_enabled ) + @property def profile(self) -> dict: """ Return profile of the column. @@ -407,7 +408,7 @@ def report(self, remove_disabled_flag: bool = False) -> dict: :rtype: Profile """ calcs_dict_keys = self._NumericStatsMixin__calculations.keys() - profile = self.profile() + profile = self.profile if remove_disabled_flag: profile_keys = list(profile.keys()) @@ -540,7 +541,7 @@ def mean(self) -> float | np.float64: """Return mean value.""" if self.match_count == 0: return 0.0 - return self.sum / self.match_count + return cast(float | np.float64, self.sum / self.match_count) @property def mode(self) -> list[float]: @@ -563,7 +564,7 @@ def median(self) -> float: :rtype: float """ if not self._has_histogram or not self._median_is_enabled: - return np.nan + return float(np.nan) return self._get_percentile([50])[0] @property @@ -622,14 +623,17 @@ def _perform_t_test( RuntimeWarning, ) invalid_stats = True - if np.isnan( - [float(mean1), float(mean2), float(var1), float(var2)] - ).any() or None in [ - mean1, - mean2, - var1, - var2, - ]: + if ( + None in [mean1, mean2, var1, var2] + or np.isnan( + [ + profiler_utils.as_float_scalar(mean1), + profiler_utils.as_float_scalar(mean2), + profiler_utils.as_float_scalar(var1), + profiler_utils.as_float_scalar(var2), + ] + ).any() + ): warnings.warn( "Null value(s) found in mean and/or variance values. " "T-test cannot be performed.", @@ -1040,10 +1044,7 @@ def _merge_biased_kurtosis( / N**3 ) third_term = ( - 6 - * delta**2 - * (match_count1**2 * M2_2 + match_count2**2 * M2_1) - / N**2 + 6 * delta**2 * (match_count1**2 * M2_2 + match_count2**2 * M2_1) / N**2 ) fourth_term = 4 * delta * (match_count1 * M3_2 - match_count2 * M3_1) / N M4 = first_term + second_term + third_term + fourth_term @@ -1106,11 +1107,12 @@ def _estimate_mode_from_histogram(self) -> list[float]: elif bin_counts[i] == cur_max and count < self._top_k_modes: highest_idxs.append(i) count += 1 - highest_idxs = np.array(highest_idxs) # type: ignore + highest_idx_array = cast(np.ndarray, np.array(highest_idxs)) - mode: npt.NDArray[np.float64] = ( - bin_edges[highest_idxs] + bin_edges[highest_idxs + 1] # type: ignore - ) / 2 + mode = cast( + np.ndarray, + (bin_edges[highest_idx_array] + bin_edges[highest_idx_array + 1]) / 2, + ) return cast(List[float], mode.tolist()) def _estimate_stats_from_histogram(self) -> np.float64: @@ -1133,9 +1135,9 @@ def _total_histogram_bin_variance( bin_edges = bin_edges.copy() bin_edges[-1] += 1e-3 - inds = np.digitize(input_array, bin_edges) + inds = cast(np.ndarray, np.digitize(input_array, bin_edges)) sum_var = 0 - non_zero_bins = np.where(bin_counts)[0] + 1 + non_zero_bins = cast(np.ndarray, np.where(bin_counts)[0] + 1) for i in non_zero_bins: elements_in_bin = input_array[inds == i] bin_var = elements_in_bin.var() @@ -1213,9 +1215,9 @@ def _select_method_for_histogram( self.histogram_methods[method]["current_loss"] = self._histogram_loss( current_diff_var[method_id], current_avg_diff_var, - current_total_var[method_id], + cast(float, current_total_var[method_id]), current_avg_total_var, - current_run_time[method_id], + cast(float, current_run_time[method_id]), current_avg_run_time, ) self.histogram_methods[method]["total_loss"] += self.histogram_methods[ @@ -1316,7 +1318,9 @@ def _get_histogram( ] = suggested_bin_count # calculate the stored histogram bins - bin_counts, bin_edges = np.histogram(values, bins=n_equal_bins) + bin_counts, bin_edges = cast( + tuple[np.ndarray, np.ndarray], np.histogram(values, bins=n_equal_bins) + ) return bin_counts, bin_edges def _merge_histogram(self, values: np.ndarray | pd.Series) -> None: @@ -1403,7 +1407,7 @@ def _assimilate_histogram( dest_hist_entity_count_per_bin: np.ndarray, dest_hist_bin_edges: np.ndarray, dest_hist_num_bin: int, - ) -> tuple[dict[str, np.ndarray[Any, Any]], float]: + ) -> tuple[dict[str, np.ndarray], float]: """ Assimilates a histogram into another histogram using specifications. @@ -1568,7 +1572,7 @@ def _get_percentile(self, percentiles: np.ndarray | list[float]) -> list[float]: bin_counts = bin_counts.astype(float) normalized_bin_counts = bin_counts / np.sum(bin_counts) - cumsum_bin_counts = np.cumsum(normalized_bin_counts) + cumsum_bin_counts = cast(np.ndarray, np.cumsum(normalized_bin_counts)) median_value = None median_bin_inds = np.abs(cumsum_bin_counts - 0.5) < 1e-10 @@ -1576,6 +1580,7 @@ def _get_percentile(self, percentiles: np.ndarray | list[float]) -> list[float]: median_value = np.mean(bin_edges[np.append([False], median_bin_inds)]) # use the floor by slightly increasing cases where no bin exist. + cumsum_bin_counts = cast(np.ndarray, cumsum_bin_counts.copy()) cumsum_bin_counts[zero_inds] += 1e-15 # add initial zero bin @@ -1700,18 +1705,24 @@ def median_abs_deviation(self) -> float | np.float64: np.append([True], np.diff(bin_edges_impose) > 1e-14) ] - bin_counts_impose_pos: npt.NDArray[np.float64] = np.interp( - bin_edges_impose, - bin_edges_pos, - np.cumsum(np.append([0], bin_counts_pos)), + bin_counts_impose_pos = cast( + np.ndarray, + np.interp( + bin_edges_impose, + bin_edges_pos, + np.cumsum(np.append([0], bin_counts_pos)), + ), ) - bin_counts_impose_neg: npt.NDArray[np.float64] = np.interp( - bin_edges_impose, - bin_edges_neg, - np.cumsum(np.append([0], bin_counts_neg)), + bin_counts_impose_neg = cast( + np.ndarray, + np.interp( + bin_edges_impose, + bin_edges_neg, + np.cumsum(np.append([0], bin_counts_neg)), + ), ) - bin_counts_impose: npt.NDArray[np.float64] = ( - bin_counts_impose_pos + bin_counts_impose_neg + bin_counts_impose = cast( + np.ndarray, bin_counts_impose_pos + bin_counts_impose_neg ) median_inds = np.abs(bin_counts_impose - 0.5) < 1e-10 @@ -1726,9 +1737,10 @@ def _get_quantiles(self) -> None: :return: list of quantiles """ - percentiles: np.ndarray = np.linspace(0, 100, (self._num_quantiles - 1) + 2)[ - 1:-1 - ] + percentile_range = cast( + np.ndarray, np.linspace(0, 100, (self._num_quantiles - 1) + 2) + ) + percentiles: np.ndarray = percentile_range[1:-1] self.quantiles = self._get_percentile(percentiles=percentiles) def _update_helper(self, df_series_clean: pd.Series, profile: dict) -> None: @@ -1821,11 +1833,15 @@ def _get_variance( # Suppress any numpy warnings as we have a custom warning for invalid # or infinite data already with np.errstate(all="ignore"): - batch_biased_variance = np.var(df_series) # Obtains biased variance + batch_biased_variance = cast(float | np.float64, np.var(df_series)) subset_properties["biased_variance"] = batch_biased_variance sum_value = subset_properties["sum"] batch_count = subset_properties["match_count"] - batch_mean = 0.0 if not batch_count else float(sum_value) / batch_count + batch_mean = ( + 0.0 + if not batch_count + else profiler_utils.as_float_scalar(sum_value) / batch_count + ) subset_properties["mean"] = batch_mean self._biased_variance = self._merge_biased_variance( self.match_count, diff --git a/dataprofiler/profilers/order_column_profile.py b/dataprofiler/profilers/order_column_profile.py index 308262324..0a4374312 100644 --- a/dataprofiler/profilers/order_column_profile.py +++ b/dataprofiler/profilers/order_column_profile.py @@ -1,4 +1,5 @@ """Index profile analysis for individual col within structured profiling.""" + from __future__ import annotations from abc import abstractmethod diff --git a/dataprofiler/profilers/profile_builder.py b/dataprofiler/profilers/profile_builder.py index 6e512658f..fbc212a73 100644 --- a/dataprofiler/profilers/profile_builder.py +++ b/dataprofiler/profilers/profile_builder.py @@ -12,7 +12,7 @@ from collections import OrderedDict, defaultdict from datetime import datetime from multiprocessing.pool import Pool -from typing import Any, Generator, List, Optional, TypeVar, cast +from typing import Any, Generator, List, Optional, Sized, TypeVar, cast import networkx as nx import numpy as np @@ -612,7 +612,7 @@ def clean_data_and_get_base_stats( ) else: sample_ind_generator = profiler_utils.partition( - sample_ids[0], chunk_size=sample_size + list(sample_ids[0]), chunk_size=sample_size ) na_columns: dict = dict() @@ -738,7 +738,7 @@ def __init__( self._min_sample_size: int = 5000 # assign data labeler - data_labeler_options = self.options.data_labeler + data_labeler_options = cast(Any, self.options).data_labeler if ( data_labeler_options.is_enabled and data_labeler_options.data_labeler_object is None @@ -751,11 +751,13 @@ def __init__( dirpath=data_labeler_options.data_labeler_dirpath, load_options=None, ) - self.options.set({"data_labeler.data_labeler_object": data_labeler}) + cast(Any, self.options).set( + {"data_labeler.data_labeler_object": data_labeler} + ) except Exception as e: profiler_utils.warn_on_profile("data_labeler", e) - self.options.set({"data_labeler.is_enabled": False}) + cast(Any, self.options).set({"data_labeler.is_enabled": False}) def _add_error_checks(self, other: BaseProfiler) -> None: """ @@ -976,7 +978,7 @@ def update_profile( f"one of the following: {self._allowed_external_data_types}" ) - if not len(data): + if not len(cast(Sized, data)): warnings.warn( "The passed dataset was empty, hence no data was " "profiled." ) @@ -1919,10 +1921,10 @@ def diff( # type: ignore[override] col_name = other_profile._profile[i].name other_profile_schema[col_name].append(i) - report["global_stats"][ - "profile_schema" - ] = profiler_utils.find_diff_of_dicts_with_diff_keys( - self_profile_schema, other_profile_schema + report["global_stats"]["profile_schema"] = ( + profiler_utils.find_diff_of_dicts_with_diff_keys( + self_profile_schema, other_profile_schema + ) ) # Only find the diff of columns if the schemas are exactly the same @@ -2101,9 +2103,9 @@ def report(self, report_options: dict = None) -> dict: self.options.null_replication_metrics.is_enabled and i in self._null_replication_metrics ): - report["data_stats"][i][ - "null_replication_metrics" - ] = self._null_replication_metrics[i] + report["data_stats"][i]["null_replication_metrics"] = ( + self._null_replication_metrics[i] + ) return _prepare_report(report, output_format, omit_keys) @@ -2344,11 +2346,11 @@ def _get_correlation( # fill correlation matrix with nan initially n_cols = len(self._profile) - corr_mat = np.full((n_cols, n_cols), np.nan) + corr_mat = cast(np.ndarray, np.full((n_cols, n_cols), np.nan)) # then, fill in the correlations for valid columns - rows = [[id] for id in clean_column_ids] - corr_mat[rows, clean_column_ids] = np.corrcoef(data, rowvar=False) + corr_rows = np.ix_(clean_column_ids, clean_column_ids) + corr_mat[corr_rows] = np.corrcoef(data, rowvar=False) return corr_mat @@ -2397,26 +2399,28 @@ def _merge_correlation(self, other: StructuredProfiler) -> pd.DataFrame: return None # get column indices without nan - col_ids1 = np.where(~np.isnan(corr_mat1).all(axis=0))[0] - col_ids2 = np.where(~np.isnan(corr_mat2).all(axis=0))[0] + col_ids1 = cast(np.ndarray, np.where(~np.isnan(corr_mat1).all(axis=0))[0]) + col_ids2 = cast(np.ndarray, np.where(~np.isnan(corr_mat2).all(axis=0))[0]) if len(col_ids1) != len(col_ids2) or len(col_ids1) <= 1: return None - if (col_ids1 != col_ids2).any(): + col_ids1_list = cast(list[int], col_ids1.tolist()) + col_ids2_list = cast(list[int], col_ids2.tolist()) + if col_ids1_list != col_ids2_list: return None mean1 = np.array( [ self._profile[idx].profile["statistics"]["mean"] for idx in range(len(self._profile)) - if idx in col_ids1 + if idx in col_ids1_list ] ) std1 = np.array( [ self._profile[idx].profile["statistics"]["stddev"] for idx in range(len(self._profile)) - if idx in col_ids1 + if idx in col_ids1_list ] ) @@ -2424,14 +2428,14 @@ def _merge_correlation(self, other: StructuredProfiler) -> pd.DataFrame: [ other._profile[idx].profile["statistics"]["mean"] for idx in range(len(self._profile)) - if idx in col_ids2 + if idx in col_ids2_list ] ) std2 = np.array( [ other._profile[idx].profile["statistics"]["stddev"] for idx in range(len(self._profile)) - if idx in col_ids2 + if idx in col_ids2_list ] ) return self._merge_correlation_helper( @@ -2559,7 +2563,7 @@ def _update_chi2(self) -> np.ndarray: """ n_cols = len(self._profile) # Fill matrix with nan initially - chi2_mat = np.full((n_cols, n_cols), np.nan) + chi2_mat = cast(np.ndarray, np.full((n_cols, n_cols), np.nan)) # Compute chi_sq for each for i in range(n_cols): data_stats_compiler1 = self._profile[i].profiles["data_stats_profile"] @@ -2568,7 +2572,7 @@ def _update_chi2(self) -> np.ndarray: continue for j in range(i, n_cols): if i == j: - chi2_mat[i][j] = 1 + chi2_mat[i, j] = 1 continue data_stats_compiler2 = self._profile[j].profiles["data_stats_profile"] profiler2 = data_stats_compiler2._profiles["category"] @@ -2581,8 +2585,8 @@ def _update_chi2(self) -> np.ndarray: profiler2.categorical_counts, profiler2.sample_size, ) - chi2_mat[i][j] = results["p-value"] - chi2_mat[j][i] = results["p-value"] + chi2_mat[i, j] = results["p-value"] + chi2_mat[j, i] = results["p-value"] return chi2_mat @@ -2610,9 +2614,11 @@ def _update_null_replication_metrics(self, clean_samples: dict) -> None: total_row_sum = np.asarray( [ - get_data_type_profiler(profile).sum - if get_data_type(profile) not in [None, "datetime"] - else np.nan + ( + get_data_type_profiler(profile).sum + if get_data_type(profile) not in [None, "datetime"] + else np.nan + ) for profile in self._profile ] ) @@ -2672,18 +2678,19 @@ def _update_null_replication_metrics(self, clean_samples: dict) -> None: mean_not_null = sum_not_null / true_count # Convert numpy arrays to lists (serializable) - sum_null = sum_null.tolist() - sum_not_null = sum_not_null.tolist() + sum_null_list = cast(list[float], sum_null.tolist()) + sum_not_null_list = cast(list[float], sum_not_null.tolist()) - mean_null = mean_null.tolist() - mean_not_null = mean_not_null.tolist() + mean_null_list = cast(list[float], mean_null.tolist()) + mean_not_null_array = cast(np.ndarray, mean_not_null) + mean_not_null_list = cast(list[float], mean_not_null_array.tolist()) # Array index serves as class label # 0 indicates not null, 1 indicates null self._null_replication_metrics[col_id] = { "class_prior": [prior_not_null, prior_null], - "class_sum": [sum_not_null, sum_null], - "class_mean": [mean_not_null, mean_null], + "class_sum": [sum_not_null_list, sum_null_list], + "class_mean": [mean_not_null_list, mean_null_list], } def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: @@ -2704,21 +2711,27 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: self_row_sum = np.asarray( [ - get_data_type_profiler(profile).sum - if get_data_type(profile) - else np.nan + ( + get_data_type_profiler(profile).sum + if get_data_type(profile) + else np.nan + ) for profile in self._profile ] ) other_row_sum = np.asarray( [ - get_data_type_profiler(profile).sum - if get_data_type(profile) - else np.nan + ( + get_data_type_profiler(profile).sum + if get_data_type(profile) + else np.nan + ) for profile in other._profile ] ) - total_row_sum: np.ndarray = self_row_sum + other_row_sum + self_row_sum = cast(np.ndarray, self_row_sum) + other_row_sum = cast(np.ndarray, other_row_sum) + total_row_sum = cast(np.ndarray, self_row_sum + other_row_sum) merged_properties: dict = defaultdict(dict) for col_id in range(len(self._profile)): self_profile = self._profile[col_id] @@ -2749,7 +2762,7 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: else None ) # Initialize zeros array of size (number of columns - 1) - sum_null = np.zeros(len(self._profile) - 1) + sum_null = cast(np.ndarray, np.zeros(len(self._profile) - 1)) # Add sum_nulls if they exist # Guarantees that at least one of self_sum_null, other_sum_null != None @@ -2759,7 +2772,7 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: if other_sum_null is not None: sum_null += np.asarray(other_sum_null) - sum_not_null = np.delete(total_row_sum, col_id) - sum_null + sum_not_null = cast(np.ndarray, np.delete(total_row_sum, col_id) - sum_null) mean_null = sum_null / null_count @@ -2770,18 +2783,19 @@ def _merge_null_replication_metrics(self, other: StructuredProfiler) -> dict: mean_not_null = sum_not_null / true_count # Convert numpy arrays to lists (serializable) - sum_null = sum_null.tolist() - sum_not_null = sum_not_null.tolist() + sum_null_list = cast(list[float], sum_null.tolist()) + sum_not_null_list = cast(list[float], sum_not_null.tolist()) - mean_null = mean_null.tolist() - mean_not_null = mean_not_null.tolist() + mean_null_list = cast(list[float], mean_null.tolist()) + mean_not_null_array = cast(np.ndarray, mean_not_null) + mean_not_null_list = cast(list[float], mean_not_null_array.tolist()) merged_properties[col_id] = { # Array index serves as class label # 0 indicates not null, 1 indicates null "class_prior": [prior_not_null, prior_null], - "class_sum": [sum_not_null, sum_null], - "class_mean": [mean_not_null, mean_null], + "class_sum": [sum_not_null_list, sum_null_list], + "class_mean": [mean_not_null_list, mean_null_list], } return merged_properties diff --git a/dataprofiler/profilers/profiler_options.py b/dataprofiler/profilers/profiler_options.py index 038acf806..6fb3eeafa 100644 --- a/dataprofiler/profilers/profiler_options.py +++ b/dataprofiler/profilers/profiler_options.py @@ -9,7 +9,7 @@ from typing import Any, Generic, TypeVar, cast from ..labelers.base_data_labeler import BaseDataLabeler -from ..plugins.__init__ import get_plugins +from ..plugins import get_plugins from . import profiler_utils from .json_decoder import load_option diff --git a/dataprofiler/profilers/profiler_utils.py b/dataprofiler/profilers/profiler_utils.py index e38e1b041..8d903abee 100644 --- a/dataprofiler/profilers/profiler_utils.py +++ b/dataprofiler/profilers/profiler_utils.py @@ -1,4 +1,5 @@ """Contains functions for profilers.""" + from __future__ import annotations import collections @@ -39,6 +40,18 @@ from .. import rng_utils +def as_float_scalar( + value: int | float | np.integer | np.floating | np.ndarray | list[float], +) -> float: + """Convert a scalar-like value to a Python float.""" + array_value = cast(np.ndarray, np.asarray(value)) + if array_value.ndim == 0: + return float(array_value) + if array_value.size == 1: + return float(array_value.item()) + raise TypeError("Expected a scalar-like numeric value.") + + def recursive_dict_update(d: dict, update_d: dict) -> dict: """ Recursive updates nested dictionaries. Updating d with update_d. @@ -122,13 +135,13 @@ def shuffle_in_chunks( values = [-1] * true_chunk_size # Generate random list of indexes - lower_bound_list = np.array(range(j, j + true_chunk_size)) - random_list = rng.integers(lower_bound_list, data_length) + lower_bound_list = cast(np.ndarray, np.array(range(j, j + true_chunk_size))) + random_list = cast(np.ndarray, rng.integers(lower_bound_list, data_length)) # shuffle the indexes for count in range(true_chunk_size): # get a random index to swap and swap it with j - k = random_list[count] + k = int(random_list[count]) indices[j], indices[k] = indices[k], indices[j] # set the swapped value to the output @@ -417,13 +430,11 @@ def __sub__(self: T, other: T) -> Any: def find_diff_of_numbers( stat1: int | float | np.float64 | np.int64 | None, stat2: int | float | np.float64 | np.int64 | None, -) -> Any: - ... +) -> Any: ... @overload -def find_diff_of_numbers(stat1: T | None, stat2: T | None) -> Any: - ... +def find_diff_of_numbers(stat1: T | None, stat2: T | None) -> Any: ... def find_diff_of_numbers(stat1, stat2): @@ -599,11 +610,11 @@ def find_diff_of_matrices( :rtype: list(list(float)) """ if matrix1 is not None and matrix2 is not None: - mat1 = np.array(matrix1, dtype=np.float64) - mat2 = np.array(matrix2, dtype=np.float64) + mat1 = cast(np.ndarray, np.array(matrix1, dtype=np.float64)) + mat2 = cast(np.ndarray, np.array(matrix2, dtype=np.float64)) - if mat1.shape == mat2.shape: - diff: np.ndarray = mat1 - mat2 + if np.shape(mat1) == np.shape(mat2): + diff: np.ndarray = cast(np.ndarray, mat1 - mat2) if ((diff == 0) | np.isnan(diff)).all(): return "unchanged" return diff diff --git a/dataprofiler/profilers/text_column_profile.py b/dataprofiler/profilers/text_column_profile.py index bea8dbd68..48333759c 100644 --- a/dataprofiler/profilers/text_column_profile.py +++ b/dataprofiler/profilers/text_column_profile.py @@ -1,7 +1,9 @@ """Text profile analysis for individual col within structured profiling..""" + from __future__ import annotations import itertools +from typing import cast import numpy as np import pandas as pd @@ -90,7 +92,7 @@ def profile(self) -> dict: :return: """ - profile = NumericStatsMixin.profile(self) + profile = cast(dict, NumericStatsMixin.profile.__get__(self, type(self))) # remove num_zeros and num_negative updated from numeric profile profile.pop("num_zeros") profile.pop("num_negatives") diff --git a/dataprofiler/profilers/unstructured_labeler_profile.py b/dataprofiler/profilers/unstructured_labeler_profile.py index 1c7b16c0f..22789c4e3 100644 --- a/dataprofiler/profilers/unstructured_labeler_profile.py +++ b/dataprofiler/profilers/unstructured_labeler_profile.py @@ -1,4 +1,5 @@ """Profile analysis for applying labels within unstructured profiling.""" + from __future__ import annotations from collections import defaultdict diff --git a/dataprofiler/profilers/unstructured_text_profile.py b/dataprofiler/profilers/unstructured_text_profile.py index 96b7d0625..3f1b6dd7f 100644 --- a/dataprofiler/profilers/unstructured_text_profile.py +++ b/dataprofiler/profilers/unstructured_text_profile.py @@ -1,4 +1,5 @@ """For profiling unstructured text data.""" + from __future__ import annotations import itertools diff --git a/dataprofiler/reports/graphs.py b/dataprofiler/reports/graphs.py index 1f0b43018..4d2672df6 100644 --- a/dataprofiler/reports/graphs.py +++ b/dataprofiler/reports/graphs.py @@ -1,4 +1,5 @@ """Contains functions for generating graph data report.""" + # !/usr/bin/env python3 from __future__ import annotations @@ -32,7 +33,7 @@ def plot_histograms( profiler: StructuredProfiler, column_names: list[int | str] | None = None, column_inds: list[int] | None = None, -) -> matplotlib.pyplot.figure: +) -> matplotlib.figure.Figure | None: """ Plot the histograms of column names that are int or float columns. @@ -108,7 +109,7 @@ def is_index_graphable_column(ind_to_graph: int) -> bool: "No plots were constructed" " because no int or float columns were found in columns" ) - return + return None # get proper tile format for graph n = len(inds_to_graph) @@ -191,7 +192,7 @@ def plot_missing_values_matrix( profiler: StructuredProfiler, ax: matplotlib.axes.Axes | None = None, title: str | None = None, -) -> matplotlib.pyplot.figure: +) -> matplotlib.figure.Figure | None: """ Generate matrix of bar graphs for missing value locations in cols of struct dataset. @@ -215,7 +216,7 @@ def plot_col_missing_values( col_profiler_list: list[StructuredColProfiler], ax: matplotlib.axes.Axes | None = None, title: str | None = None, -) -> matplotlib.pyplot.figure: +) -> matplotlib.figure.Figure | None: """ Generate bar graph of missing value locations within a col. @@ -243,7 +244,7 @@ def plot_col_missing_values( warnings.warn( "There was no data in the profiles to plot missing " "column values." ) - return + return None # bar width settings and height settings for each null value # width = 1, height = 1 would be no gaps @@ -262,7 +263,7 @@ def plot_col_missing_values( ax = fig.add_subplot(111) is_own_fig = True # in case user passed their own axes - fig = ax.figure + fig = cast(matplotlib.figure.Figure, ax.figure) # loop through eac column plotting their null values for col_id, col_profiler in enumerate(col_profiler_list): diff --git a/dataprofiler/reports/utils.py b/dataprofiler/reports/utils.py index a10b8fe59..975dc7d88 100644 --- a/dataprofiler/reports/utils.py +++ b/dataprofiler/reports/utils.py @@ -1,4 +1,5 @@ """Contains functions for checking for installations/dependencies.""" + import sys import warnings from typing import Any, Callable, List, TypeVar, cast diff --git a/resources/labelers/column_name_labeler/data_labeler_parameters.json b/dataprofiler/resources/labelers/column_name_labeler/data_labeler_parameters.json similarity index 100% rename from resources/labelers/column_name_labeler/data_labeler_parameters.json rename to dataprofiler/resources/labelers/column_name_labeler/data_labeler_parameters.json diff --git a/resources/labelers/column_name_labeler/label_mapping.json b/dataprofiler/resources/labelers/column_name_labeler/label_mapping.json similarity index 100% rename from resources/labelers/column_name_labeler/label_mapping.json rename to dataprofiler/resources/labelers/column_name_labeler/label_mapping.json diff --git a/resources/labelers/column_name_labeler/model_parameters.json b/dataprofiler/resources/labelers/column_name_labeler/model_parameters.json similarity index 100% rename from resources/labelers/column_name_labeler/model_parameters.json rename to dataprofiler/resources/labelers/column_name_labeler/model_parameters.json diff --git a/resources/labelers/column_name_labeler/postprocessor_parameters.json b/dataprofiler/resources/labelers/column_name_labeler/postprocessor_parameters.json similarity index 100% rename from resources/labelers/column_name_labeler/postprocessor_parameters.json rename to dataprofiler/resources/labelers/column_name_labeler/postprocessor_parameters.json diff --git a/resources/labelers/column_name_labeler/preprocessor_parameters.json b/dataprofiler/resources/labelers/column_name_labeler/preprocessor_parameters.json similarity index 100% rename from resources/labelers/column_name_labeler/preprocessor_parameters.json rename to dataprofiler/resources/labelers/column_name_labeler/preprocessor_parameters.json diff --git a/resources/labelers/regex_model/data_labeler_parameters.json b/dataprofiler/resources/labelers/regex_model/data_labeler_parameters.json similarity index 100% rename from resources/labelers/regex_model/data_labeler_parameters.json rename to dataprofiler/resources/labelers/regex_model/data_labeler_parameters.json diff --git a/resources/labelers/regex_model/label_mapping.json b/dataprofiler/resources/labelers/regex_model/label_mapping.json similarity index 100% rename from resources/labelers/regex_model/label_mapping.json rename to dataprofiler/resources/labelers/regex_model/label_mapping.json diff --git a/resources/labelers/regex_model/model_parameters.json b/dataprofiler/resources/labelers/regex_model/model_parameters.json similarity index 100% rename from resources/labelers/regex_model/model_parameters.json rename to dataprofiler/resources/labelers/regex_model/model_parameters.json diff --git a/resources/labelers/regex_model/postprocessor_parameters.json b/dataprofiler/resources/labelers/regex_model/postprocessor_parameters.json similarity index 100% rename from resources/labelers/regex_model/postprocessor_parameters.json rename to dataprofiler/resources/labelers/regex_model/postprocessor_parameters.json diff --git a/resources/labelers/regex_model/preprocessor_parameters.json b/dataprofiler/resources/labelers/regex_model/preprocessor_parameters.json similarity index 100% rename from resources/labelers/regex_model/preprocessor_parameters.json rename to dataprofiler/resources/labelers/regex_model/preprocessor_parameters.json diff --git a/resources/labelers/structured_model/data_labeler_parameters.json b/dataprofiler/resources/labelers/structured_model/data_labeler_parameters.json similarity index 100% rename from resources/labelers/structured_model/data_labeler_parameters.json rename to dataprofiler/resources/labelers/structured_model/data_labeler_parameters.json diff --git a/resources/labelers/structured_model/label_mapping.json b/dataprofiler/resources/labelers/structured_model/label_mapping.json similarity index 100% rename from resources/labelers/structured_model/label_mapping.json rename to dataprofiler/resources/labelers/structured_model/label_mapping.json diff --git a/resources/labelers/structured_model/variables/variables.data-00000-of-00001 b/dataprofiler/resources/labelers/structured_model/model.keras similarity index 88% rename from resources/labelers/structured_model/variables/variables.data-00000-of-00001 rename to dataprofiler/resources/labelers/structured_model/model.keras index 95732bf16..795d637da 100644 Binary files a/resources/labelers/structured_model/variables/variables.data-00000-of-00001 and b/dataprofiler/resources/labelers/structured_model/model.keras differ diff --git a/resources/labelers/structured_model/model_parameters.json b/dataprofiler/resources/labelers/structured_model/model_parameters.json similarity index 100% rename from resources/labelers/structured_model/model_parameters.json rename to dataprofiler/resources/labelers/structured_model/model_parameters.json diff --git a/resources/labelers/structured_model/postprocessor_parameters.json b/dataprofiler/resources/labelers/structured_model/postprocessor_parameters.json similarity index 100% rename from resources/labelers/structured_model/postprocessor_parameters.json rename to dataprofiler/resources/labelers/structured_model/postprocessor_parameters.json diff --git a/resources/labelers/structured_model/preprocessor_parameters.json b/dataprofiler/resources/labelers/structured_model/preprocessor_parameters.json similarity index 100% rename from resources/labelers/structured_model/preprocessor_parameters.json rename to dataprofiler/resources/labelers/structured_model/preprocessor_parameters.json diff --git a/resources/labelers/unstructured_model/data_labeler_parameters.json b/dataprofiler/resources/labelers/unstructured_model/data_labeler_parameters.json similarity index 100% rename from resources/labelers/unstructured_model/data_labeler_parameters.json rename to dataprofiler/resources/labelers/unstructured_model/data_labeler_parameters.json diff --git a/resources/labelers/unstructured_model/label_mapping.json b/dataprofiler/resources/labelers/unstructured_model/label_mapping.json similarity index 100% rename from resources/labelers/unstructured_model/label_mapping.json rename to dataprofiler/resources/labelers/unstructured_model/label_mapping.json diff --git a/resources/labelers/unstructured_model/variables/variables.data-00000-of-00001 b/dataprofiler/resources/labelers/unstructured_model/model.keras similarity index 88% rename from resources/labelers/unstructured_model/variables/variables.data-00000-of-00001 rename to dataprofiler/resources/labelers/unstructured_model/model.keras index 95732bf16..795d637da 100644 Binary files a/resources/labelers/unstructured_model/variables/variables.data-00000-of-00001 and b/dataprofiler/resources/labelers/unstructured_model/model.keras differ diff --git a/resources/labelers/unstructured_model/model_parameters.json b/dataprofiler/resources/labelers/unstructured_model/model_parameters.json similarity index 100% rename from resources/labelers/unstructured_model/model_parameters.json rename to dataprofiler/resources/labelers/unstructured_model/model_parameters.json diff --git a/resources/labelers/unstructured_model/postprocessor_parameters.json b/dataprofiler/resources/labelers/unstructured_model/postprocessor_parameters.json similarity index 100% rename from resources/labelers/unstructured_model/postprocessor_parameters.json rename to dataprofiler/resources/labelers/unstructured_model/postprocessor_parameters.json diff --git a/resources/labelers/unstructured_model/preprocessor_parameters.json b/dataprofiler/resources/labelers/unstructured_model/preprocessor_parameters.json similarity index 100% rename from resources/labelers/unstructured_model/preprocessor_parameters.json rename to dataprofiler/resources/labelers/unstructured_model/preprocessor_parameters.json diff --git a/dataprofiler/rng_utils.py b/dataprofiler/rng_utils.py index 329066658..2fd14f0f7 100644 --- a/dataprofiler/rng_utils.py +++ b/dataprofiler/rng_utils.py @@ -1,4 +1,5 @@ """Create a random number generator using a manual seed DATAPROFILER_SEED.""" + import os import warnings diff --git a/dataprofiler/settings.py b/dataprofiler/settings.py index 1ba017f4e..a81c3477c 100644 --- a/dataprofiler/settings.py +++ b/dataprofiler/settings.py @@ -1,2 +1,3 @@ """Configure settings for dataprofiler.""" + _seed = None diff --git a/dataprofiler/tests/labelers/test_char_tf_load_model.py b/dataprofiler/tests/labelers/test_char_tf_load_model.py index fbfde0c49..6160b8fa4 100644 --- a/dataprofiler/tests/labelers/test_char_tf_load_model.py +++ b/dataprofiler/tests/labelers/test_char_tf_load_model.py @@ -6,14 +6,13 @@ import numpy as np import pandas as pd -import pkg_resources import tensorflow as tf +from dataprofiler.labelers import utils as labeler_utils from dataprofiler.labelers.char_load_tf_model import CharLoadTFModel _file_dir = os.path.dirname(os.path.abspath(__file__)) -_resource_labeler_dir = pkg_resources.resource_filename("resources", "labelers") - +default_labeler_dir = labeler_utils.find_resources_dir("labelers") mock_model_parameters = { "model_path": "project/example/path/fake_model.h5", @@ -228,6 +227,26 @@ def test_predict(self, *mocks): self.assertIn("conf", result) self.assertEqual((2, 2, model.num_labels), np.array(result["conf"]).shape) + def test_normalize_old_list_output_model(self, *mocks): + inputs = tf.keras.Input(shape=(2,), dtype=tf.int64) + embedded = tf.keras.layers.Embedding(input_dim=100, output_dim=8)(inputs) + softmax_output = tf.keras.layers.Dense( + self.label_mapping["ADDRESS"] + 1, + activation="softmax", + )(embedded) + argmax_output = tf.keras.layers.Lambda( + lambda x: tf.cast(tf.argmax(x, axis=2), tf.int64) + )(softmax_output) + old_format_model = tf.keras.Model(inputs, [softmax_output, argmax_output]) + + normalized_model = CharLoadTFModel._normalize_model_outputs(old_format_model) + + self.assertIsInstance(normalized_model.output, dict) + self.assertSetEqual( + set(normalized_model.output.keys()), + {CharLoadTFModel._SOFTMAX_OUTPUT, CharLoadTFModel._ARGMAX_OUTPUT}, + ) + def test_fit_and_predict(self, *mocks): # model model = CharLoadTFModel(self.model_path, self.label_mapping) @@ -272,7 +291,7 @@ def test_fit_and_predict(self, *mocks): ) # predict after fitting on just the text - model.predict(data_gen[0][0]) + model.predict([data_gen[0][0]]) @mock.patch("os.makedirs", return_value=None) def test_validation_evaluate_and_classification_report(self, *mocks): @@ -303,7 +322,9 @@ def test_param_validation(self, *mocks): "fake_extra_param": "fails", } model = CharLoadTFModel( - self.model_path, label_mapping=self.label_mapping, parameters=parameters + self.model_path, + label_mapping=self.label_mapping, + parameters=parameters, ) model._construct_model() self.assertDictEqual(parameters, model._parameters) diff --git a/dataprofiler/tests/labelers/test_character_level_cnn_model.py b/dataprofiler/tests/labelers/test_character_level_cnn_model.py index ad549cc53..311da0066 100644 --- a/dataprofiler/tests/labelers/test_character_level_cnn_model.py +++ b/dataprofiler/tests/labelers/test_character_level_cnn_model.py @@ -6,13 +6,18 @@ import numpy as np import pandas as pd -import pkg_resources import tensorflow as tf -from dataprofiler.labelers.character_level_cnn_model import CharacterLevelCnnModel +from dataprofiler.labelers import utils as labeler_utils +from dataprofiler.labelers.character_level_cnn_model import ( + ArgMaxLayer, + CharacterLevelCnnModel, + EncodingLayer, + ThreshArgMaxLayer, +) _file_dir = os.path.dirname(os.path.abspath(__file__)) -_resource_labeler_dir = pkg_resources.resource_filename("resources", "labelers") +_resource_labeler_dir = labeler_utils.find_resources_dir("labelers") mock_model_parameters = { @@ -250,6 +255,36 @@ def test_validation_evaluate_and_classification_report(self, *mocks): self.assertIsNotNone(f1_report) self.assertEqual(11, f1_report["ADDRESS"]["support"]) + def test_normalize_old_list_output_model(self): + default_ind = self.label_mapping["UNKNOWN"] + num_labels = max(self.label_mapping.values()) + 1 + inputs = tf.keras.Input(shape=(2, 4)) + hidden = tf.keras.layers.Dense(8, activation="relu")(inputs) + softmax_output = tf.keras.layers.Dense(num_labels, activation="softmax")(hidden) + argmax_output = ArgMaxLayer()(softmax_output) + threshold_output = ThreshArgMaxLayer( + threshold_=0.0, + num_labels_=num_labels, + default_ind=default_ind, + )(argmax_output, softmax_output) + old_format_model = tf.keras.Model( + inputs, [softmax_output, argmax_output, threshold_output] + ) + + normalized_model = CharacterLevelCnnModel._normalize_model_outputs( + old_format_model, default_ind, num_labels + ) + + self.assertIsInstance(normalized_model.output, dict) + self.assertSetEqual( + set(normalized_model.output.keys()), + { + CharacterLevelCnnModel._SOFTMAX_OUTPUT, + CharacterLevelCnnModel._ARGMAX_OUTPUT, + CharacterLevelCnnModel._THRESH_OUTPUT, + }, + ) + def test_fit_and_predict_with_new_labels(self): # Initialize model cnn_model = CharacterLevelCnnModel(self.label_mapping) @@ -272,7 +307,7 @@ def test_fit_and_predict_with_new_labels(self): ) # predict after fitting on just the text - cnn_model.predict(data_gen[0][0]) + cnn_model.predict([data_gen[0][0]]) def test_fit_and_predict_with_new_labels_set_via_method(self): # Initialize model @@ -301,7 +336,7 @@ def test_fit_and_predict_with_new_labels_set_via_method(self): history, f1, f1_report = cnn_model.fit(data_gen, cv_gen) # test predict on just the text - cnn_model.predict(data_gen[0][0]) + cnn_model.predict([data_gen[0][0]]) def test_validation(self): @@ -368,9 +403,8 @@ def test_input_encoding(self): max_char_encoding_id = 127 max_len = 10 - encode_output = cnn_model._char_encoding_layer( - input_str_tensor, max_char_encoding_id, max_len - ).numpy()[0] + encode_layer = EncodingLayer(max_char_encoding_id, max_len) + encode_output = encode_layer.call(input_str_tensor).numpy()[0] expected_output = [117, 102, 116, 117, 0, 0, 0, 0, 0, 0] self.assertCountEqual(encode_output, expected_output) @@ -428,7 +462,7 @@ def test_save(self, mock_open, *mocks): StringIO.close(mock_file) @mock.patch("tensorflow.keras.Model.save", return_value=None) - @mock.patch("tensorflow.keras.models.load_model", return_value=mock.Mock()) + @mock.patch("tensorflow.keras.models.load_model", return_value=mock.MagicMock()) @mock.patch("builtins.open", side_effect=mock_open) def test_load(self, *mocks): dir = os.path.join(_resource_labeler_dir, "unstructured_model/") @@ -444,8 +478,8 @@ def test_model_construct(self): cnn_model.details() expected_layers = [ - "input_1", - "lambda", + "input_layer", + "encoding_layer", "embedding", "conv1d", "dropout", @@ -463,9 +497,9 @@ def test_model_construct(self): "dropout_4", "dense_1", "dropout_5", - "dense_2", - "tf_op_layer_ArgMax", - "thresh_arg_max_layer", + "softmax_output", + "argmax_output", + "thresh_argmax_output", ] model_layers = [layer.name for layer in cnn_model._model.layers] self.assertEqual(len(expected_layers), len(model_layers)) diff --git a/dataprofiler/tests/labelers/test_column_name_model.py b/dataprofiler/tests/labelers/test_column_name_model.py index 58f90839e..dfd4274ee 100644 --- a/dataprofiler/tests/labelers/test_column_name_model.py +++ b/dataprofiler/tests/labelers/test_column_name_model.py @@ -6,13 +6,13 @@ from unittest import mock import numpy as np -import pkg_resources import dataprofiler as dp +from dataprofiler.labelers import utils as labeler_utils from dataprofiler.labelers.column_name_model import ColumnNameModel -_file_dir = os.path.dirname(os.path.abspath(__file__)) -_resource_labeler_dir = pkg_resources.resource_filename("resources", "labelers") +_resource_labeler_dir = labeler_utils.find_resources_dir("labelers") + mock_model_parameters = { "true_positive_dict": [ diff --git a/dataprofiler/tests/labelers/test_data_labelers.py b/dataprofiler/tests/labelers/test_data_labelers.py index bbde1c506..e7ef03837 100644 --- a/dataprofiler/tests/labelers/test_data_labelers.py +++ b/dataprofiler/tests/labelers/test_data_labelers.py @@ -149,11 +149,12 @@ def test_load_from_library(self, *mocks): @mock.patch("tensorflow.keras.models.load_model") def test_load_from_disk(self, *mocks): - import pkg_resources + from dataprofiler.labelers import utils as labeler_utils - default_labeler_dir = pkg_resources.resource_filename( - "resources", "labelers/structured_model" + default_labeler_dir = str( + labeler_utils.find_resources_dir("labelers/structured_model") ) + data_labeler = dp.DataLabeler.load_from_disk(default_labeler_dir) self.assertIsInstance(data_labeler, BaseDataLabeler) @@ -398,7 +399,7 @@ def test_has_public_functions(self, *args): @staticmethod def _setup_mock_load_model(mock_load_model): - mock_load_model.return_value = mock.Mock() + mock_load_model.return_value = mock.MagicMock() def test_load_labeler(self, mock_open, mock_load_model): diff --git a/dataprofiler/tests/labelers/test_data_processing.py b/dataprofiler/tests/labelers/test_data_processing.py index 00b4b088b..7624cccab 100644 --- a/dataprofiler/tests/labelers/test_data_processing.py +++ b/dataprofiler/tests/labelers/test_data_processing.py @@ -7,8 +7,8 @@ from unittest import mock import numpy as np -import pkg_resources +from dataprofiler.labelers import utils as labeler_utils from dataprofiler.labelers.data_processing import ( BaseDataProcessor, CharEncodedPreprocessor, @@ -224,7 +224,9 @@ def test_load_from_library(self, mocked_load, *mocks): BaseDataProcessor.load_from_library("default") # assert called with proper load_processor dirpath - default_labeler_dir = pkg_resources.resource_filename("resources", "labelers") + + default_labeler_dir = labeler_utils.find_resources_dir("labelers") + mocked_load.assert_called_with(os.path.join(default_labeler_dir, "default")) @mock.patch("builtins.open") diff --git a/dataprofiler/tests/labelers/test_integration_column_name_data_labeler.py b/dataprofiler/tests/labelers/test_integration_column_name_data_labeler.py index bcc136ae3..8b19731f1 100644 --- a/dataprofiler/tests/labelers/test_integration_column_name_data_labeler.py +++ b/dataprofiler/tests/labelers/test_integration_column_name_data_labeler.py @@ -1,9 +1,9 @@ import unittest import numpy as np -import pkg_resources import dataprofiler as dp +from dataprofiler.labelers import utils as labeler_utils from dataprofiler.labelers.column_name_model import ColumnNameModel from dataprofiler.labelers.data_labelers import BaseDataLabeler from dataprofiler.labelers.data_processing import ( @@ -11,7 +11,7 @@ DirectPassPreprocessor, ) -default_labeler_dir = pkg_resources.resource_filename("resources", "labelers") +default_labeler_dir = labeler_utils.find_resources_dir("labelers") class TestColumnNameDataLabeler(unittest.TestCase): diff --git a/dataprofiler/tests/labelers/test_integration_regex_data_labeler.py b/dataprofiler/tests/labelers/test_integration_regex_data_labeler.py index 1cb753723..df72b99e9 100644 --- a/dataprofiler/tests/labelers/test_integration_regex_data_labeler.py +++ b/dataprofiler/tests/labelers/test_integration_regex_data_labeler.py @@ -2,18 +2,25 @@ import unittest import numpy as np -import pkg_resources +from dataprofiler.labelers import utils as labeler_utils from dataprofiler.labelers.data_labelers import BaseDataLabeler -default_labeler_dir = pkg_resources.resource_filename("resources", "labelers") +default_labeler_dir = labeler_utils.find_resources_dir("labelers") class TestRegexDataLabeler(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.data = np.array( - ["123 Fake St.", "1/2/2020", "nice.", "4/3/22", "abc", "333-44-2341"] + [ + "123 Fake St.", + "1/2/2020", + "nice.", + "4/3/22", + "abc", + "333-44-2341", + ] ).reshape((-1,)) cls.data_labeler = BaseDataLabeler.load_from_disk( os.path.join(default_labeler_dir, "regex_model") diff --git a/dataprofiler/tests/labelers/test_labeler_utils.py b/dataprofiler/tests/labelers/test_labeler_utils.py index f59a43e3f..34dd280ad 100644 --- a/dataprofiler/tests/labelers/test_labeler_utils.py +++ b/dataprofiler/tests/labelers/test_labeler_utils.py @@ -1,12 +1,13 @@ import logging +import tempfile import unittest -from unittest import mock +from pathlib import Path import numpy as np import pandas as pd import tensorflow as tf -from dataprofiler.labelers import labeler_utils +from dataprofiler.labelers import labeler_utils, utils class TestEvaluateAccuracy(unittest.TestCase): @@ -235,9 +236,7 @@ def test_verbose(self): self.assertIn("f1-score ", log_output) self.assertIn("F1 Score: ", log_output) - @mock.patch("dataprofiler.labelers.labeler_utils.classification_report") - @mock.patch("pandas.DataFrame") - def test_save_conf_mat(self, mock_dataframe, mock_report): + def test_save_conf_mat(self): # ideally mock out the actual contents written to file, but # would be difficult to get this completely worked out. @@ -248,28 +247,25 @@ def test_save_conf_mat(self, mock_dataframe, mock_report): [0, 1, 2], ] ) - expected_row_col_names = dict( - columns=["pred:PAD", "pred:UNKNOWN", "pred:OTHER"], - index=["true:PAD", "true:UNKNOWN", "true:OTHER"], - ) - mock_instance_df = mock.Mock(spec=pd.DataFrame)() - mock_dataframe.return_value = mock_instance_df - - # still omit bc confusion mat should include all despite omit - f1, f1_report = labeler_utils.evaluate_accuracy( - self.y_pred, - self.y_true, - self.num_labels, - self.reverse_label_mapping, - omitted_labels=["PAD"], - verbose=False, - confusion_matrix_file="test.csv", - ) + expected_columns = ["pred:PAD", "pred:UNKNOWN", "pred:OTHER"] + expected_index = ["true:PAD", "true:UNKNOWN", "true:OTHER"] - self.assertTrue((mock_dataframe.call_args[0][0] == expected_conf_mat).all()) - self.assertDictEqual(expected_row_col_names, mock_dataframe.call_args[1]) + with tempfile.NamedTemporaryFile() as tmpFile: + # still omit bc confusion mat should include all despite omit + f1, f1_report = labeler_utils.evaluate_accuracy( + self.y_pred, + self.y_true, + self.num_labels, + self.reverse_label_mapping, + omitted_labels=["PAD"], + verbose=False, + confusion_matrix_file=tmpFile.name, + ) - mock_instance_df.to_csv.assert_called() + df1 = pd.read_csv(tmpFile.name, index_col=0) + self.assertListEqual(list(df1.columns), expected_columns) + self.assertListEqual(list(df1.index), expected_index) + np.testing.assert_array_equal(df1.values, expected_conf_mat) class TestTFFunctions(unittest.TestCase): @@ -299,3 +295,17 @@ def test_hide_tf_logger_warnings(self): # make change and validate updated filter labeler_utils.hide_tf_logger_warnings() self.assertEqual(1 + num_loggers, len(logger.filters)) + + +class TestFindResourcesDir(unittest.TestCase): + def test_find_resources_dir(self): + resource_dir = utils.find_resources_dir() + self.assertTrue(resource_dir.is_dir()) + self.assertEqual(Path("resources").name, Path(str(resource_dir)).name) + + labeler_dir = utils.find_resources_dir("labelers") + self.assertTrue(labeler_dir.is_dir()) + self.assertEqual(Path("labelers").name, Path(str(labeler_dir)).name) + + with self.assertRaisesRegex(FileNotFoundError, "Resource not found"): + utils.find_resources_dir("does/not/exist") diff --git a/dataprofiler/tests/labelers/test_regex_model.py b/dataprofiler/tests/labelers/test_regex_model.py index 6a279307e..66ac64482 100644 --- a/dataprofiler/tests/labelers/test_regex_model.py +++ b/dataprofiler/tests/labelers/test_regex_model.py @@ -5,12 +5,12 @@ from unittest import mock import numpy as np -import pkg_resources +from dataprofiler.labelers import utils as labeler_utils from dataprofiler.labelers.regex_model import RegexModel _file_dir = os.path.dirname(os.path.abspath(__file__)) -_resource_labeler_dir = pkg_resources.resource_filename("resources", "labelers") +_resource_labeler_dir = labeler_utils.find_resources_dir("labelers") mock_model_parameters = { @@ -161,7 +161,8 @@ def test_param_validation(self): for invalid_param_set in invalid_parameters: with self.assertRaises(ValueError): RegexModel( - label_mapping=self.label_mapping, parameters=invalid_param_set + label_mapping=self.label_mapping, + parameters=invalid_param_set, ) @mock.patch("sys.stdout", new_callable=StringIO) diff --git a/dataprofiler/tests/plugins/test_plugins.py b/dataprofiler/tests/plugins/test_plugins.py index ec148a526..91677e5c1 100644 --- a/dataprofiler/tests/plugins/test_plugins.py +++ b/dataprofiler/tests/plugins/test_plugins.py @@ -2,7 +2,7 @@ from collections import defaultdict from unittest import mock -from dataprofiler.plugins.__init__ import get_plugins, load_plugins +from dataprofiler.plugins import get_plugins, load_plugins from dataprofiler.plugins.decorators import plugin_decorator, plugins_dict @@ -24,12 +24,12 @@ def test_plugin(): test_get_dict = get_plugins("test") self.assertDictEqual({"mock_test": test_plugin}, test_get_dict) - @mock.patch("dataprofiler.plugins.__init__.importlib.util") - @mock.patch("dataprofiler.plugins.__init__.os.path.isdir") - @mock.patch("dataprofiler.plugins.__init__.os.listdir") + @mock.patch("dataprofiler.plugins.importlib.util") + @mock.patch("dataprofiler.plugins.os.path.isdir") + @mock.patch("dataprofiler.plugins.os.listdir") def test_load_plugin(self, mock_listdir, mock_isdir, mock_importlib_util): - mock_listdir.side_effect = ( - lambda folder_dir: ["__pycache__", "py"] + mock_listdir.side_effect = lambda folder_dir: ( + ["__pycache__", "py"] if folder_dir.endswith("plugins") else ["stillnotrealpy", "a.json", None] ) @@ -38,10 +38,8 @@ def test_load_plugin(self, mock_listdir, mock_isdir, mock_importlib_util): load_plugins() mock_importlib_util.spec_from_file_location.assert_not_called() - mock_listdir.side_effect = ( - lambda folder_dir: ["folder"] - if folder_dir.endswith("plugins") - else ["file.py"] + mock_listdir.side_effect = lambda folder_dir: ( + ["folder"] if folder_dir.endswith("plugins") else ["file.py"] ) mock_spec = mock.Mock() mock_importlib_util.spec_from_file_location.return_value = mock_spec diff --git a/dataprofiler/tests/profilers/profiler_options/test_datalabeler_options.py b/dataprofiler/tests/profilers/profiler_options/test_datalabeler_options.py index e2794c78a..2ed3b6246 100644 --- a/dataprofiler/tests/profilers/profiler_options/test_datalabeler_options.py +++ b/dataprofiler/tests/profilers/profiler_options/test_datalabeler_options.py @@ -1,4 +1,3 @@ -pass import json from unittest import mock diff --git a/dataprofiler/tests/profilers/profiler_options/test_text_options.py b/dataprofiler/tests/profilers/profiler_options/test_text_options.py index b26509e91..431b81a65 100644 --- a/dataprofiler/tests/profilers/profiler_options/test_text_options.py +++ b/dataprofiler/tests/profilers/profiler_options/test_text_options.py @@ -1,4 +1,3 @@ -pass import json from unittest import mock diff --git a/dataprofiler/tests/profilers/test_base_column_profilers.py b/dataprofiler/tests/profilers/test_base_column_profilers.py index 4ab7182cf..2fc8e8235 100644 --- a/dataprofiler/tests/profilers/test_base_column_profilers.py +++ b/dataprofiler/tests/profilers/test_base_column_profilers.py @@ -176,11 +176,13 @@ def test_cannot_instantiate(self): """showing we normally can't instantiate an abstract class""" with self.assertRaises(TypeError) as e: BaseColumnPrimitiveTypeProfiler() - self.assertEqual( - "Can't instantiate abstract class BaseColumnPrimitiveTypeProfiler " - "with abstract methods _update_helper, profile, report, update", - str(e.exception), + error_message = str(e.exception) + self.assertIn( + "Can't instantiate abstract class " "BaseColumnPrimitiveTypeProfiler", + error_message, ) + for abstract_method in ("_update_helper", "profile", "report", "update"): + self.assertIn(abstract_method, error_message) def test_combine_unqiue_sets(self): a = [1, 2, 3] diff --git a/dataprofiler/tests/profilers/test_categorical_column_profile.py b/dataprofiler/tests/profilers/test_categorical_column_profile.py index 55d2ea68e..0786d0b0f 100644 --- a/dataprofiler/tests/profilers/test_categorical_column_profile.py +++ b/dataprofiler/tests/profilers/test_categorical_column_profile.py @@ -324,7 +324,15 @@ def test_categorical_mapping(self): ) self.assertEqual(num_null_types, len(column_profile.null_types)) self.assertEqual(num_nan_count, len(column_profile.null_types_index["nan"])) - expected = {"abcd": 2, "aa": 2, "b": 1, "4": 1, "3": 1, "2": 2, "dfd": 1} + expected = { + "abcd": 2, + "aa": 2, + "b": 1, + "4": 1, + "3": 1, + "2": 2, + "dfd": 1, + } self.assertDictEqual(expected, cat_profiler._categories) num_null_types = 4 num_nan_count = 2 @@ -448,7 +456,19 @@ def test_categorical_merge(self): ["abcd", "aa", "abcd", "aa", "b", "4", "3", "2", "dfd", "2", np.nan] ) df2 = pd.Series( - ["1", "null", "ee", "NaN", "ff", "nan", "gg", "None", "aa", "b", "ee"] + [ + "1", + "null", + "ee", + "NaN", + "ff", + "nan", + "gg", + "None", + "aa", + "b", + "ee", + ] ) # Expected is based off insertion order @@ -673,10 +693,12 @@ def test_categorical_merge(self): self.assertIsNone(merge_stop_conditions_not_met._stopped_at_unique_count) self.assertIsNone(merge_stop_conditions_not_met._stopped_at_unique_ratio) self.assertEqual( - 0.99, merge_stop_conditions_not_met.stop_condition_unique_value_ratio + 0.99, + merge_stop_conditions_not_met.stop_condition_unique_value_ratio, ) self.assertEqual( - 12, merge_stop_conditions_not_met.max_sample_size_to_check_stop_condition + 12, + merge_stop_conditions_not_met.max_sample_size_to_check_stop_condition, ) def test_gini_impurity(self): @@ -731,6 +753,11 @@ def test_categorical_diff(self): }, } actual_diff = profile.diff(profile2) + self.assertAlmostEqual( + expected_diff.get("statistics").get("chi2-test").pop("p-value"), + actual_diff.get("statistics").get("chi2-test").pop("p-value"), + places=10, + ) self.assertDictEqual(expected_diff, actual_diff) # Test with one categorical column matching @@ -754,7 +781,10 @@ def test_categorical_diff(self): profile2.update(df_not_categorical) expected_diff = { "categorical": [True, False], - "statistics": {"unique_count": -10, "unique_ratio": -0.7142857142857143}, + "statistics": { + "unique_count": -10, + "unique_ratio": -0.7142857142857143, + }, } self.assertDictEqual(expected_diff, profile.diff(profile2)) @@ -984,7 +1014,12 @@ def test_json_decode_after_update(self): deserialized.update(df_categorical) assert deserialized.sample_size == 14 - assert deserialized.categorical_counts == {"c": 5, "b": 4, "a": 4, "d": 1} + assert deserialized.categorical_counts == { + "c": 5, + "b": 4, + "a": 4, + "d": 1, + } def test_cms_max_num_heavy_hitters(self): df_categorical = pd.Series(["a"] * 5 + ["b"] * 5 + ["c"] * 10) diff --git a/dataprofiler/tests/profilers/test_column_profile_compilers.py b/dataprofiler/tests/profilers/test_column_profile_compilers.py index 1e0afc124..7ae1aa079 100644 --- a/dataprofiler/tests/profilers/test_column_profile_compilers.py +++ b/dataprofiler/tests/profilers/test_column_profile_compilers.py @@ -24,11 +24,12 @@ def test_cannot_instantiate(self): """showing we normally can't instantiate an abstract class""" with self.assertRaises(TypeError) as e: col_pro_compilers.BaseCompiler() - self.assertRegex( - str(e.exception), - "Can't instantiate abstract class BaseCompiler with " - "abstract methods? report", + error_message = str(e.exception) + self.assertIn( + "Can't instantiate abstract class BaseCompiler", + error_message, ) + self.assertIn("report", error_message) @mock.patch.multiple( col_pro_compilers.BaseCompiler, @@ -64,7 +65,8 @@ def test_add_profilers(self): compiler1.name = "compiler1" compiler2.name = "compiler2" with self.assertRaisesRegex( - ValueError, "Column profile names are unmatched: " "compiler1 != compiler2" + ValueError, + "Column profile names are unmatched: " "compiler1 != compiler2", ): compiler1 + compiler2 @@ -95,7 +97,9 @@ def test_no_profilers_error(self): self.assertEqual("Must add profilers.", str(e.exception)) @mock.patch.multiple( - col_pro_compilers.BaseCompiler, __abstractmethods__=set(), _profilers="mock" + col_pro_compilers.BaseCompiler, + __abstractmethods__=set(), + _profilers="mock", ) def test_no_options_error(self): with self.assertRaisesRegex( @@ -230,7 +234,10 @@ def test_diff_primitive_compilers(self): "stddev": 3.285085839971525, "t-test": { "t-statistic": 0.4155260166386663, - "conservative": {"deg_of_free": 1.0, "p-value": 0.749287157907667}, + "conservative": { + "deg_of_free": 1.0, + "p-value": 0.749287157907667, + }, "welch": { "deg_of_free": 3.6288111187629117, "p-value": 0.7011367179395704, @@ -256,6 +263,19 @@ def test_diff_primitive_compilers(self): profile_diff["statistics"].pop("median_absolute_deviation"), places=2, ) + self.assertAlmostEqual( + expected_diff["statistics"].get("t-test").get("welch").pop("p-value"), + profile_diff["statistics"].get("t-test").get("welch").pop("p-value"), + places=10, + ) + self.assertAlmostEqual( + expected_diff["statistics"] + .get("t-test") + .get("conservative") + .pop("p-value"), + profile_diff["statistics"].get("t-test").get("conservative").pop("p-value"), + places=10, + ) self.assertDictEqual(expected_diff, profile_diff) # Test different compilers @@ -354,6 +374,22 @@ def test_disabling_columns_during_primitive_diff(self): profile_diff["statistics"].pop("median_absolute_deviation"), places=2, ) + self.assertAlmostEqual( + expected_diff.get("statistics").get("t-test").get("welch").pop("p-value"), + profile_diff.get("statistics").get("t-test").get("welch").pop("p-value"), + places=10, + ) + self.assertAlmostEqual( + expected_diff.get("statistics") + .get("t-test") + .get("conservative") + .pop("p-value"), + profile_diff.get("statistics") + .get("t-test") + .get("conservative") + .pop("p-value"), + places=10, + ) self.assertDictEqual(expected_diff, profile_diff) # Test disabling all columns in one compiler @@ -576,7 +612,10 @@ def test_json_encode_after_update(self): "data": { "name": "test", "_profiles": { - "order": {"class": "OrderColumn", "data": {"an": "order"}}, + "order": { + "class": "OrderColumn", + "data": {"an": "order"}, + }, "category": { "class": "CategoricalColumn", "data": {"this": "category"}, @@ -713,7 +752,11 @@ def test_compiler_data_labeler_diff(self, *mocks): expected_diff = { "statistics": { "avg_predictions": {"a": "unchanged", "b": -0.7, "c": 0.7}, - "label_representation": {"a": -0.84, "b": "unchanged", "c": 0.84}, + "label_representation": { + "a": -0.84, + "b": "unchanged", + "c": 0.84, + }, }, "data_label": [["a"], [], ["b"]], } @@ -762,7 +805,9 @@ def test_json_encode_after_update(self, mock_instance, *mocks): compiler = col_pro_compilers.ColumnDataLabelerCompiler(data) with mock.patch.object( - compiler._profiles["data_labeler"], "__dict__", {"data_label": "INTEGER"} + compiler._profiles["data_labeler"], + "__dict__", + {"data_label": "INTEGER"}, ): serialized = json.dumps(compiler, cls=ProfileEncoder) @@ -1016,7 +1061,20 @@ def test_compiler_stats_diff(self, *mocks): "statistics": { "vocab": [ ["H", "l"], - ["e", "o", " ", "T", "h", "i", "s", "a", "t", "g", "r", "n"], + [ + "e", + "o", + " ", + "T", + "h", + "i", + "s", + "a", + "t", + "g", + "r", + "n", + ], ["u", "k", "w", "m", "y", "9"], ], "vocab_count": [ @@ -1037,7 +1095,11 @@ def test_compiler_stats_diff(self, *mocks): }, {"m": 2, "9": 2, "u": 1, "k": 1, "w": 1, "y": 1}, ], - "words": [["Hello", "test"], ["grant"], ["unknown", "name", "9"]], + "words": [ + ["Hello", "test"], + ["grant"], + ["unknown", "name", "9"], + ], "word_count": [ {"Hello": 2, "test": 1}, {"grant": "unchanged"}, @@ -1097,7 +1159,20 @@ def test_compiler_stats_diff(self, *mocks): "statistics": { "vocab": [ ["H", "l"], - ["e", "o", " ", "T", "h", "i", "s", "a", "t", "g", "r", "n"], + [ + "e", + "o", + " ", + "T", + "h", + "i", + "s", + "a", + "t", + "g", + "r", + "n", + ], ["u", "k", "w", "m", "y", "9"], ], "vocab_count": [ @@ -1118,7 +1193,11 @@ def test_compiler_stats_diff(self, *mocks): }, {"m": 2, "9": 2, "u": 1, "k": 1, "w": 1, "y": 1}, ], - "words": [["Hello", "test"], ["grant"], ["unknown", "name", "9"]], + "words": [ + ["Hello", "test"], + ["grant"], + ["unknown", "name", "9"], + ], "word_count": [ {"Hello": 2, "test": 1}, {"grant": "unchanged"}, diff --git a/dataprofiler/tests/profilers/test_float_column_profile.py b/dataprofiler/tests/profilers/test_float_column_profile.py index d79fdd641..19fe4c8a6 100644 --- a/dataprofiler/tests/profilers/test_float_column_profile.py +++ b/dataprofiler/tests/profilers/test_float_column_profile.py @@ -626,7 +626,8 @@ def test_null_values_for_histogram(self): } self.assertEqual( - expected_histogram["bin_counts"].tolist(), histogram["bin_counts"].tolist() + expected_histogram["bin_counts"].tolist(), + histogram["bin_counts"].tolist(), ) self.assertCountEqual(expected_histogram["bin_edges"], histogram["bin_edges"]) @@ -759,7 +760,8 @@ def test_profile_histogram_w_updates(self): self.assertIsNotNone(merged_profiler.histogram_selection) histogram = profile["histogram"] self.assertEqual( - expected_histogram["bin_counts"].tolist(), histogram["bin_counts"].tolist() + expected_histogram["bin_counts"].tolist(), + histogram["bin_counts"].tolist(), ) self.assertCountEqual( np.round(expected_histogram["bin_edges"], 12), @@ -833,7 +835,14 @@ def test_total_histogram_bin_variance(self): def test_histogram_loss(self): # run time is small - diff_var, avg_diffvar, total_var, avg_totalvar, run_time, avg_runtime = ( + ( + diff_var, + avg_diffvar, + total_var, + avg_totalvar, + run_time, + avg_runtime, + ) = ( 0.3, 0.2, 0.1, @@ -843,12 +852,24 @@ def test_histogram_loss(self): ) expected_loss = 0.1 / 0.2 + 0.05 / 0.05 est_loss = FloatColumn._histogram_loss( - diff_var, avg_diffvar, total_var, avg_totalvar, run_time, avg_runtime + diff_var, + avg_diffvar, + total_var, + avg_totalvar, + run_time, + avg_runtime, ) self.assertEqual(expected_loss, est_loss) # run time is big - diff_var, avg_diffvar, total_var, avg_totalvar, run_time, avg_runtime = ( + ( + diff_var, + avg_diffvar, + total_var, + avg_totalvar, + run_time, + avg_runtime, + ) = ( 0.3, 0.2, 0.1, @@ -858,7 +879,12 @@ def test_histogram_loss(self): ) expected_loss = 0.1 / 0.2 + 0.05 / 0.05 + 8 / 14 est_loss = FloatColumn._histogram_loss( - diff_var, avg_diffvar, total_var, avg_totalvar, run_time, avg_runtime + diff_var, + avg_diffvar, + total_var, + avg_totalvar, + run_time, + avg_runtime, ) self.assertEqual(expected_loss, est_loss) @@ -866,7 +892,15 @@ def test_select_method_for_histogram(self): data = pd.Series([], dtype=object) profiler = FloatColumn(data.name) profiler.update(data) - list_method = ["auto", "fd", "doane", "scott", "rice", "sturges", "sqrt"] + list_method = [ + "auto", + "fd", + "doane", + "scott", + "rice", + "sturges", + "sqrt", + ] current_exact_var = 0 # sqrt has the least current loss current_est_var = np.array([0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.005]) @@ -877,7 +911,10 @@ def test_select_method_for_histogram(self): for i, method in enumerate(list_method): profiler.histogram_methods[method]["total_loss"] = list_total_loss[i] selected_method = profiler._select_method_for_histogram( - current_exact_var, current_est_var, current_total_var, current_run_time + current_exact_var, + current_est_var, + current_total_var, + current_run_time, ) self.assertEqual(selected_method, "sqrt") @@ -894,7 +931,10 @@ def test_select_method_for_histogram(self): for i, method in enumerate(list_method): profiler.histogram_methods[method]["total_loss"] = list_total_loss[i] selected_method = profiler._select_method_for_histogram( - current_exact_var, current_est_var, current_total_var, current_run_time + current_exact_var, + current_est_var, + current_total_var, + current_run_time, ) self.assertEqual(selected_method, "sturges") @@ -921,7 +961,12 @@ def test_merge_histogram(self): profiler._merge_histogram(input_array) merged_hist = profiler._histogram_for_profile("sqrt")[0] - expected_bin_counts, expected_bin_edges = [5, 2, 2], [0.5, 2.0, 3.5, 5.0] + expected_bin_counts, expected_bin_edges = [5, 2, 2], [ + 0.5, + 2.0, + 3.5, + 5.0, + ] self.assertEqual(expected_bin_counts, merged_hist["bin_counts"].tolist()) self.assertCountEqual(expected_bin_edges, merged_hist["bin_edges"]) @@ -1326,7 +1371,8 @@ def test_profile_merge(self): self.assertEqual(profiler3.min, expected_profile.pop("min")) self.assertEqual(profiler3.max, expected_profile.pop("max")) self.assertEqual( - histogram["bin_counts"].tolist(), expected_histogram["bin_counts"].tolist() + histogram["bin_counts"].tolist(), + expected_histogram["bin_counts"].tolist(), ) self.assertCountEqual(histogram["bin_edges"], expected_histogram["bin_edges"]) @@ -1539,7 +1585,10 @@ def test_histogram_option_integration(self): self.assertIsNone(num_profiler.histogram_selection) self.assertEqual(["sturges"], num_profiler.histogram_bin_method_names) - options.histogram_and_quantiles.bin_count_or_method = ["sturges", "doane"] + options.histogram_and_quantiles.bin_count_or_method = [ + "sturges", + "doane", + ] num_profiler = FloatColumn(name="test2", options=options) self.assertIsNone(num_profiler.histogram_selection) self.assertEqual(["sturges", "doane"], num_profiler.histogram_bin_method_names) @@ -1553,7 +1602,8 @@ def test_histogram_option_integration(self): # case when just 1 unique value, should just set bin size to be 1 num_profiler.update(pd.Series(["1", "1"])) self.assertEqual( - 1, len(num_profiler.histogram_methods["custom"]["histogram"]["bin_counts"]) + 1, + len(num_profiler.histogram_methods["custom"]["histogram"]["bin_counts"]), ) # case when more than 1 unique value, by virtue of a streaming update @@ -1703,7 +1753,10 @@ def test_diff(self): }, "t-test": { "t-statistic": 0.5393164101529813, - "conservative": {"deg_of_free": 2.0, "p-value": 0.643676756587475}, + "conservative": { + "deg_of_free": 2.0, + "p-value": 0.643676756587475, + }, "welch": { "deg_of_free": 4.999127432888682, "p-value": 0.6128117908944144, @@ -1728,6 +1781,11 @@ def test_diff(self): np.testing.assert_almost_equal( sorted(expected_diff_mode[i]), sorted(diff_mode[i]), 2 ) + self.assertAlmostEqual( + expected_diff.get("t-test").get("welch").pop("p-value"), + profile_diff.get("t-test").get("welch").pop("p-value"), + places=10, + ) self.assertAlmostEqual( expected_diff.pop("median_absolute_deviation"), profile_diff.pop("median_absolute_deviation"), diff --git a/dataprofiler/tests/profilers/test_int_column_profile.py b/dataprofiler/tests/profilers/test_int_column_profile.py index 961b33c8c..960e5318c 100644 --- a/dataprofiler/tests/profilers/test_int_column_profile.py +++ b/dataprofiler/tests/profilers/test_int_column_profile.py @@ -1087,6 +1087,11 @@ def test_diff(self): np.testing.assert_almost_equal( sorted(expected_diff_mode[i]), sorted(diff_mode[i]), 2 ) + self.assertAlmostEqual( + expected_diff.get("t-test").get("welch").pop("p-value"), + profile_diff.get("t-test").get("welch").pop("p-value"), + places=10, + ) self.assertAlmostEqual( expected_diff.pop("median_absolute_deviation"), profile_diff.pop("median_absolute_deviation"), diff --git a/dataprofiler/tests/profilers/test_numeric_stats_mixin_profile.py b/dataprofiler/tests/profilers/test_numeric_stats_mixin_profile.py index e112781ab..5dd262445 100644 --- a/dataprofiler/tests/profilers/test_numeric_stats_mixin_profile.py +++ b/dataprofiler/tests/profilers/test_numeric_stats_mixin_profile.py @@ -759,7 +759,7 @@ def test_profile(self): # Validate that the times dictionary is empty self.assertEqual(defaultdict(float), num_profiler.times) - profile = num_profiler.profile() + profile = num_profiler.profile # pop out the histogram and quartiles to test separately from the # rest of the dict as we need comparison with some precision histogram = profile.pop("histogram") diff --git a/dataprofiler/tests/profilers/test_profile_builder.py b/dataprofiler/tests/profilers/test_profile_builder.py index c4e604737..0c48051b4 100644 --- a/dataprofiler/tests/profilers/test_profile_builder.py +++ b/dataprofiler/tests/profilers/test_profile_builder.py @@ -2156,9 +2156,13 @@ def test_diff_categorical_chi2_test(self, *mocks): "deg_of_free": 2, "p-value": 0.3099238764710244, } - self.assertDictEqual( - expected_chi2_test_dict, diff["data_stats"][0]["statistics"]["chi2-test"] + chi2_diff = diff["data_stats"][0]["statistics"]["chi2-test"] + self.assertAlmostEqual( + expected_chi2_test_dict.pop("p-value"), + chi2_diff.pop("p-value"), + places=10, ) + self.assertDictEqual(expected_chi2_test_dict, chi2_diff) @mock.patch( "dataprofiler.profilers.data_labeler_column_profile.DataLabelerColumn.update" diff --git a/dataprofiler/tests/profilers/test_profiler_utils.py b/dataprofiler/tests/profilers/test_profiler_utils.py index 4eee1963a..0ea0c0fc0 100644 --- a/dataprofiler/tests/profilers/test_profiler_utils.py +++ b/dataprofiler/tests/profilers/test_profiler_utils.py @@ -472,7 +472,6 @@ def test_odd_merge_profile_list(self, mock_data_labeler, *mocks): class TestAutoMultiProcessToggle(unittest.TestCase): - """ Validate profile_utils.auto_multiprocess_toggle is properly working. """ diff --git a/dataprofiler/tests/profilers/test_text_column_profile.py b/dataprofiler/tests/profilers/test_text_column_profile.py index 12fb1d27b..a51f808f1 100644 --- a/dataprofiler/tests/profilers/test_text_column_profile.py +++ b/dataprofiler/tests/profilers/test_text_column_profile.py @@ -41,7 +41,20 @@ def test_profiled_vocab(self): ] ).apply(str) df2 = pd.Series( - ["1", "1", "ee", "ff", "ff", "gg", "gg", "abcd", "aa", "b", "ee", "b"] + [ + "1", + "1", + "ee", + "ff", + "ff", + "gg", + "gg", + "abcd", + "aa", + "b", + "ee", + "b", + ] ).apply(str) df3 = pd.Series( [ @@ -112,7 +125,20 @@ def batch_variance(mean_a, var_a, count_a, mean_b, var_b, count_b): ] ).apply(str) df2 = pd.Series( - ["1", "1", "ee", "ff", "ff", "gg", "gg", "abcd", "aa", "b", "ee", "b"] + [ + "1", + "1", + "ee", + "ff", + "ff", + "gg", + "gg", + "abcd", + "aa", + "b", + "ee", + "b", + ] ).apply(str) df3 = pd.Series( [ @@ -451,7 +477,18 @@ def test_profile_merge_with_different_options(self): options.histogram_and_quantiles.bin_count_or_method = None df = pd.Series( - ["pancake", "banana", "lighthouse", "aa", "b", "4", "3", "2", "dfd", "2"] + [ + "pancake", + "banana", + "lighthouse", + "aa", + "b", + "4", + "3", + "2", + "dfd", + "2", + ] ) profiler1 = TextColumn("Text", options=options) @@ -489,7 +526,8 @@ def test_profile_merge_with_different_options(self): def test_text_column_with_wrong_options(self): with self.assertRaisesRegex( - ValueError, "TextColumn parameter 'options' must be of" " type TextOptions." + ValueError, + "TextColumn parameter 'options' must be of" " type TextOptions.", ): profiler = TextColumn("Text", options="wrong_data_type") @@ -531,7 +569,10 @@ def test_histogram_option_integration(self): self.assertIsNone(num_profiler.histogram_selection) self.assertEqual(["sturges"], num_profiler.histogram_bin_method_names) - options.histogram_and_quantiles.bin_count_or_method = ["sturges", "doane"] + options.histogram_and_quantiles.bin_count_or_method = [ + "sturges", + "doane", + ] num_profiler = TextColumn(name="test2", options=options) self.assertIsNone(num_profiler.histogram_selection) self.assertEqual(["sturges", "doane"], num_profiler.histogram_bin_method_names) @@ -545,7 +586,8 @@ def test_histogram_option_integration(self): # case when just 1 unique value, should just set bin size to be 1 num_profiler.update(pd.Series(["1", "1"])) self.assertEqual( - 1, len(num_profiler.histogram_methods["custom"]["histogram"]["bin_counts"]) + 1, + len(num_profiler.histogram_methods["custom"]["histogram"]["bin_counts"]), ) # case when more than 1 unique value, by virtue of a streaming update @@ -589,7 +631,10 @@ def test_diff(self): ), "t-test": { "t-statistic": -1.9339958714826413, - "conservative": {"deg_of_free": 8.0, "p-value": 0.08916903961929257}, + "conservative": { + "deg_of_free": 8.0, + "p-value": 0.08916903961929257, + }, "welch": { "deg_of_free": 15.761400272034564, "p-value": 0.07127621949432528, @@ -607,6 +652,11 @@ def test_diff(self): np.testing.assert_almost_equal( sorted(expected_diff_mode[i]), sorted(diff_mode[i]), 2 ) + self.assertAlmostEqual( + expected_diff.get("t-test").get("welch").pop("p-value"), + profile_diff.get("t-test").get("welch").pop("p-value"), + places=10, + ) self.assertAlmostEqual( expected_diff.pop("median_absolute_deviation"), profile_diff.pop("median_absolute_deviation"), @@ -646,96 +696,125 @@ def test_json_encode_after_update(self, time): # popping vocab and comparing as set below since order is random serialized_vocab = serialized_dict["data"].pop("vocab") - serialized = json.dumps(serialized_dict) - - expected = json.dumps( - { - "class": "TextColumn", - "data": { - "min": 1.0, - "max": 12.0, - "_top_k_modes": 5, - "sum": 38.0, - "_biased_variance": 9.33884297520661, - "_biased_skewness": 1.8025833203700588, - "_biased_kurtosis": 2.7208317017777395, - "_median_is_enabled": True, - "_median_abs_dev_is_enabled": True, - "max_histogram_bin": 100000, - "min_histogram_bin": 1000, - "histogram_bin_method_names": ["custom"], - "histogram_selection": None, - "user_set_histogram_bin": 5, - "bias_correction": True, - "_mode_is_enabled": True, - "num_zeros": 0, - "num_negatives": 0, - "_num_quantiles": 1000, - "histogram_methods": { - "custom": { - "total_loss": 0.0, - "current_loss": 0.0, - "suggested_bin_count": 5, - "histogram": {"bin_counts": None, "bin_edges": None}, - } - }, - "_stored_histogram": { - "total_loss": 7.63, - "current_loss": 7.63, - "suggested_bin_count": 1000, + expected_dict = { + "class": "TextColumn", + "data": { + "min": 1.0, + "max": 12.0, + "_top_k_modes": 5, + "sum": 38.0, + "_biased_variance": 9.33884297520661, + "_biased_skewness": 1.8025833203700588, + "_biased_kurtosis": 2.7208317017777395, + "_median_is_enabled": True, + "_median_abs_dev_is_enabled": True, + "max_histogram_bin": 100000, + "min_histogram_bin": 1000, + "histogram_bin_method_names": ["custom"], + "histogram_selection": None, + "user_set_histogram_bin": 5, + "bias_correction": True, + "_mode_is_enabled": True, + "num_zeros": 0, + "num_negatives": 0, + "_num_quantiles": 1000, + "histogram_methods": { + "custom": { + "total_loss": 0.0, + "current_loss": 0.0, + "suggested_bin_count": 5, "histogram": { - "bin_counts": [6, 4, 0, 0, 1], - "bin_edges": [1.0, 3.2, 5.4, 7.6000000000000005, 9.8, 12.0], + "bin_counts": None, + "bin_edges": None, }, + } + }, + "_stored_histogram": { + "total_loss": 7.63, + "current_loss": 7.63, + "suggested_bin_count": 1000, + "histogram": { + "bin_counts": [6, 4, 0, 0, 1], + "bin_edges": [ + 1.0, + 3.2, + 5.4, + 7.6000000000000005, + 9.8, + 12.0, + ], }, - "_batch_history": [ - { - "match_count": 11, - "sample_size": 11, - "min": 1.0, - "max": 12.0, - "sum": 38.0, - "biased_variance": 9.33884297520661, - "mean": 3.4545454545454546, - "biased_skewness": 1.8025833203700588, - "biased_kurtosis": 2.7208317017777395, - } - ], - "_NumericStatsMixin__calculations": { - "min": "_get_min", - "max": "_get_max", - "sum": "_get_sum", - "variance": "_get_variance", - "skewness": "_get_skewness", - "kurtosis": "_get_kurtosis", - "histogram_and_quantiles": "_get_histogram_and_quantiles", - }, - "name": None, - "col_index": np.nan, - "sample_size": 11, - "metadata": {}, - "times": { - "vocab": 1.0, + }, + "_batch_history": [ + { + "match_count": 11, + "sample_size": 11, "min": 1.0, - "max": 1.0, - "sum": 1.0, - "variance": 1.0, - "skewness": 1.0, - "kurtosis": 1.0, - "histogram_and_quantiles": 1.0, - }, - "thread_safe": True, - "match_count": 11, - "_TextColumn__calculations": {"vocab": "_update_vocab"}, - "type": "string", + "max": 12.0, + "sum": 38.0, + "biased_variance": 9.33884297520661, + "mean": 3.4545454545454546, + "biased_skewness": 1.8025833203700588, + "biased_kurtosis": 2.7208317017777395, + } + ], + "_NumericStatsMixin__calculations": { + "min": "_get_min", + "max": "_get_max", + "sum": "_get_sum", + "variance": "_get_variance", + "skewness": "_get_skewness", + "kurtosis": "_get_kurtosis", + "histogram_and_quantiles": "_get_histogram_and_quantiles", }, - } - ) + "name": None, + "col_index": np.nan, + "sample_size": 11, + "metadata": {}, + "times": { + "vocab": 1.0, + "min": 1.0, + "max": 1.0, + "sum": 1.0, + "variance": 1.0, + "skewness": 1.0, + "kurtosis": 1.0, + "histogram_and_quantiles": 1.0, + }, + "thread_safe": True, + "match_count": 11, + "_TextColumn__calculations": {"vocab": "_update_vocab"}, + "type": "string", + }, + } + + for field in ("_biased_variance", "_biased_skewness", "_biased_kurtosis"): + self.assertAlmostEqual( + serialized_dict["data"][field], expected_dict["data"][field] + ) + serialized_dict["data"][field] = expected_dict["data"][field] + + for field in ( + "min", + "max", + "sum", + "biased_variance", + "mean", + "biased_skewness", + "biased_kurtosis", + ): + self.assertAlmostEqual( + serialized_dict["data"]["_batch_history"][0][field], + expected_dict["data"]["_batch_history"][0][field], + ) + serialized_dict["data"]["_batch_history"][0][field] = expected_dict["data"][ + "_batch_history" + ][0][field] expected_vocab = profiler.vocab expected_quantiles = profiler.quantiles - self.assertEqual(serialized, expected) + self.assertEqual(json.dumps(serialized_dict), json.dumps(expected_dict)) self.assertSetEqual(set(serialized_vocab), set(expected_vocab)) self.assertListEqual(serialized_quantiles, expected_quantiles) diff --git a/dataprofiler/tests/space_time_analysis/structured_space_time_analysis.py b/dataprofiler/tests/space_time_analysis/structured_space_time_analysis.py index df57854f6..5af7bc2aa 100644 --- a/dataprofiler/tests/space_time_analysis/structured_space_time_analysis.py +++ b/dataprofiler/tests/space_time_analysis/structured_space_time_analysis.py @@ -1,4 +1,5 @@ """Contains space and time analysis tests for the Dataprofiler""" + import json import os import random diff --git a/dataprofiler/tests/test_data_profiler.py b/dataprofiler/tests/test_data_profiler.py index ef7664cea..5dd12e69d 100644 --- a/dataprofiler/tests/test_data_profiler.py +++ b/dataprofiler/tests/test_data_profiler.py @@ -29,8 +29,6 @@ def setUpClass(cls): def test_set_seed(self): import dataprofiler as dp - self.assertEqual(dp.settings._seed, None) - dp.set_seed(5) self.assertEqual(dp.settings._seed, 5) @@ -56,46 +54,6 @@ def test_data_profiling(self): self.assertIsNotNone(profile.profile) self.assertIsNotNone(profile.report()) - def test_no_snappy(self): - import importlib - import sys - import types - - orig_import = __import__ - # necessary for any wrapper around the library to test if snappy caught - # as an issue - - def reload_data_profiler(): - """Recursively reload modules.""" - sys_modules = sys.modules.copy() - for module_name, module in sys_modules.items(): - # Only reload top level of the dataprofiler - if "dataprofiler" in module_name and len(module_name.split(".")) < 3: - if isinstance(module, types.ModuleType): - importlib.reload(module) - - def import_mock(name, *args, **kwargs): - if name == "snappy": - raise ImportError("test") - return orig_import(name, *args, **kwargs) - - with mock.patch("builtins.__import__", side_effect=import_mock): - with self.assertWarns(ImportWarning) as w: - import dataprofiler - - reload_data_profiler() - - self.assertEqual( - str(w.warning), - "Snappy must be installed to use parquet/avro datasets." - "\n\n" - "For macOS use Homebrew:\n" - "\t`brew install snappy`" - "\n\n" - "For linux use apt-get:\n`" - "\tsudo apt-get -y install libsnappy-dev`\n", - ) - def test_no_tensorflow(self): import sys diff --git a/dataprofiler/tests/test_dp_logging.py b/dataprofiler/tests/test_dp_logging.py index 7f78903ee..99496e314 100644 --- a/dataprofiler/tests/test_dp_logging.py +++ b/dataprofiler/tests/test_dp_logging.py @@ -22,12 +22,6 @@ def tearDownClass(cls): root_logger.removeHandler(dp_logging.get_logger()) dp_logging._dp_logger = None - def test_default_verbosity(self, mock_stdout): - # Ensure that default effective level is INFO - self.assertEqual( - logging.INFO, logging.getLogger("DataProfiler").getEffectiveLevel() - ) - def test_set_verbosity(self, mock_stdout): from dataprofiler import dp_logging diff --git a/dataprofiler/tests/test_rng_utils.py b/dataprofiler/tests/test_rng_utils.py index 6ee2ed35c..8a4c4d229 100644 --- a/dataprofiler/tests/test_rng_utils.py +++ b/dataprofiler/tests/test_rng_utils.py @@ -1,4 +1,5 @@ """Validates that generator intakes DATAPROFILER_SEED properly.""" + import os import unittest import unittest.mock diff --git a/dataprofiler/version.py b/dataprofiler/version.py deleted file mode 100644 index 1136efae1..000000000 --- a/dataprofiler/version.py +++ /dev/null @@ -1,13 +0,0 @@ -"""File contains the version number for the package.""" - -MAJOR = 0 -MINOR = 11 -MICRO = 0 -POST = None # otherwise None - -VERSION = "%d.%d.%d" % (MAJOR, MINOR, MICRO) - -_post_str = "" -if POST: - _post_str = f".post{POST}" -__version__ = VERSION + _post_str diff --git a/requirements-dev.txt b/requirements-dev.txt index cff8f51a0..163dae508 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,9 @@ -check-manifest>=0.48 +check-manifest>=0.50 black>=24.3.0 isort==5.12.0 -pre-commit==2.19.0 +pre-commit==4.3.0 tox==3.25.1 +tox-conda==0.10.2 types-setuptools==67.7.0.1 types-python-dateutil==2.8.19.12 types-requests==2.30.0.0 diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 000000000..36517b45f --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,7 @@ +Sphinx>=5.0.0 +sphinx-rtd-theme +nbsphinx +furo +nbsphinx-link +pre-commit +tornado diff --git a/requirements-ml.txt b/requirements-ml.txt index ff525fec1..c8b373f22 100644 --- a/requirements-ml.txt +++ b/requirements-ml.txt @@ -1,7 +1,5 @@ scikit-learn>=0.23.2 -keras>=2.4.3,<3.0.0 +keras>3.4.0,<4.0.0 rapidfuzz>=2.6.1 -tensorflow>=2.6.4,<2.15.0; sys.platform != 'darwin' -tensorflow>=2.6.4,<2.15.0; sys_platform == 'darwin' and platform_machine != 'arm64' -tensorflow-macos>=2.6.4,<2.15.0; sys_platform == 'darwin' and platform_machine == 'arm64' +tensorflow>=2.16.0 tqdm>=4.0.0 diff --git a/requirements-test.txt b/requirements-test.txt index 6c981cf9c..cf127b607 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,9 +1,9 @@ coverage>=5.0.1 -dask>=2.29.0,<2024.2.0 +dask[dask-expr,dataframe]>=2024.4.1 fsspec>=0.3.3 pytest>=6.0.1 pytest-cov>=2.8.1 pytest-xdist>=2.1.0 pytest-forked>=1.3.0 toolz>=0.10.0 -memray>=1.7.0,<1.12.0 +memray>=1.18.0 diff --git a/requirements.txt b/requirements.txt index a45dc34ae..241f3b725 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,17 @@ h5py>=2.10.0 wheel>=0.33.1 -numpy>=1.22.0 +numpy<2.0.0 pandas>=1.1.2 python-dateutil>=2.7.5 pytz>=2020.1 pyarrow>=1.0.1 chardet>=3.0.4 -fastavro>=1.0.0.post1 -python-snappy>=0.5.4 +fastavro>=1.1.0 +python-snappy>=0.7.1 charset-normalizer>=1.3.6 psutil>=4.0.0 scipy>=1.10.0 -requests>=2.28.1 +requests>=2.32.0 networkx>=2.5.1 typing-extensions>=3.10.0.2 HLL>=2.0.3 diff --git a/resources/__init__.py b/resources/__init__.py deleted file mode 100644 index dd86bffe7..000000000 --- a/resources/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Contains resources for labelers.""" -# empty init diff --git a/resources/labelers/structured_model/keras_metadata.pb b/resources/labelers/structured_model/keras_metadata.pb deleted file mode 100644 index dcc84a213..000000000 --- a/resources/labelers/structured_model/keras_metadata.pb +++ /dev/null @@ -1,29 +0,0 @@ - -ã`root"_tf_keras_network*Á`{"name": "functional_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "class_name": "Functional", "config": {"name": "functional_1", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "dtype": "string", "sparse": false, "ragged": false, "name": "input_1"}, "name": "input_1", "inbound_nodes": []}, {"class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAgAAAAQAAAATAAAAcxIAAAB0AGoBfACIAIgBgwN9AXwBUwApAU4pAtoWQ2hhcmFj\ndGVyTGV2ZWxDbm5Nb2RlbNoUX2NoYXJfZW5jb2RpbmdfbGF5ZXIpAtoJaW5wdXRfc3RyWg5jaGFy\nX2luX3ZlY3RvcikC2hRtYXhfY2hhcl9lbmNvZGluZ19pZNoKbWF4X2xlbmd0aKkA+lMvaG9tZS91\nYnVudHUvbmV3LWRwL0RhdGFQcm9maWxlci9kYXRhcHJvZmlsZXIvbGFiZWxlcnMvY2hhcmFjdGVy\nX2xldmVsX2Nubl9tb2RlbC5wedoRZW5jb2RpbmdfZnVuY3Rpb25TAgAAcwYAAAAAAQQBCgE=\n", null, {"class_name": "__tuple__", "items": [127, 3400]}]}, "function_type": "lambda", "module": "dataprofiler.labelers.character_level_cnn_model", "output_shape": {"class_name": "__tuple__", "items": [3400]}, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "name": "lambda", "inbound_nodes": [[["input_1", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding", "trainable": true, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 3400]}, "dtype": "float32", "input_dim": 129, "output_dim": 64, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": 3400}, "name": "embedding", "inbound_nodes": [[["lambda", 0, 0, {}]]]}, {"class_name": "Conv1D", "config": {"name": "conv1d", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "conv1d", "inbound_nodes": [[["embedding", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout", "inbound_nodes": [[["conv1d", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization", "inbound_nodes": [[["dropout", 0, 0, {}]]]}, {"class_name": "Conv1D", "config": {"name": "conv1d_1", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "conv1d_1", "inbound_nodes": [[["batch_normalization", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_1", "inbound_nodes": [[["conv1d_1", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_1", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_1", "inbound_nodes": [[["dropout_1", 0, 0, {}]]]}, {"class_name": "Conv1D", "config": {"name": "conv1d_2", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "conv1d_2", "inbound_nodes": [[["batch_normalization_1", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_2", "inbound_nodes": [[["conv1d_2", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_2", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_2", "inbound_nodes": [[["dropout_2", 0, 0, {}]]]}, {"class_name": "Conv1D", "config": {"name": "conv1d_3", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "conv1d_3", "inbound_nodes": [[["batch_normalization_2", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_3", "inbound_nodes": [[["conv1d_3", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_3", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_3", "inbound_nodes": [[["dropout_3", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 96, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense", "inbound_nodes": [[["batch_normalization_3", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_4", "inbound_nodes": [[["dense", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 96, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_1", "inbound_nodes": [[["dropout_4", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_5", "inbound_nodes": [[["dense_1", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 24, "activation": "softmax", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_2", "inbound_nodes": [[["dropout_5", 0, 0, {}]]]}, {"class_name": "TensorFlowOpLayer", "config": {"name": "ArgMax", "trainable": true, "dtype": "float32", "node_def": {"name": "ArgMax", "op": "ArgMax", "input": ["dense_2/truediv", "ArgMax/dimension"], "attr": {"Tidx": {"type": "DT_INT32"}, "output_type": {"type": "DT_INT64"}, "T": {"type": "DT_FLOAT"}}}, "constants": {"1": -1}}, "name": "tf_op_layer_ArgMax", "inbound_nodes": [[["dense_2", 0, 0, {}]]]}, {"class_name": "ThreshArgMaxLayer", "config": {"layer was saved without config": true}, "name": "thresh_arg_max_layer", "inbound_nodes": [[["tf_op_layer_ArgMax", 0, 0, {"confidence_layer": ["dense_2", 0, 0]}]]]}], "input_layers": [["input_1", 0, 0]], "output_layers": [["dense_2", 0, 0], ["tf_op_layer_ArgMax", 0, 0], ["thresh_arg_max_layer", 0, 0]]}, "shared_object_id": 52, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, null]}, "ndim": 2, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, null]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, null]}, "string", "input_1"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, null]}, "string", "input_1"]}, "keras_version": "2.6.0", "backend": "tensorflow", "model_config": {"class_name": "Functional"}, "training_config": {"loss": {"dense_2": "categorical_crossentropy"}, "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "acc", "dtype": "float32", "fn": "categorical_accuracy"}, "shared_object_id": 54}, {"class_name": "Custom>F1Score", "config": {"name": "dense_2_f1_score", "dtype": "float32", "num_classes": 24, "average": "micro", "threshold": null}, "shared_object_id": 55}], [null], [null]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Adam", "config": {"name": "Adam", "learning_rate": 0.0010000000474974513, "decay": 0.0, "beta_1": 0.8999999761581421, "beta_2": 0.9990000128746033, "epsilon": 1e-07, "amsgrad": false}}}}2 -ú root.layer-0"_tf_keras_input_layer*Ê{"class_name": "InputLayer", "name": "input_1", "dtype": "string", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "dtype": "string", "sparse": false, "ragged": false, "name": "input_1"}}2 -Å root.layer-1"_tf_keras_layer*›{"name": "lambda", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAgAAAAQAAAATAAAAcxIAAAB0AGoBfACIAIgBgwN9AXwBUwApAU4pAtoWQ2hhcmFj\ndGVyTGV2ZWxDbm5Nb2RlbNoUX2NoYXJfZW5jb2RpbmdfbGF5ZXIpAtoJaW5wdXRfc3RyWg5jaGFy\nX2luX3ZlY3RvcikC2hRtYXhfY2hhcl9lbmNvZGluZ19pZNoKbWF4X2xlbmd0aKkA+lMvaG9tZS91\nYnVudHUvbmV3LWRwL0RhdGFQcm9maWxlci9kYXRhcHJvZmlsZXIvbGFiZWxlcnMvY2hhcmFjdGVy\nX2xldmVsX2Nubl9tb2RlbC5wedoRZW5jb2RpbmdfZnVuY3Rpb25TAgAAcwYAAAAAAQQBCgE=\n", null, {"class_name": "__tuple__", "items": [127, 3400]}]}, "function_type": "lambda", "module": "dataprofiler.labelers.character_level_cnn_model", "output_shape": {"class_name": "__tuple__", "items": [3400]}, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "inbound_nodes": [[["input_1", 0, 0, {}]]], "shared_object_id": 1}2 -‚root.layer_with_weights-0"_tf_keras_layer*Ë{"name": "embedding", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 3400]}, "stateful": false, "must_restore_from_config": false, "class_name": "Embedding", "config": {"name": "embedding", "trainable": true, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 3400]}, "dtype": "float32", "input_dim": 129, "output_dim": 64, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}, "shared_object_id": 2}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": 3400}, "inbound_nodes": [[["lambda", 0, 0, {}]]], "shared_object_id": 3, "build_input_shape": {"class_name": "TensorShape", "items": [null, null]}}2 -ç root.layer_with_weights-1"_tf_keras_layer*° {"name": "conv1d", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Conv1D", "config": {"name": "conv1d", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 4}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 5}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["embedding", 0, 0, {}]]], "shared_object_id": 6, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 3, "axes": {"-1": 64}}, "shared_object_id": 56}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 64]}}2 -¦ root.layer-4"_tf_keras_layer*ü{"name": "dropout", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["conv1d", 0, 0, {}]]], "shared_object_id": 7}2 -· root.layer_with_weights-2"_tf_keras_layer*€ {"name": "batch_normalization", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "BatchNormalization", "config": {"name": "batch_normalization", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 8}, "gamma_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 9}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 10}, "moving_variance_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 11}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "inbound_nodes": [[["dropout", 0, 0, {}]]], "shared_object_id": 12, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {"2": 48}}, "shared_object_id": 57}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -ø root.layer_with_weights-3"_tf_keras_layer*Á {"name": "conv1d_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Conv1D", "config": {"name": "conv1d_1", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 13}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 14}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["batch_normalization", 0, 0, {}]]], "shared_object_id": 15, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 3, "axes": {"-1": 48}}, "shared_object_id": 58}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -­ root.layer-7"_tf_keras_layer*ƒ{"name": "dropout_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["conv1d_1", 0, 0, {}]]], "shared_object_id": 16}2 -¿  root.layer_with_weights-4"_tf_keras_layer*ˆ {"name": "batch_normalization_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "BatchNormalization", "config": {"name": "batch_normalization_1", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 17}, "gamma_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 18}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 19}, "moving_variance_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 20}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "inbound_nodes": [[["dropout_1", 0, 0, {}]]], "shared_object_id": 21, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {"2": 48}}, "shared_object_id": 59}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -ú  -root.layer_with_weights-5"_tf_keras_layer*à {"name": "conv1d_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Conv1D", "config": {"name": "conv1d_2", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 22}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 23}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["batch_normalization_1", 0, 0, {}]]], "shared_object_id": 24, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 3, "axes": {"-1": 48}}, "shared_object_id": 60}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -®  root.layer-10"_tf_keras_layer*ƒ{"name": "dropout_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["conv1d_2", 0, 0, {}]]], "shared_object_id": 25}2 -¿  root.layer_with_weights-6"_tf_keras_layer*ˆ {"name": "batch_normalization_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "BatchNormalization", "config": {"name": "batch_normalization_2", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 26}, "gamma_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 27}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 28}, "moving_variance_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 29}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "inbound_nodes": [[["dropout_2", 0, 0, {}]]], "shared_object_id": 30, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {"2": 48}}, "shared_object_id": 61}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -ú  root.layer_with_weights-7"_tf_keras_layer*à {"name": "conv1d_3", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Conv1D", "config": {"name": "conv1d_3", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 31}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 32}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["batch_normalization_2", 0, 0, {}]]], "shared_object_id": 33, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 3, "axes": {"-1": 48}}, "shared_object_id": 62}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -® root.layer-13"_tf_keras_layer*ƒ{"name": "dropout_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["conv1d_3", 0, 0, {}]]], "shared_object_id": 34}2 -¿ root.layer_with_weights-8"_tf_keras_layer*ˆ {"name": "batch_normalization_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "BatchNormalization", "config": {"name": "batch_normalization_3", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 35}, "gamma_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 36}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 37}, "moving_variance_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 38}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "inbound_nodes": [[["dropout_3", 0, 0, {}]]], "shared_object_id": 39, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {"2": 48}}, "shared_object_id": 63}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -„root.layer_with_weights-9"_tf_keras_layer*Í{"name": "dense", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 96, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 40}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 41}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["batch_normalization_3", 0, 0, {}]]], "shared_object_id": 42, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 48}}, "shared_object_id": 64}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -« root.layer-16"_tf_keras_layer*€{"name": "dropout_4", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["dense", 0, 0, {}]]], "shared_object_id": 43}2 -ýroot.layer_with_weights-10"_tf_keras_layer*Å{"name": "dense_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 96, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 44}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 45}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_4", 0, 0, {}]]], "shared_object_id": 46, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 96}}, "shared_object_id": 65}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 96]}}2 -­ root.layer-18"_tf_keras_layer*‚{"name": "dropout_5", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["dense_1", 0, 0, {}]]], "shared_object_id": 47}2 -€root.layer_with_weights-11"_tf_keras_layer*È{"name": "dense_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 24, "activation": "softmax", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 48}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 49}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_5", 0, 0, {}]]], "shared_object_id": 50, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 96}}, "shared_object_id": 66}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 96]}}2 -í root.layer-20"_tf_keras_layer*Â{"name": "tf_op_layer_ArgMax", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": true, "class_name": "TensorFlowOpLayer", "config": {"name": "ArgMax", "trainable": true, "dtype": "float32", "node_def": {"name": "ArgMax", "op": "ArgMax", "input": ["dense_2/truediv", "ArgMax/dimension"], "attr": {"Tidx": {"type": "DT_INT32"}, "output_type": {"type": "DT_INT64"}, "T": {"type": "DT_FLOAT"}}}, "constants": {"1": -1}}, "inbound_nodes": [[["dense_2", 0, 0, {}]]], "shared_object_id": 51}2 -Æroot.layer_with_weights-12"_tf_keras_layer*Ž{"name": "thresh_arg_max_layer", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "ThreshArgMaxLayer", "config": {"layer was saved without config": true}}2 -º“root.keras_api.metrics.0"_tf_keras_metric*‚{"class_name": "Mean", "name": "loss", "dtype": "float32", "config": {"name": "loss", "dtype": "float32"}, "shared_object_id": 67}2 -Ê”root.keras_api.metrics.1"_tf_keras_metric*’{"class_name": "Mean", "name": "dense_2_loss", "dtype": "float32", "config": {"name": "dense_2_loss", "dtype": "float32"}, "shared_object_id": 68}2 -ã•root.keras_api.metrics.2"_tf_keras_metric*«{"class_name": "MeanMetricWrapper", "name": "acc", "dtype": "float32", "config": {"name": "acc", "dtype": "float32", "fn": "categorical_accuracy"}, "shared_object_id": 54}2 -––root.keras_api.metrics.3"_tf_keras_metric*Þ{"class_name": "Custom>F1Score", "name": "dense_2_f1_score", "dtype": "float32", "config": {"name": "dense_2_f1_score", "dtype": "float32", "num_classes": 24, "average": "micro", "threshold": null}, "shared_object_id": 55}2 \ No newline at end of file diff --git a/resources/labelers/structured_model/saved_model.pb b/resources/labelers/structured_model/saved_model.pb deleted file mode 100644 index 76274cae0..000000000 Binary files a/resources/labelers/structured_model/saved_model.pb and /dev/null differ diff --git a/resources/labelers/structured_model/variables/variables.index b/resources/labelers/structured_model/variables/variables.index deleted file mode 100644 index 627e9a577..000000000 Binary files a/resources/labelers/structured_model/variables/variables.index and /dev/null differ diff --git a/resources/labelers/unstructured_model/keras_metadata.pb b/resources/labelers/unstructured_model/keras_metadata.pb deleted file mode 100644 index dcc84a213..000000000 --- a/resources/labelers/unstructured_model/keras_metadata.pb +++ /dev/null @@ -1,29 +0,0 @@ - -ã`root"_tf_keras_network*Á`{"name": "functional_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "class_name": "Functional", "config": {"name": "functional_1", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "dtype": "string", "sparse": false, "ragged": false, "name": "input_1"}, "name": "input_1", "inbound_nodes": []}, {"class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAgAAAAQAAAATAAAAcxIAAAB0AGoBfACIAIgBgwN9AXwBUwApAU4pAtoWQ2hhcmFj\ndGVyTGV2ZWxDbm5Nb2RlbNoUX2NoYXJfZW5jb2RpbmdfbGF5ZXIpAtoJaW5wdXRfc3RyWg5jaGFy\nX2luX3ZlY3RvcikC2hRtYXhfY2hhcl9lbmNvZGluZ19pZNoKbWF4X2xlbmd0aKkA+lMvaG9tZS91\nYnVudHUvbmV3LWRwL0RhdGFQcm9maWxlci9kYXRhcHJvZmlsZXIvbGFiZWxlcnMvY2hhcmFjdGVy\nX2xldmVsX2Nubl9tb2RlbC5wedoRZW5jb2RpbmdfZnVuY3Rpb25TAgAAcwYAAAAAAQQBCgE=\n", null, {"class_name": "__tuple__", "items": [127, 3400]}]}, "function_type": "lambda", "module": "dataprofiler.labelers.character_level_cnn_model", "output_shape": {"class_name": "__tuple__", "items": [3400]}, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "name": "lambda", "inbound_nodes": [[["input_1", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding", "trainable": true, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 3400]}, "dtype": "float32", "input_dim": 129, "output_dim": 64, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": 3400}, "name": "embedding", "inbound_nodes": [[["lambda", 0, 0, {}]]]}, {"class_name": "Conv1D", "config": {"name": "conv1d", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "conv1d", "inbound_nodes": [[["embedding", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout", "inbound_nodes": [[["conv1d", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization", "inbound_nodes": [[["dropout", 0, 0, {}]]]}, {"class_name": "Conv1D", "config": {"name": "conv1d_1", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "conv1d_1", "inbound_nodes": [[["batch_normalization", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_1", "inbound_nodes": [[["conv1d_1", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_1", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_1", "inbound_nodes": [[["dropout_1", 0, 0, {}]]]}, {"class_name": "Conv1D", "config": {"name": "conv1d_2", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "conv1d_2", "inbound_nodes": [[["batch_normalization_1", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_2", "inbound_nodes": [[["conv1d_2", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_2", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_2", "inbound_nodes": [[["dropout_2", 0, 0, {}]]]}, {"class_name": "Conv1D", "config": {"name": "conv1d_3", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "conv1d_3", "inbound_nodes": [[["batch_normalization_2", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_3", "inbound_nodes": [[["conv1d_3", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_3", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_3", "inbound_nodes": [[["dropout_3", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 96, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense", "inbound_nodes": [[["batch_normalization_3", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_4", "inbound_nodes": [[["dense", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 96, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_1", "inbound_nodes": [[["dropout_4", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "name": "dropout_5", "inbound_nodes": [[["dense_1", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 24, "activation": "softmax", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_2", "inbound_nodes": [[["dropout_5", 0, 0, {}]]]}, {"class_name": "TensorFlowOpLayer", "config": {"name": "ArgMax", "trainable": true, "dtype": "float32", "node_def": {"name": "ArgMax", "op": "ArgMax", "input": ["dense_2/truediv", "ArgMax/dimension"], "attr": {"Tidx": {"type": "DT_INT32"}, "output_type": {"type": "DT_INT64"}, "T": {"type": "DT_FLOAT"}}}, "constants": {"1": -1}}, "name": "tf_op_layer_ArgMax", "inbound_nodes": [[["dense_2", 0, 0, {}]]]}, {"class_name": "ThreshArgMaxLayer", "config": {"layer was saved without config": true}, "name": "thresh_arg_max_layer", "inbound_nodes": [[["tf_op_layer_ArgMax", 0, 0, {"confidence_layer": ["dense_2", 0, 0]}]]]}], "input_layers": [["input_1", 0, 0]], "output_layers": [["dense_2", 0, 0], ["tf_op_layer_ArgMax", 0, 0], ["thresh_arg_max_layer", 0, 0]]}, "shared_object_id": 52, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, null]}, "ndim": 2, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, null]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, null]}, "string", "input_1"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, null]}, "string", "input_1"]}, "keras_version": "2.6.0", "backend": "tensorflow", "model_config": {"class_name": "Functional"}, "training_config": {"loss": {"dense_2": "categorical_crossentropy"}, "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "acc", "dtype": "float32", "fn": "categorical_accuracy"}, "shared_object_id": 54}, {"class_name": "Custom>F1Score", "config": {"name": "dense_2_f1_score", "dtype": "float32", "num_classes": 24, "average": "micro", "threshold": null}, "shared_object_id": 55}], [null], [null]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Adam", "config": {"name": "Adam", "learning_rate": 0.0010000000474974513, "decay": 0.0, "beta_1": 0.8999999761581421, "beta_2": 0.9990000128746033, "epsilon": 1e-07, "amsgrad": false}}}}2 -ú root.layer-0"_tf_keras_input_layer*Ê{"class_name": "InputLayer", "name": "input_1", "dtype": "string", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "dtype": "string", "sparse": false, "ragged": false, "name": "input_1"}}2 -Å root.layer-1"_tf_keras_layer*›{"name": "lambda", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Lambda", "config": {"name": "lambda", "trainable": true, "dtype": "float32", "function": {"class_name": "__tuple__", "items": ["4wEAAAAAAAAAAgAAAAQAAAATAAAAcxIAAAB0AGoBfACIAIgBgwN9AXwBUwApAU4pAtoWQ2hhcmFj\ndGVyTGV2ZWxDbm5Nb2RlbNoUX2NoYXJfZW5jb2RpbmdfbGF5ZXIpAtoJaW5wdXRfc3RyWg5jaGFy\nX2luX3ZlY3RvcikC2hRtYXhfY2hhcl9lbmNvZGluZ19pZNoKbWF4X2xlbmd0aKkA+lMvaG9tZS91\nYnVudHUvbmV3LWRwL0RhdGFQcm9maWxlci9kYXRhcHJvZmlsZXIvbGFiZWxlcnMvY2hhcmFjdGVy\nX2xldmVsX2Nubl9tb2RlbC5wedoRZW5jb2RpbmdfZnVuY3Rpb25TAgAAcwYAAAAAAQQBCgE=\n", null, {"class_name": "__tuple__", "items": [127, 3400]}]}, "function_type": "lambda", "module": "dataprofiler.labelers.character_level_cnn_model", "output_shape": {"class_name": "__tuple__", "items": [3400]}, "output_shape_type": "raw", "output_shape_module": null, "arguments": {}}, "inbound_nodes": [[["input_1", 0, 0, {}]]], "shared_object_id": 1}2 -‚root.layer_with_weights-0"_tf_keras_layer*Ë{"name": "embedding", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 3400]}, "stateful": false, "must_restore_from_config": false, "class_name": "Embedding", "config": {"name": "embedding", "trainable": true, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 3400]}, "dtype": "float32", "input_dim": 129, "output_dim": 64, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}, "shared_object_id": 2}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": 3400}, "inbound_nodes": [[["lambda", 0, 0, {}]]], "shared_object_id": 3, "build_input_shape": {"class_name": "TensorShape", "items": [null, null]}}2 -ç root.layer_with_weights-1"_tf_keras_layer*° {"name": "conv1d", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Conv1D", "config": {"name": "conv1d", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 4}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 5}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["embedding", 0, 0, {}]]], "shared_object_id": 6, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 3, "axes": {"-1": 64}}, "shared_object_id": 56}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 64]}}2 -¦ root.layer-4"_tf_keras_layer*ü{"name": "dropout", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["conv1d", 0, 0, {}]]], "shared_object_id": 7}2 -· root.layer_with_weights-2"_tf_keras_layer*€ {"name": "batch_normalization", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "BatchNormalization", "config": {"name": "batch_normalization", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 8}, "gamma_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 9}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 10}, "moving_variance_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 11}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "inbound_nodes": [[["dropout", 0, 0, {}]]], "shared_object_id": 12, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {"2": 48}}, "shared_object_id": 57}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -ø root.layer_with_weights-3"_tf_keras_layer*Á {"name": "conv1d_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Conv1D", "config": {"name": "conv1d_1", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 13}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 14}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["batch_normalization", 0, 0, {}]]], "shared_object_id": 15, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 3, "axes": {"-1": 48}}, "shared_object_id": 58}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -­ root.layer-7"_tf_keras_layer*ƒ{"name": "dropout_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["conv1d_1", 0, 0, {}]]], "shared_object_id": 16}2 -¿  root.layer_with_weights-4"_tf_keras_layer*ˆ {"name": "batch_normalization_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "BatchNormalization", "config": {"name": "batch_normalization_1", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 17}, "gamma_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 18}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 19}, "moving_variance_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 20}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "inbound_nodes": [[["dropout_1", 0, 0, {}]]], "shared_object_id": 21, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {"2": 48}}, "shared_object_id": 59}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -ú  -root.layer_with_weights-5"_tf_keras_layer*à {"name": "conv1d_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Conv1D", "config": {"name": "conv1d_2", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 22}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 23}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["batch_normalization_1", 0, 0, {}]]], "shared_object_id": 24, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 3, "axes": {"-1": 48}}, "shared_object_id": 60}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -®  root.layer-10"_tf_keras_layer*ƒ{"name": "dropout_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["conv1d_2", 0, 0, {}]]], "shared_object_id": 25}2 -¿  root.layer_with_weights-6"_tf_keras_layer*ˆ {"name": "batch_normalization_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "BatchNormalization", "config": {"name": "batch_normalization_2", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 26}, "gamma_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 27}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 28}, "moving_variance_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 29}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "inbound_nodes": [[["dropout_2", 0, 0, {}]]], "shared_object_id": 30, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {"2": 48}}, "shared_object_id": 61}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -ú  root.layer_with_weights-7"_tf_keras_layer*à {"name": "conv1d_3", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Conv1D", "config": {"name": "conv1d_3", "trainable": true, "dtype": "float32", "filters": 48, "kernel_size": {"class_name": "__tuple__", "items": [13]}, "strides": {"class_name": "__tuple__", "items": [1]}, "padding": "same", "data_format": "channels_last", "dilation_rate": {"class_name": "__tuple__", "items": [1]}, "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 31}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 32}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["batch_normalization_2", 0, 0, {}]]], "shared_object_id": 33, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 3, "axes": {"-1": 48}}, "shared_object_id": 62}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -® root.layer-13"_tf_keras_layer*ƒ{"name": "dropout_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["conv1d_3", 0, 0, {}]]], "shared_object_id": 34}2 -¿ root.layer_with_weights-8"_tf_keras_layer*ˆ {"name": "batch_normalization_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "BatchNormalization", "config": {"name": "batch_normalization_3", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 35}, "gamma_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 36}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 37}, "moving_variance_initializer": {"class_name": "Ones", "config": {}, "shared_object_id": 38}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "inbound_nodes": [[["dropout_3", 0, 0, {}]]], "shared_object_id": 39, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {"2": 48}}, "shared_object_id": 63}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -„root.layer_with_weights-9"_tf_keras_layer*Í{"name": "dense", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 96, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 40}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 41}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["batch_normalization_3", 0, 0, {}]]], "shared_object_id": 42, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 48}}, "shared_object_id": 64}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 48]}}2 -« root.layer-16"_tf_keras_layer*€{"name": "dropout_4", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_4", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["dense", 0, 0, {}]]], "shared_object_id": 43}2 -ýroot.layer_with_weights-10"_tf_keras_layer*Å{"name": "dense_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 96, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 44}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 45}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_4", 0, 0, {}]]], "shared_object_id": 46, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 96}}, "shared_object_id": 65}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 96]}}2 -­ root.layer-18"_tf_keras_layer*‚{"name": "dropout_5", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dropout", "config": {"name": "dropout_5", "trainable": true, "dtype": "float32", "rate": 0.073, "noise_shape": null, "seed": null}, "inbound_nodes": [[["dense_1", 0, 0, {}]]], "shared_object_id": 47}2 -€root.layer_with_weights-11"_tf_keras_layer*È{"name": "dense_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 24, "activation": "softmax", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 48}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 49}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["dropout_5", 0, 0, {}]]], "shared_object_id": 50, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 96}}, "shared_object_id": 66}, "build_input_shape": {"class_name": "TensorShape", "items": [null, null, 96]}}2 -í root.layer-20"_tf_keras_layer*Â{"name": "tf_op_layer_ArgMax", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": true, "class_name": "TensorFlowOpLayer", "config": {"name": "ArgMax", "trainable": true, "dtype": "float32", "node_def": {"name": "ArgMax", "op": "ArgMax", "input": ["dense_2/truediv", "ArgMax/dimension"], "attr": {"Tidx": {"type": "DT_INT32"}, "output_type": {"type": "DT_INT64"}, "T": {"type": "DT_FLOAT"}}}, "constants": {"1": -1}}, "inbound_nodes": [[["dense_2", 0, 0, {}]]], "shared_object_id": 51}2 -Æroot.layer_with_weights-12"_tf_keras_layer*Ž{"name": "thresh_arg_max_layer", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "class_name": "ThreshArgMaxLayer", "config": {"layer was saved without config": true}}2 -º“root.keras_api.metrics.0"_tf_keras_metric*‚{"class_name": "Mean", "name": "loss", "dtype": "float32", "config": {"name": "loss", "dtype": "float32"}, "shared_object_id": 67}2 -Ê”root.keras_api.metrics.1"_tf_keras_metric*’{"class_name": "Mean", "name": "dense_2_loss", "dtype": "float32", "config": {"name": "dense_2_loss", "dtype": "float32"}, "shared_object_id": 68}2 -ã•root.keras_api.metrics.2"_tf_keras_metric*«{"class_name": "MeanMetricWrapper", "name": "acc", "dtype": "float32", "config": {"name": "acc", "dtype": "float32", "fn": "categorical_accuracy"}, "shared_object_id": 54}2 -––root.keras_api.metrics.3"_tf_keras_metric*Þ{"class_name": "Custom>F1Score", "name": "dense_2_f1_score", "dtype": "float32", "config": {"name": "dense_2_f1_score", "dtype": "float32", "num_classes": 24, "average": "micro", "threshold": null}, "shared_object_id": 55}2 \ No newline at end of file diff --git a/resources/labelers/unstructured_model/saved_model.pb b/resources/labelers/unstructured_model/saved_model.pb deleted file mode 100644 index 76274cae0..000000000 Binary files a/resources/labelers/unstructured_model/saved_model.pb and /dev/null differ diff --git a/resources/labelers/unstructured_model/variables/variables.index b/resources/labelers/unstructured_model/variables/variables.index deleted file mode 100644 index 627e9a577..000000000 Binary files a/resources/labelers/unstructured_model/variables/variables.index and /dev/null differ diff --git a/setup.cfg b/setup.cfg index dd0e2235f..181a009af 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,19 @@ +[versioneer] +vcs = git +style = pep440 +versionfile_source = dataprofiler/_version.py +versionfile_build = dataprofiler/_version.py +tag_prefix = "" +parentdir_prefix = dataprofiler- [flake8] max-line-length = 88 extend-ignore = E203 +exclude = versioneer.py, dataprofiler/_version.py [isort] + multi_line_output=3 -skip=dataprofiler/tests/data/,venv/ +skip=dataprofiler/tests/data/,venv/, versioneer.py, dataprofiler/_version.py profile=black include_trailing_comma=True force_grid_wrap=0 @@ -12,6 +21,9 @@ use_parentheses=True line_length=88 [mypy] +files = dataprofiler +exclude = (^|.*/)dataprofiler/tests/ +implicit_optional = True warn_return_any = True warn_unused_configs = True ignore_missing_imports = True diff --git a/setup.py b/setup.py index f8b5eaf8e..19abdf844 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ """A setuptools for the Data Profiler Application and Python Libraries.""" -import os - # To use a consistent encoding from codecs import open from os import path @@ -10,7 +8,7 @@ from setuptools import find_packages, setup # Load package version -from dataprofiler.version import __version__ +import versioneer here = path.abspath(path.dirname(__file__)) @@ -42,9 +40,6 @@ reports_packages = f.read().splitlines() resource_dir = "resources" -default_labeler_files = [ - (d, [os.path.join(d, f) for f in files]) for d, _, files in os.walk(resource_dir) -] DESCRIPTION = ( @@ -53,8 +48,9 @@ setup( name="DataProfiler", - version=__version__, - python_requires=">=3.8", + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), + python_requires=">=3.10", description=DESCRIPTION, long_description=LONG_DESCRIPTION, long_description_content_type="text/markdown", @@ -109,15 +105,14 @@ # # If there are data files included in your packages that need to be # # installed, specify them here. If using Python 2.6 or less, then these # # have to be included in MANIFEST.in as well. - # package_data={ - # 'data': [], - # }, + package_data={ + "dataprofiler": [f"dataprofiler/{resource_dir}/*"], + }, # # # Although 'package_data' is the preferred approach, in some case you may # # need to place data files outside of your packages. See: # # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa # # In this case, 'data_file' will be installed into '/my_data' - data_files=default_labeler_files, include_package_data=True, ) diff --git a/tox.ini b/tox.ini index 55fa50147..9b7294818 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, docs, pypi-description, manifest, precom +envlist = py310, py311, py312, py313, pypi-description, manifest, precom [testenv] @@ -16,32 +16,37 @@ deps = -rrequirements-reports.txt -rrequirements-test.txt commands = - python3 -m pytest dataprofiler/tests/ --cov=dataprofiler --cov-fail-under=80 --cov-report=xml:dist/coverage.xml --forked + python3 -m pytest dataprofiler/tests/ --cov=dataprofiler --cov-fail-under=80 --cov-report=xml:coverage.xml +# add "docs" to `envlist` to run the docs build #[testenv:docs] #extras = docs #changedir = docs #commands = sphinx-build -b html source _build [testenv:pypi-description] -skip_install = true deps = + {[testenv]deps} twine wheel - pip >= 19.0.0 +skip_install = true commands = python setup.py sdist bdist_wheel twine check dist/* [testenv:manifest] -deps = check-manifest +deps = + {[testenv]deps} + check-manifest skip_install = true commands = check-manifest # skip isort for infinite loop issues between tox and top level settings [testenv:precom] skip_install = true -deps = pre-commit +deps = + {[testenv]deps} + pre-commit commands = pre-commit run black --all-files --verbose # if you use the walrus operator on Python 3.8 disable the flake8 check diff --git a/versioneer.py b/versioneer.py new file mode 100644 index 000000000..fcbc15bd1 --- /dev/null +++ b/versioneer.py @@ -0,0 +1,1741 @@ +# Version: 0.19 + +"""The Versioneer - like a rocketeer, but for versions. +The Versioneer +============== +* like a rocketeer, but for versions! +* https://github.com/python-versioneer/python-versioneer +* Brian Warner +* License: Public Domain +* Compatible with: Python 3.6, 3.7, 3.8, 3.9 and pypy3 +* [![Latest Version][pypi-image]][pypi-url] +* [![Build Status][travis-image]][travis-url] +This is a tool for managing a recorded version number in distutils-based +python projects. The goal is to remove the tedious and error-prone "update +the embedded version string" step from your release process. Making a new +release should be as easy as recording a new tag in your version-control +system, and maybe making new tarballs. +## Quick Install +* `pip install versioneer` to somewhere in your $PATH +* add a `[versioneer]` section to your setup.cfg (see [Install](INSTALL.md)) +* run `versioneer install` in your source tree, commit the results +* Verify version information with `python setup.py version` +## Version Identifiers +Source trees come from a variety of places: +* a version-control system checkout (mostly used by developers) +* a nightly tarball, produced by build automation +* a snapshot tarball, produced by a web-based VCS browser, like github's + "tarball from tag" feature +* a release tarball, produced by "setup.py sdist", distributed through PyPI +Within each source tree, the version identifier (either a string or a number, +this tool is format-agnostic) can come from a variety of places: +* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows + about recent "tags" and an absolute revision-id +* the name of the directory into which the tarball was unpacked +* an expanded VCS keyword ($Id$, etc) +* a `_version.py` created by some earlier build step +For released software, the version identifier is closely related to a VCS +tag. Some projects use tag names that include more than just the version +string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool +needs to strip the tag prefix to extract the version identifier. For +unreleased software (between tags), the version identifier should provide +enough information to help developers recreate the same tree, while also +giving them an idea of roughly how old the tree is (after version 1.2, before +version 1.3). Many VCS systems can report a description that captures this, +for example `git describe --tags --dirty --always` reports things like +"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the +0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has +uncommitted changes). +The version identifier is used for multiple purposes: +* to allow the module to self-identify its version: `myproject.__version__` +* to choose a name and prefix for a 'setup.py sdist' tarball +## Theory of Operation +Versioneer works by adding a special `_version.py` file into your source +tree, where your `__init__.py` can import it. This `_version.py` knows how to +dynamically ask the VCS tool for version information at import time. +`_version.py` also contains `$Revision$` markers, and the installation +process marks `_version.py` to have this marker rewritten with a tag name +during the `git archive` command. As a result, generated tarballs will +contain enough information to get the proper version. +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. +## Installation +See [INSTALL.md](./INSTALL.md) for detailed installation instructions. +## Version-String Flavors +Code which uses Versioneer can learn about its version string at runtime by +importing `_version` from your main `__init__.py` file and running the +`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can +import the top-level `versioneer.py` and run `get_versions()`. +Both functions return a dictionary with different flavors of version +information: +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". +* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the + commit date in ISO 8601 format. This will be None if the date is not + available. +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the +developers). `version` is suitable for display in an "about" box or a CLI +`--version` output: it can be easily compared against release notes and lists +of bugs fixed in various releases. +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions +## Styles +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". +Other styles are available. See [details.md](details.md) in the Versioneer +source tree for descriptions. +## Debugging +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). +## Known Limitations +Some situations are known to cause problems for Versioneer. This details the +most significant ones. More can be found on Github +[issues page](https://github.com/python-versioneer/python-versioneer/issues). +### Subprojects +Versioneer has limited support for source trees in which `setup.py` is not in +the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are +two common reasons why `setup.py` might not be in the root: +* Source trees which contain multiple subprojects, such as + [Buildbot](https://github.com/buildbot/buildbot), which contains both + "master" and "slave" subprojects, each with their own `setup.py`, + `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI + distributions (and upload multiple independently-installable tarballs). +* Source trees whose main purpose is to contain a C library, but which also + provide bindings to Python (and perhaps other languages) in subdirectories. +Versioneer will look for `.git` in parent directories, and most operations +should get the right version string. However `pip` and `setuptools` have bugs +and implementation details which frequently cause `pip install .` from a +subproject directory to fail to find a correct version string (so it usually +defaults to `0+unknown`). +`pip install --editable .` should work correctly. `setup.py install` might +work too. +Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in +some later version. +[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking +this issue. The discussion in +[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the +issue from the Versioneer side in more detail. +[pip PR#3176](https://github.com/pypa/pip/pull/3176) and +[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve +pip to let Versioneer work correctly. +Versioneer-0.16 and earlier only looked for a `.git` directory next to the +`setup.cfg`, so subprojects were completely unsupported with those releases. +### Editable installs with setuptools <= 18.5 +`setup.py develop` and `pip install --editable .` allow you to install a +project into a virtualenv once, then continue editing the source code (and +test) without re-installing after every change. +"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a +convenient way to specify executable scripts that should be installed along +with the python package. +These both work as expected when using modern setuptools. When using +setuptools-18.5 or earlier, however, certain operations will cause +`pkg_resources.DistributionNotFound` errors when running the entrypoint +script, which must be resolved by re-installing the package. This happens +when the install happens with one version, then the egg_info data is +regenerated while a different version is checked out. Many setup.py commands +cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into +a different virtualenv), so this can be surprising. +[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes +this one, but upgrading to a newer version of setuptools should probably +resolve it. +## Updating Versioneer +To upgrade your project to a new release of Versioneer, do the following: +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files +## Future Directions +This tool is designed to make it easily extended to other version-control +systems: all VCS-specific components are in separate directories like +src/git/ . The top-level `versioneer.py` script is assembled from these +components by running make-versioneer.py . In the future, make-versioneer.py +will take a VCS name as an argument, and will construct a version of +`versioneer.py` that is specific to the given VCS. It might also take the +configuration arguments that are currently provided manually during +installation by editing setup.py . Alternatively, it might go the other +direction and include code from all supported VCS systems, reducing the +number of intermediate scripts. +## Similar projects +* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time + dependency +* [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of + versioneer +## License +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . +[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg +[pypi-url]: https://pypi.python.org/pypi/versioneer/ +[travis-image]: +https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg +[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer +""" + +import configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_root(): + """Get the project root directory. + We require that all commands are run from the project root, i.e. the + directory that contains setup.py, setup.cfg, and versioneer.py . + """ + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + me_dir = os.path.normcase(os.path.splitext(me)[0]) + vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) + if me_dir != vsr_dir: + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py) + ) + except NameError: + pass + return root + + +def get_config_from_root(root): + """Read the project setup.cfg file to determine Versioneer config.""" + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.ConfigParser() + with open(setup_cfg) as f: + parser.read_file(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Create decorator to mark a method as the handler of a VCS.""" + + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) + break + except OSError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print(f"unable to find command, tried {commands}") + return None, None + stdout = p.communicate()[0].strip().decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +LONG_VERSION_PY[ + "git" +] = r''' +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. +# This file is released into the public domain. Generated by +# versioneer-0.19 (https://github.com/python-versioneer/python-versioneer) +"""Git implementation of _version.py.""" +import errno +import os +import re +import subprocess +import sys +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.verbose = False + return cfg +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" +LONG_VERSION_PY = {} +HANDLERS = {} +def register_vcs_handler(vcs, method): # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %%s" %% dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %%s" %% (commands,)) + return None, None + stdout = p.communicate()[0].strip().decode() + if p.returncode != 0: + if verbose: + print("unable to run %%s (error)" %% dispcmd) + print("stdout was %%s" %% stdout) + return None, p.returncode + return stdout, p.returncode +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + if verbose: + print("Tried directories %%s but none started with prefix %%s" %% + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %%d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%%s', no digits" %% ",".join(refs - tags)) + if verbose: + print("likely tags: %%s" %% ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %%s" %% r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %%s not under git control" %% root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + # now we have TAG-NUM-gHEX or HEX + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + # commit: short hex revision ID + pieces["short"] = mo.group(3) + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], + cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + return pieces +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered +def render_pep440_pre(pieces): + """TAG[.post0.devDISTANCE] -- No -dirty. + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post0.dev%%d" %% pieces["distance"] + else: + # exception #1 + rendered = "0.post0.dev%%d" %% pieces["distance"] + return rendered +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + The ".dev0" means dirty. + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + Like 'git describe --tags --dirty --always'. + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + if not style or style == "default": + style = "pep440" # the default + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + cfg = get_config() + verbose = cfg.verbose + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} +''' + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs) + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except OSError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = {r.strip() for r in refnames.strip("()").split(",")} + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = {r for r in refs if re.search(r"\d", r)} + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix) :] + if verbose: + print("picking %s" % r) + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[: git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = "tag '{}' doesn't start with prefix '{}'".format( + full_tag, + tag_prefix, + ) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix) :] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def do_vcs_install(manifest_in, versionfile_source, ipy): + """Git-specific installation logic for Versioneer. + For Git, this means creating/changing .gitattributes to mark _version.py + for export-subst keyword substitution. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) + try: + me = __file__ + if me.endswith(".pyc") or me.endswith(".pyo"): + me = os.path.splitext(me)[0] + ".py" + versioneer_file = os.path.relpath(me) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) + present = False + try: + f = open(".gitattributes") + for line in f.readlines(): + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + f.close() + except OSError: + pass + if not present: + f = open(".gitattributes", "a+") + f.write("%s export-subst\n" % versionfile_source) + f.close() + files.append(".gitattributes") + run_command(GITS, ["add", "--"] + files) + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +SHORT_VERSION_PY = """ +# This file was generated by 'versioneer.py' (0.19) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. +import json +version_json = ''' +%s +''' # END VERSION_JSON +def get_versions(): + return json.loads(version_json) +""" + + +def versions_from_file(filename): + """Try to determine the version from _version.py if present.""" + try: + with open(filename) as f: + contents = f.read() + except OSError: + raise NotThisMethod("unable to read _version.py") + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) + if not mo: + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + + +def write_to_version_file(filename, versions): + """Write the given version number to the given _version.py file.""" + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + + print("set {} to '{}'".format(filename, versions["version"])) + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post0.devDISTANCE] -- No -dirty. + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post0.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post0.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + The ".dev0" means dirty. + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + Like 'git describe --tags --dirty --always'. + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } + + +class VersioneerBadRootError(Exception): + """The project root directory is unknown or missing key files.""" + + +def get_versions(verbose=False): + """Get the project version from whatever source is available. + Returns dict with two keys: 'version' and 'full'. + """ + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) + return ver + except NotThisMethod: + pass + + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print(f"got version from file {versionfile_abs} {ver}") + return ver + except NotThisMethod: + pass + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) + return ver + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: + pass + + if verbose: + print("unable to compute version") + + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } + + +def get_version(): + """Get the short version string for this project.""" + return get_versions()["version"] + + +def get_cmdclass(cmdclass=None): + """Get the custom setuptools/distutils subclasses used by Versioneer. + If the package uses a different cmdclass (e.g. one from numpy), it + should be provide as an argument. + """ + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/python-versioneer/python-versioneer/issues/52 + + cmds = {} if cmdclass is None else cmdclass.copy() + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + print(" date: %s" % vers.get("date")) + if vers["error"]: + print(" error: %s" % vers["error"]) + + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + # pip install: + # copies source tree to a tempdir before running egg_info/etc + # if .git isn't copied too, 'git describe' will fail + # then does setup.py bdist_wheel, or sometimes setup.py install + # setup.py egg_info -> ? + + # we override different "build_py" commands for both environments + if "build_py" in cmds: + _build_py = cmds["build_py"] + elif "setuptools" in sys.modules: + from setuptools.command.build_py import build_py as _build_py + else: + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + cmds["build_py"] = cmd_build_py + + if "setuptools" in sys.modules: + from setuptools.command.build_ext import build_ext as _build_ext + else: + from distutils.command.build_ext import build_ext as _build_ext + + class cmd_build_ext(_build_ext): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_ext.run(self) + if self.inplace: + # build_ext --inplace will only build extensions in + # build/lib<..> dir with no _version.py to write to. + # As in place builds will already have a _version.py + # in the module dir, we do not need to write one. + return + # now locate _version.py in the new build/ directory and replace + # it with an updated value + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + cmds["build_ext"] = cmd_build_ext + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + + # nczeczulin reports that py2exe won't like the pep440-style string + # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. + # setup(console=[{ + # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION + # "product_version": versioneer.get_version(), + # ... + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + if "py2exe" in sys.modules: # py2exe enabled? + from py2exe.distutils_buildexe import py2exe as _py2exe + + class cmd_py2exe(_py2exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _py2exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + cmds["py2exe"] = cmd_py2exe + + # we override different "sdist" commands for both environments + if "sdist" in cmds: + _sdist = cmds["sdist"] + elif "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- +You will also need to edit your setup.py to use the results: + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = +""" + +INIT_PY_SNIPPET = """ +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions +""" + + +def do_setup(): + """Do main VCS-independent setup function for installing Versioneer.""" + root = get_root() + try: + cfg = get_config_from_root(root) + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") + if os.path.exists(ipy): + try: + with open(ipy) as f: + old = f.read() + except OSError: + old = "" + if INIT_PY_SNIPPET not in old: + print(" appending to %s" % ipy) + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) + else: + print(" %s unmodified" % ipy) + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in) as f: + for line in f: + if line.startswith("include "): + for include in line.split()[1:]: + simple_includes.add(include) + except OSError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: + f.write("include versioneer.py\n") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print( + " appending versionfile_source ('%s') to MANIFEST.in" + % cfg.versionfile_source + ) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") + + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-subst keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 + + +def scan_setup_py(): + """Validate the contents of setup.py against Versioneer's expectations.""" + found = set() + setters = False + errors = 0 + with open("setup.py") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1)