diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..343b82a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,58 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Lint (ruff) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install ruff + - run: ruff check src tests + - run: ruff format --check src tests + + test: + name: Test (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install + run: python -m pip install -e ".[dev,contour]" + - name: Test + run: pytest --cov --cov-report=term-missing + + build: + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install build twine + - run: python -m build + - run: twine check dist/* + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..ff46db3 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,47 @@ +name: Publish to PyPI + +# Publishes when a version tag (v1.2.3) is pushed. Uses PyPI "trusted +# publishing" (OpenID Connect), so no API token is stored in the repository: +# register this workflow once at https://pypi.org/manage/account/publishing/ +# (owner: adaj, repository: predspot, workflow: publish.yml, environment: pypi). + +on: + push: + tags: ["v[0-9]+.[0-9]+.[0-9]+*"] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Check that the tag matches the package version + run: | + python -m pip install -e . + PKG_VERSION="v$(python -c 'import predspot; print(predspot.__version__)')" + echo "tag=${GITHUB_REF_NAME} package=${PKG_VERSION}" + test "${GITHUB_REF_NAME}" = "${PKG_VERSION}" + - run: python -m pip install build twine + - run: python -m build + - run: twine check dist/* + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/predspot + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index 1c0482f..e971876 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ build/ dist/ *.egg-info/ +.coverage +htmlcov/ +.ruff_cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5b45ac8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,49 @@ +# Changelog + +All notable changes to Predspot are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses +[Semantic Versioning](https://semver.org/). + +## [Unreleased] + +## [0.2.0] - 2026-09 + +Revival release: the code base now targets Python 3.10+ with current +versions of pandas (>= 2.2), GeoPandas (>= 1.0), scikit-learn and statsmodels. + +### Added +- `QuadratCount` mapping (event counts per cell) as a first-class alternative + to `KDE`, usable with hexagonal (`create_gridhexagonal`) and square + (`create_gridsquares`) grids inside `PredictionPipeline`. +- `pipeline.build_default_pipeline` and reproducible + `pipeline.generate_testdata(..., seed=...)`. +- `PredictionPipeline.features`, `.next_time` and `random_state`. +- Test suite (pytest) and continuous integration for Python 3.10-3.13. +- `pyproject.toml` packaging (src layout) and automated PyPI publishing. + +### Changed +- `tfreq` is optional in the feature classes (inferred from the series). +- Debug `print`s replaced with the `logging` module (`predspot` logger); the + `debug=` arguments were removed. +- Wrapper estimators expose their inner estimator as `.estimator` + (previously `._estimator`). +- `Dataset` no longer modifies the input DataFrame and requires the study + area to have a CRS. +- Grid centroids are computed in a projected CRS; grids accept study areas in + any CRS. +- `geojsoncontour` is an optional dependency (`pip install predspot[contour]`). + +### Removed +- `QuadratCount2`, `KGrid` and the hard dependencies on `descartes`, + `contextily` and `rtree`. + +### Fixed +- Compatibility with pandas 2/3 (`'ME'` offsets, `DataFrame.append`, + positional `Series` indexing), GeoPandas 1.x (`sjoin(predicate=)`, CRS + strings, `gpd.datasets`) and scikit-learn 1.x (`FeatureUnion` internals). +- `Seasonality`/`Trend` never called `STL(...).fit()`. +- `FeatureScaling` had no `fit`, so scalers inside a `Pipeline` were never fitted. + +## [0.1.3] - 2020 + +Original master's thesis release. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5a46c84 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,29 @@ +# Contributing + +Thanks for your interest in Predspot! Issues and pull requests are welcome. + +## Development setup + +```bash +git clone https://github.com/adaj/predspot.git +cd predspot +python -m venv .venv && source .venv/bin/activate +pip install -e ".[dev,contour]" +``` + +## Checks + +```bash +ruff check src tests # lint +ruff format src tests # format +pytest # tests (~10 s) +``` + +CI runs the same checks on Python 3.10 to 3.13 for every pull request. + +## Releasing + +1. Bump `__version__` in `src/predspot/__init__.py` and update `CHANGELOG.md`. +2. Merge to `master`, then tag and push: `git tag v0.2.0 && git push origin v0.2.0`. +3. The `Publish to PyPI` workflow builds the distribution and uploads it via + PyPI trusted publishing. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9da02ae --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE README.md CHANGELOG.md CONTRIBUTING.md +recursive-include tests *.py +prune docs diff --git a/README.md b/README.md index 5ca6949..81c7214 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # Predspot +[![CI](https://github.com/adaj/predspot/actions/workflows/ci.yml/badge.svg)](https://github.com/adaj/predspot/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/predspot.svg)](https://pypi.org/project/predspot/) +[![Python](https://img.shields.io/pypi/pyversions/predspot.svg)](https://pypi.org/project/predspot/) +[![License: BSD-3](https://img.shields.io/badge/license-BSD--3--Clause-blue.svg)](LICENSE) + ## Overview ๐Ÿ“– Predspot is a Python library for spatio-temporal crime prediction and hotspot detection. It combines machine learning techniques with spatial analysis to help predict and visualize crime patterns across time and space. @@ -89,23 +94,32 @@ Predspot has four main modules: Predspot requires Python 3.10 or newer. +```bash +pip install predspot # from PyPI +pip install "predspot[contour]" # + GeoJSON contour export (geojsoncontour) +``` + +From source, for development: + ```bash git clone https://github.com/adaj/predspot.git cd predspot -pip install . +pip install -e ".[dev,contour]" ``` Core dependencies (installed automatically): pandas, geopandas, shapely, -numpy, scipy, scikit-learn, statsmodels and matplotlib. The optional -`geojsoncontour` package enables `predspot.utilities.contour_geojson`. +numpy, scipy, scikit-learn, statsmodels and matplotlib. ### Tests ๐Ÿงช ```bash -pip install pytest -pytest +ruff check src tests # lint +pytest # ~10 s ``` +See [CONTRIBUTING.md](CONTRIBUTING.md) for the release process and +[CHANGELOG.md](CHANGELOG.md) for what changed between versions. + ### Input Data Format ๐Ÿ“Š The crime data should be a pandas DataFrame with the following required columns: diff --git a/install.md b/install.md deleted file mode 100644 index 16269a2..0000000 --- a/install.md +++ /dev/null @@ -1,6 +0,0 @@ -``` -$ git clone https://github.com/adaj/predspot.git -$ cd predspot -$ pip install . # Python >= 3.10 -$ pip install pytest && pytest -``` diff --git a/predspot/__init__.py b/predspot/__init__.py deleted file mode 100644 index 9ba0d1f..0000000 --- a/predspot/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Predspot โ€” predicting crime hotspots with machine learning. - -Typical use:: - - from predspot import Dataset, PredictionPipeline - from predspot.crime_mapping import KDE, create_gridpoints - from predspot.feature_engineering import Seasonality, Trend, Diff - from predspot.utilities import PandasFeatureUnion -""" - -from predspot import (crime_mapping, dataset_preparation, feature_engineering, - ml_modelling, utilities) -from predspot.crime_mapping import (KDE, QuadratCount, create_gridhexagonal, - create_gridpoints, create_gridsquares) -from predspot.dataset_preparation import Dataset -from predspot.ml_modelling import PredictionPipeline -from predspot.utilities import PandasFeatureUnion - -__version__ = '0.2.0' - -__all__ = [ - 'Dataset', 'PredictionPipeline', 'PandasFeatureUnion', - 'KDE', 'QuadratCount', - 'create_gridpoints', 'create_gridhexagonal', 'create_gridsquares', - 'crime_mapping', 'dataset_preparation', 'feature_engineering', - 'ml_modelling', 'utilities', -] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f014c0a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,95 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "predspot" +dynamic = ["version"] +description = "Predicting crime hotspots with machine learning" +readme = "README.md" +license = "BSD-3-Clause" +license-files = ["LICENSE"] +requires-python = ">=3.10" +authors = [ + { name = "Adelson Araujo", email = "adelson.dias@gmail.com" }, +] +keywords = [ + "crime", "hotspots", "spatio-temporal", "kernel density estimation", + "geopandas", "scikit-learn", "forecasting", "criminology", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: GIS", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dependencies = [ + "numpy>=1.24", + "pandas>=2.2", + "geopandas>=1.0", + "shapely>=2.0", + "scipy>=1.10", + "scikit-learn>=1.3", + "statsmodels>=0.14", + "matplotlib>=3.7", +] + +[project.optional-dependencies] +contour = ["geojsoncontour>=0.4"] +dev = [ + "pytest>=8", + "pytest-cov>=5", + "ruff>=0.6", + "build>=1.2", + "twine>=5", +] + +[project.urls] +Homepage = "https://github.com/adaj/predspot" +Documentation = "https://adaj.github.io/predspot/" +Repository = "https://github.com/adaj/predspot" +Issues = "https://github.com/adaj/predspot/issues" +Changelog = "https://github.com/adaj/predspot/blob/master/CHANGELOG.md" + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.dynamic] +version = { attr = "predspot.__version__" } + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra" +filterwarnings = [ + "error::DeprecationWarning:predspot", + "error::FutureWarning:predspot", +] + +[tool.coverage.run] +source = ["predspot"] +branch = true + +[tool.ruff] +line-length = 100 +target-version = "py310" +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "B", "UP", "NPY", "PD"] +ignore = [ + "PD011", # .values is used deliberately for numpy interop +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["B011"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 331ba9b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -numpy>=1.24 -pandas>=2.2 -geopandas>=1.0 -shapely>=2.0 -scipy>=1.10 -scikit-learn>=1.3 -statsmodels>=0.14 -matplotlib>=3.7 diff --git a/setup.py b/setup.py deleted file mode 100644 index b41b761..0000000 --- a/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -from setuptools import setup - -setup( - name='predspot', - version='0.2.0', - description="Predicting crime hotspots with machine learning", - url='https://github.com/adaj/predspot', - author="Adelson Araujo", - author_email='adelson.dias@gmail.com', - packages=['predspot'], - install_requires=[ - 'numpy>=1.24', - 'pandas>=2.2', - 'geopandas>=1.0', - 'shapely>=2.0', - 'scipy>=1.10', - 'scikit-learn>=1.3', - 'statsmodels>=0.14', - 'matplotlib>=3.7', - ], - extras_require={'contour': ['geojsoncontour']}, - classifiers=[ - 'Intended Audience :: Science/Research', - 'License :: BSD 3-Clause License' - ], - python_requires='>=3.10', -) diff --git a/src/predspot/__init__.py b/src/predspot/__init__.py new file mode 100644 index 0000000..2dae115 --- /dev/null +++ b/src/predspot/__init__.py @@ -0,0 +1,46 @@ +""" +Predspot โ€” predicting crime hotspots with machine learning. + +Typical use:: + + from predspot import Dataset, PredictionPipeline + from predspot.crime_mapping import KDE, create_gridpoints + from predspot.feature_engineering import Seasonality, Trend, Diff + from predspot.utilities import PandasFeatureUnion +""" + +from predspot import ( + crime_mapping, + dataset_preparation, + feature_engineering, + ml_modelling, + utilities, +) +from predspot.crime_mapping import ( + KDE, + QuadratCount, + create_gridhexagonal, + create_gridpoints, + create_gridsquares, +) +from predspot.dataset_preparation import Dataset +from predspot.ml_modelling import PredictionPipeline +from predspot.utilities import PandasFeatureUnion + +__version__ = "0.2.0" + +__all__ = [ + "Dataset", + "PredictionPipeline", + "PandasFeatureUnion", + "KDE", + "QuadratCount", + "create_gridpoints", + "create_gridhexagonal", + "create_gridsquares", + "crime_mapping", + "dataset_preparation", + "feature_engineering", + "ml_modelling", + "utilities", +] diff --git a/predspot/crime_mapping.py b/src/predspot/crime_mapping.py similarity index 79% rename from predspot/crime_mapping.py rename to src/predspot/crime_mapping.py index 623593f..941538d 100644 --- a/predspot/crime_mapping.py +++ b/src/predspot/crime_mapping.py @@ -16,7 +16,7 @@ inside :class:`predspot.ml_modelling.PredictionPipeline`. """ -__author__ = 'Adelson Araujo' +__author__ = "Adelson Araujo" import logging import math @@ -26,7 +26,7 @@ import numpy as np import pandas as pd from scipy.stats import gaussian_kde -from shapely.geometry import Point, Polygon +from shapely.geometry import Polygon from sklearn.base import BaseEstimator, TransformerMixin logger = logging.getLogger(__name__) @@ -41,9 +41,16 @@ # Public aliases accepted for the time frequency and the pandas offset alias # they map to. Old pandas used 'M' for month end; pandas >= 2.2 uses 'ME'. TFREQ_ALIASES = { - 'M': 'ME', 'ME': 'ME', 'MONTH': 'ME', 'MONTHLY': 'ME', - 'W': 'W', 'WEEK': 'W', 'WEEKLY': 'W', - 'D': 'D', 'DAY': 'D', 'DAILY': 'D', + "M": "ME", + "ME": "ME", + "MONTH": "ME", + "MONTHLY": "ME", + "W": "W", + "WEEK": "W", + "WEEKLY": "W", + "D": "D", + "DAY": "D", + "DAILY": "D", } @@ -62,28 +69,27 @@ def normalize_tfreq(tfreq): """ key = str(tfreq).upper() if key not in TFREQ_ALIASES: - raise ValueError( - f"Invalid tfreq {tfreq!r}. Choose (M)onthly, (W)eekly or (D)aily.") + raise ValueError(f"Invalid tfreq {tfreq!r}. Choose (M)onthly, (W)eekly or (D)aily.") return TFREQ_ALIASES[key] def tfreq_offset(tfreq): """Return the :class:`pandas.DateOffset` that advances one period of ``tfreq``.""" alias = normalize_tfreq(tfreq) - if alias == 'ME': + if alias == "ME": return pd.offsets.MonthEnd(1) - if alias == 'W': + if alias == "W": return pd.offsets.Week(1) return pd.offsets.Day(1) def _check_bbox(bbox): if not isinstance(bbox, gpd.GeoDataFrame): - raise TypeError('bbox must be a geopandas GeoDataFrame.') + raise TypeError("bbox must be a geopandas GeoDataFrame.") if bbox.crs is None: raise ValueError('bbox must have a CRS (e.g. bbox.set_crs("EPSG:4326")).') if len(bbox) == 0: - raise ValueError('bbox is empty.') + raise ValueError("bbox is empty.") def _wgs84_bounds(bbox): @@ -93,11 +99,9 @@ def _wgs84_bounds(bbox): def _clip_to_bbox(grid, bbox): """Keep only grid rows that intersect ``bbox`` (both in the same CRS).""" - keep = gpd.sjoin(grid, bbox[['geometry']], how='inner', - predicate='intersects').index.unique() + keep = gpd.sjoin(grid, bbox[["geometry"]], how="inner", predicate="intersects").index.unique() if len(keep) == 0: - raise ValueError( - 'resolution too big/coarse. No cells intersect the study area.') + raise ValueError("resolution too big/coarse. No cells intersect the study area.") return grid.loc[grid.index.isin(keep)] @@ -106,8 +110,8 @@ def _add_centroid_lonlat(grid): projected = grid.geometry.to_crs(grid.estimate_utm_crs()) centroids = projected.centroid.to_crs(WGS84) grid = grid.copy() - grid['lon'] = centroids.x.values - grid['lat'] = centroids.y.values + grid["lon"] = centroids.x.values + grid["lat"] = centroids.y.values return grid @@ -131,9 +135,9 @@ def create_gridpoints(bbox, resolution, return_coords=False): ``return_coords`` is True, a tuple ``(gridpoints, lonv, latv)``. """ if resolution <= 0: - raise ValueError('resolution must be a positive number of kilometers.') + raise ValueError("resolution must be a positive number of kilometers.") _check_bbox(bbox) - logger.debug('Creating point grid with resolution %s km', resolution) + logger.debug("Creating point grid with resolution %s km", resolution) b_w, b_s, b_e, b_n = _wgs84_bounds(bbox) nlon = max(int(np.ceil((b_e - b_w) / (resolution / KM_PER_DEG_LON))), 2) @@ -141,10 +145,10 @@ def create_gridpoints(bbox, resolution, return_coords=False): lonv, latv = np.meshgrid(np.linspace(b_w, b_e, nlon), np.linspace(b_s, b_n, nlat)) lon, lat = lonv.ravel(), latv.ravel() gridpoints = gpd.GeoDataFrame( - {'lon': lon, 'lat': lat}, - geometry=gpd.points_from_xy(lon, lat), crs=WGS84).to_crs(bbox.crs) + {"lon": lon, "lat": lat}, geometry=gpd.points_from_xy(lon, lat), crs=WGS84 + ).to_crs(bbox.crs) gridpoints = _clip_to_bbox(gridpoints, bbox) - gridpoints.index.name = 'places' + gridpoints.index.name = "places" if return_coords: return gridpoints, lonv, latv return gridpoints @@ -163,10 +167,12 @@ def create_hexagon(side, x, y): Returns: Polygon: The hexagon. """ - return Polygon([ - (x + math.cos(math.radians(angle)) * side, - y + math.sin(math.radians(angle)) * side) - for angle in range(0, 360, 60)]) + return Polygon( + [ + (x + math.cos(math.radians(angle)) * side, y + math.sin(math.radians(angle)) * side) + for angle in range(0, 360, 60) + ] + ) def create_gridhexagonal(bbox, resolution): @@ -186,12 +192,12 @@ def create_gridhexagonal(bbox, resolution): ``lon`` and ``lat`` (centroid) columns and an index named ``places``. """ if resolution <= 0: - raise ValueError('resolution must be a positive number of kilometers.') + raise ValueError("resolution must be a positive number of kilometers.") _check_bbox(bbox) - logger.debug('Creating hexagonal grid with resolution %s km', resolution) + logger.debug("Creating hexagonal grid with resolution %s km", resolution) # Side length such that the hexagon area equals resolution**2. - side_km = math.sqrt(resolution ** 2 * 2 / (3 * math.sqrt(3))) + side_km = math.sqrt(resolution**2 * 2 / (3 * math.sqrt(3))) side = side_km / KM_PER_DEG_LAT # degrees (isotropic approximation) x_min, y_min, x_max, y_max = _wgs84_bounds(bbox) @@ -222,7 +228,7 @@ def create_gridhexagonal(bbox, resolution): grid = gpd.GeoDataFrame(geometry=hexagons, crs=WGS84).to_crs(bbox.crs) grid = _clip_to_bbox(grid, bbox) grid = _add_centroid_lonlat(grid) - grid.index.name = 'places' + grid.index.name = "places" return grid @@ -239,21 +245,22 @@ def create_gridsquares(bbox, resolution=1): ``lon`` and ``lat`` (centroid) columns and an index named ``places``. """ if resolution <= 0: - raise ValueError('resolution must be a positive number of kilometers.') + raise ValueError("resolution must be a positive number of kilometers.") _check_bbox(bbox) - logger.debug('Creating square grid with resolution %s km', resolution) + logger.debug("Creating square grid with resolution %s km", resolution) x0, y0, xf, yf = _wgs84_bounds(bbox) dx = resolution / KM_PER_DEG_LON dy = resolution / KM_PER_DEG_LAT xs = np.arange(x0, xf, dx) ys = np.arange(y0, yf, dy) - squares = [Polygon([(x, y), (x + dx, y), (x + dx, y + dy), (x, y + dy)]) - for x in xs for y in ys] + squares = [ + Polygon([(x, y), (x + dx, y), (x + dx, y + dy), (x, y + dy)]) for x in xs for y in ys + ] grid = gpd.GeoDataFrame(geometry=squares, crs=WGS84).to_crs(bbox.crs) grid = _clip_to_bbox(grid, bbox) grid = _add_centroid_lonlat(grid) - grid.index.name = 'places' + grid.index.name = "places" return grid @@ -284,15 +291,20 @@ def __init__(self, tfreq, grid, start_time=None, end_time=None): self.end_time = end_time self._tfreq = normalize_tfreq(tfreq) - missing = [c for c in ('geometry', 'lon', 'lat') if c not in grid.columns] + missing = [c for c in ("geometry", "lon", "lat") if c not in grid.columns] if missing: raise ValueError( - f'Input grid must have `geometry`, `lon` and `lat` columns; missing {missing}.') + f"Input grid must have `geometry`, `lon` and `lat` columns; missing {missing}." + ) self._grid = grid self._start_time = pd.to_datetime(start_time) if start_time else None self._end_time = pd.to_datetime(end_time) if end_time else None - logger.debug('%s initialised with tfreq=%s and %d places', - type(self).__name__, self._tfreq, len(grid)) + logger.debug( + "%s initialised with tfreq=%s and %d places", + type(self).__name__, + self._tfreq, + len(grid), + ) @abstractmethod def fit_grid(self, data_points): @@ -329,13 +341,13 @@ def transform(self, data_points): pandas.Series: Values named ``crime_density`` indexed by ``(t, places)``, sorted. """ - if 't' not in data_points.columns: - raise ValueError('data_points must have a `t` timestamp column.') - events = data_points.set_index(pd.DatetimeIndex(data_points['t'])).sort_index() + if "t" not in data_points.columns: + raise ValueError("data_points must have a `t` timestamp column.") + events = data_points.set_index(pd.DatetimeIndex(data_points["t"])).sort_index() chunks = {label: chunk for label, chunk in events.resample(self._tfreq)} labels = pd.DatetimeIndex(list(chunks.keys())) time_index = self._time_index(labels) - logger.debug('Mapping %d events over %d periods', len(events), len(time_index)) + logger.debug("Mapping %d events over %d periods", len(events), len(time_index)) zeros = dict.fromkeys(self._grid.index, 0.0) rows = [] @@ -347,9 +359,9 @@ def transform(self, data_points): rows.append(self.fit_grid(chunk)) frame = pd.DataFrame(rows, index=time_index) frame = frame.reindex(columns=self._grid.index) - stseries = frame.stack() - stseries.index.names = ['t', 'places'] - stseries.name = 'crime_density' + stseries = frame.stack() # noqa: PD013 - long format with (t, places) index + stseries.index.names = ["t", "places"] + stseries.name = "crime_density" return stseries.sort_index() @@ -371,20 +383,20 @@ class KDE(SpatioTemporalMapping): time), or a positive number used directly as the KDE factor. """ - def __init__(self, tfreq, grid, start_time=None, end_time=None, bandwidth='silverman'): + def __init__(self, tfreq, grid, start_time=None, end_time=None, bandwidth="silverman"): super().__init__(tfreq, grid, start_time, end_time) self.bandwidth = bandwidth if isinstance(bandwidth, str): method = bandwidth.lower() - if method == 'auto': - method = 'silverman' - if method not in ('silverman', 'scott'): + if method == "auto": + method = "silverman" + if method not in ("silverman", "scott"): raise ValueError("bandwidth must be 'silverman', 'scott' or a number.") self._bw_method = method self._factor = None else: if bandwidth <= 0: - raise ValueError('bandwidth must be a positive number.') + raise ValueError("bandwidth must be a positive number.") self._bw_method = None self._factor = float(bandwidth) self._kernel = None @@ -413,13 +425,14 @@ def fit_grid(self, data_points, as_df=False): self._kernel = gaussian_kde(xy, bw_method=bw) if self._factor is None: self._factor = float(self._kernel.factor) - logger.debug('KDE bandwidth factor estimated with %s: %.5f', - self._bw_method, self._factor) - values = self._kernel(self._grid[['lon', 'lat']].values.T) - density = pd.DataFrame({'crime_density': values}, index=self._grid.index) + logger.debug( + "KDE bandwidth factor estimated with %s: %.5f", self._bw_method, self._factor + ) + values = self._kernel(self._grid[["lon", "lat"]].values.T) + density = pd.DataFrame({"crime_density": values}, index=self._grid.index) if as_df: return density - return density['crime_density'].to_dict() + return density["crime_density"].to_dict() class QuadratCount(SpatioTemporalMapping): @@ -438,9 +451,11 @@ class QuadratCount(SpatioTemporalMapping): def __init__(self, tfreq, grid, start_time=None, end_time=None): super().__init__(tfreq, grid, start_time, end_time) - if not grid.geom_type.isin(['Polygon', 'MultiPolygon']).all(): - raise ValueError('QuadratCount requires a polygonal grid ' - '(see create_gridhexagonal / create_gridsquares).') + if not grid.geom_type.isin(["Polygon", "MultiPolygon"]).all(): + raise ValueError( + "QuadratCount requires a polygonal grid " + "(see create_gridhexagonal / create_gridsquares)." + ) def fit_grid(self, data_points, as_df=False): """ @@ -453,15 +468,15 @@ def fit_grid(self, data_points, as_df=False): Returns: dict or DataFrame: Number of events per cell. """ - points = data_points[['geometry']].to_crs(self._grid.crs) - joined = gpd.sjoin(points, self._grid[['geometry']], how='inner', - predicate='within') + points = data_points[["geometry"]].to_crs(self._grid.crs) + joined = gpd.sjoin(points, self._grid[["geometry"]], how="inner", predicate="within") # The right index column is named after the grid index ('places'); # fall back to geopandas' default name otherwise. - col = 'places' if 'places' in joined.columns else 'index_right' + col = "places" if "places" in joined.columns else "index_right" counts = joined.groupby(col).size().reindex(self._grid.index, fill_value=0) - density = pd.DataFrame({'crime_density': counts.astype(float).values}, - index=self._grid.index) + density = pd.DataFrame( + {"crime_density": counts.astype(float).values}, index=self._grid.index + ) if as_df: return density - return density['crime_density'].to_dict() + return density["crime_density"].to_dict() diff --git a/predspot/dataset_preparation.py b/src/predspot/dataset_preparation.py similarity index 69% rename from predspot/dataset_preparation.py rename to src/predspot/dataset_preparation.py index 4f91cd3..8dda27c 100644 --- a/predspot/dataset_preparation.py +++ b/src/predspot/dataset_preparation.py @@ -7,7 +7,7 @@ of points in WGS84 and offers simple plotting and splitting helpers. """ -__author__ = 'Adelson Araujo' +__author__ = "Adelson Araujo" import logging @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) WGS84 = "EPSG:4326" -REQUIRED_COLUMNS = ('tag', 't', 'lon', 'lat') +REQUIRED_COLUMNS = ("tag", "t", "lon", "lat") class Dataset: @@ -40,17 +40,20 @@ class Dataset: def __init__(self, crimes, study_area): if not isinstance(study_area, gpd.GeoDataFrame): - raise TypeError('study_area must be a geopandas GeoDataFrame.') + raise TypeError("study_area must be a geopandas GeoDataFrame.") if study_area.crs is None: - raise ValueError('study_area must have a CRS set ' - '(e.g. study_area.set_crs("EPSG:4326")).') + raise ValueError( + 'study_area must have a CRS set (e.g. study_area.set_crs("EPSG:4326")).' + ) if not isinstance(crimes, pd.DataFrame): - raise TypeError('crimes must be a pandas DataFrame.') + raise TypeError("crimes must be a pandas DataFrame.") missing = [c for c in REQUIRED_COLUMNS if c not in crimes.columns] if missing: - raise ValueError('Input crime data must have at least `tag`, `t`, ' - f'`lon` and `lat` as columns; missing {missing}.') - logger.debug('Preparing dataset with %d crime events', len(crimes)) + raise ValueError( + "Input crime data must have at least `tag`, `t`, " + f"`lon` and `lat` as columns; missing {missing}." + ) + logger.debug("Preparing dataset with %d crime events", len(crimes)) self._study_area = study_area if isinstance(crimes, gpd.GeoDataFrame) and crimes.crs is not None: @@ -58,19 +61,22 @@ def __init__(self, crimes, study_area): else: events = crimes.copy() events = gpd.GeoDataFrame( - events.drop(columns=['geometry'], errors='ignore'), - geometry=gpd.points_from_xy(events['lon'], events['lat']), - crs=WGS84) - events['t'] = pd.to_datetime(events['t']) + events.drop(columns=["geometry"], errors="ignore"), + geometry=gpd.points_from_xy(events["lon"], events["lat"]), + crs=WGS84, + ) + events["t"] = pd.to_datetime(events["t"]) self._crimes = events def __repr__(self): - counts = self._crimes['tag'].value_counts().to_dict() - return ('predspot.Dataset<\n' - f' crimes = GeoDataFrame({self._crimes.shape[0]}),\n' - f' >> {counts}\n' - f' study_area = GeoDataFrame({self._study_area.shape[0]}),\n' - '>') + counts = self._crimes["tag"].value_counts().to_dict() + return ( + "predspot.Dataset<\n" + f" crimes = GeoDataFrame({self._crimes.shape[0]}),\n" + f" >> {counts}\n" + f" study_area = GeoDataFrame({self._study_area.shape[0]}),\n" + ">" + ) @property def crimes(self): @@ -85,8 +91,7 @@ def study_area(self): @property def shape(self): """dict: Shapes of ``crimes`` and ``study_area``.""" - return {'crimes': self._crimes.shape, - 'study_area': self._study_area.shape} + return {"crimes": self._crimes.shape, "study_area": self._study_area.shape} def plot(self, ax=None, crime_samples=1000, **kwargs): """ @@ -101,13 +106,13 @@ def plot(self, ax=None, crime_samples=1000, **kwargs): Returns: matplotlib.axes.Axes: The axes drawn on. """ - area_kwargs = {'color': 'white', 'edgecolor': 'black'} - area_kwargs.update(kwargs.pop('study_area', {})) + area_kwargs = {"color": "white", "edgecolor": "black"} + area_kwargs.update(kwargs.pop("study_area", {})) study_area = self.study_area.to_crs(WGS84) ax = study_area.plot(ax=ax, **area_kwargs) n = min(crime_samples, len(self.crimes)) - crimes_kwargs = {'marker': 'x'} - crimes_kwargs.update(kwargs.pop('crimes', {})) + crimes_kwargs = {"marker": "x"} + crimes_kwargs.update(kwargs.pop("crimes", {})) self.crimes.sample(n).plot(ax=ax, **crimes_kwargs) return ax @@ -123,8 +128,8 @@ def train_test_split(self, test_size=0.25, random_state=None): tuple: ``(train_dataset, test_dataset)``. """ if not 0 < test_size < 1: - raise ValueError('test_size must be between 0 and 1.') + raise ValueError("test_size must be between 0 and 1.") test = self.crimes.sample(frac=test_size, random_state=random_state) train = self.crimes.drop(index=test.index) - logger.debug('Split dataset: train=%d, test=%d', len(train), len(test)) + logger.debug("Split dataset: train=%d, test=%d", len(train), len(test)) return Dataset(train, self.study_area), Dataset(test, self.study_area) diff --git a/predspot/feature_engineering.py b/src/predspot/feature_engineering.py similarity index 80% rename from predspot/feature_engineering.py rename to src/predspot/feature_engineering.py index 5cfa02c..48ec5ed 100644 --- a/predspot/feature_engineering.py +++ b/src/predspot/feature_engineering.py @@ -16,7 +16,7 @@ observed one, so that the fitted model can forecast the next period. """ -__author__ = 'Adelson Araujo' +__author__ = "Adelson Araujo" import logging from abc import abstractmethod @@ -47,8 +47,9 @@ def infer_offset(time_index): time_index = pd.DatetimeIndex(time_index).unique().sort_values() freq = pd.infer_freq(time_index) if len(time_index) >= 3 else None if freq is None: - raise ValueError('Could not infer the time frequency of the series; ' - 'pass `tfreq` explicitly.') + raise ValueError( + "Could not infer the time frequency of the series; pass `tfreq` explicitly." + ) return pd.tseries.frequencies.to_offset(freq) @@ -64,7 +65,7 @@ class TimeSeriesFeatures(BaseEstimator, TransformerMixin): def __init__(self, lags, tfreq=None): if not isinstance(lags, int) or lags < 2: - raise ValueError('`lags` must be an integer greater than 1.') + raise ValueError("`lags` must be an integer greater than 1.") self.lags = lags self.tfreq = tfreq self._offset = tfreq_offset(tfreq) if tfreq is not None else None @@ -72,7 +73,7 @@ def __init__(self, lags, tfreq=None): @property def label(self): """str: Prefix of the feature columns (override in subclasses).""" - return 'feature' + return "feature" @abstractmethod def apply_ts_decomposition(self, ts): @@ -102,10 +103,10 @@ def make_lag_df(self, ts): original series restricted to the same index. """ if len(ts) <= self.lags: - raise ValueError('`lags` is higher than the number of time periods.') + raise ValueError("`lags` is higher than the number of time periods.") lag_df = pd.concat([ts.shift(lag) for lag in range(1, self.lags + 1)], axis=1) - lag_df = lag_df.iloc[self.lags:] - lag_df.columns = [f'{self.label}_{i}' for i in range(1, self.lags + 1)] + lag_df = lag_df.iloc[self.lags :] + lag_df.columns = [f"{self.label}_{i}" for i in range(1, self.lags + 1)] return lag_df, ts.loc[lag_df.index] def transform(self, stseries): @@ -119,21 +120,22 @@ def transform(self, stseries): pandas.DataFrame: Features indexed by ``(t, places)``, including one row for the period after the last observed one. """ - times = stseries.index.get_level_values('t') + times = stseries.index.get_level_values("t") offset = self._offset if self._offset is not None else infer_offset(times) - places = stseries.index.get_level_values('places').unique() - logger.debug('%s: computing %d lags for %d places', - type(self).__name__, self.lags, len(places)) + places = stseries.index.get_level_values("places").unique() + logger.debug( + "%s: computing %d lags for %d places", type(self).__name__, self.lags, len(places) + ) frames = [] for place in places: - ts = stseries.xs(place, level='places').sort_index() + ts = stseries.xs(place, level="places").sort_index() ts = self.apply_ts_decomposition(ts) ts.loc[ts.index[-1] + offset] = None # next period, to be forecast f, _ = self.make_lag_df(ts) - f['places'] = place - frames.append(f.set_index('places', append=True)) + f["places"] = place + frames.append(f.set_index("places", append=True)) X = pd.concat(frames) - X.index.names = ['t', 'places'] + X.index.names = ["t", "places"] return X.sort_index() @@ -142,7 +144,7 @@ class AR(TimeSeriesFeatures): @property def label(self): - return 'ar' + return "ar" def apply_ts_decomposition(self, ts): return ts @@ -153,7 +155,7 @@ class Diff(TimeSeriesFeatures): @property def label(self): - return 'diff' + return "diff" def apply_ts_decomposition(self, ts): return ts.diff().iloc[1:] @@ -166,8 +168,10 @@ class _STLFeatures(TimeSeriesFeatures): def apply_ts_decomposition(self, ts): if len(ts) < 2 * self.lags: - raise ValueError(f'{type(self).__name__} needs at least 2 * lags ' - f'({2 * self.lags}) periods; got {len(ts)}.') + raise ValueError( + f"{type(self).__name__} needs at least 2 * lags " + f"({2 * self.lags}) periods; got {len(ts)}." + ) result = STL(ts, period=self.lags).fit() return getattr(result, self.component) @@ -175,21 +179,21 @@ def apply_ts_decomposition(self, ts): class Seasonality(_STLFeatures): """Lags of the seasonal component of an STL decomposition (period = lags).""" - component = 'seasonal' + component = "seasonal" @property def label(self): - return 'seasonal' + return "seasonal" class Trend(_STLFeatures): """Lags of the trend component of an STL decomposition (period = lags).""" - component = 'trend' + component = "trend" @property def label(self): - return 'trend' + return "trend" class FeatureScaling(TransformerMixin, BaseEstimator): @@ -209,8 +213,7 @@ def fit(self, x, y=None): return self def __sklearn_is_fitted__(self): - return getattr(self, 'is_fitted_', False) + return getattr(self, "is_fitted_", False) def transform(self, x): - return pd.DataFrame(self.estimator.transform(x), - index=x.index, columns=x.columns) + return pd.DataFrame(self.estimator.transform(x), index=x.index, columns=x.columns) diff --git a/predspot/ml_modelling.py b/src/predspot/ml_modelling.py similarity index 79% rename from predspot/ml_modelling.py rename to src/predspot/ml_modelling.py index 738e567..3107ac4 100644 --- a/predspot/ml_modelling.py +++ b/src/predspot/ml_modelling.py @@ -7,7 +7,7 @@ attached to their ``(t, places)`` labels. """ -__author__ = 'Adelson Araujo' +__author__ = "Adelson Araujo" import logging @@ -22,7 +22,7 @@ idx = pd.IndexSlice -SCORERS = {'r2': r2_score, 'mse': mean_squared_error} +SCORERS = {"r2": r2_score, "mse": mean_squared_error} class FeatureSelection(TransformerMixin, BaseEstimator): @@ -42,7 +42,7 @@ def fit(self, x, y=None): return self def __sklearn_is_fitted__(self): - return getattr(self, 'is_fitted_', False) + return getattr(self, "is_fitted_", False) @property def support_(self): @@ -50,8 +50,9 @@ def support_(self): return self.estimator.support_ def transform(self, x): - return pd.DataFrame(self.estimator.transform(x), index=x.index, - columns=x.columns[self.estimator.support_]) + return pd.DataFrame( + self.estimator.transform(x), index=x.index, columns=x.columns[self.estimator.support_] + ) class Model(RegressorMixin, BaseEstimator): @@ -71,15 +72,14 @@ def fit(self, x, y=None): return self def __sklearn_is_fitted__(self): - return getattr(self, 'is_fitted_', False) + return getattr(self, "is_fitted_", False) @property def feature_importances_(self): return self.estimator.feature_importances_ def predict(self, x): - return pd.DataFrame(self.estimator.predict(x), index=x.index, - columns=['crime_density']) + return pd.DataFrame(self.estimator.predict(x), index=x.index, columns=["crime_density"]) class PredictionPipeline(RegressorMixin, BaseEstimator): @@ -140,7 +140,7 @@ def next_time(self): def _check_fitted(self): if self._X is None: - raise RuntimeError('This pipeline was not fitted yet.') + raise RuntimeError("This pipeline was not fitted yet.") @property def feature_importances(self): @@ -156,18 +156,19 @@ def feature_importances(self): pandas.DataFrame: Importance per feature, sorted descending. """ self._check_fitted() - steps = getattr(self.estimator, 'steps', [('model', self.estimator)]) + steps = getattr(self.estimator, "steps", [("model", self.estimator)]) model = steps[-1][1] try: importances = model.feature_importances_ except AttributeError as exc: - raise AttributeError('The estimator does not expose feature_importances_.') from exc + raise AttributeError("The estimator does not expose feature_importances_.") from exc columns = self._X.columns for _, step in steps[:-1]: - if hasattr(step, 'support_'): + if hasattr(step, "support_"): columns = columns[step.support_] - return (pd.DataFrame({'importance': importances}, index=columns) - .sort_values('importance', ascending=False)) + return pd.DataFrame({"importance": importances}, index=columns).sort_values( + "importance", ascending=False + ) def fit(self, dataset, y=None): """ @@ -180,17 +181,17 @@ def fit(self, dataset, y=None): Returns: PredictionPipeline: ``self``. """ - logger.debug('Fitting prediction pipeline') + logger.debug("Fitting prediction pipeline") self._dataset = dataset self._stseries = self.mapping.fit_transform(dataset.crimes) self._X = self.fextraction.fit_transform(self._stseries) - t0 = self._X.index.get_level_values('t').min() - tf = self._stseries.index.get_level_values('t').max() + t0 = self._X.index.get_level_values("t").min() + tf = self._stseries.index.get_level_values("t").max() X = self._X.loc[t0:tf].sample(frac=1, random_state=self.random_state) y = self._stseries.loc[X.index] self.estimator.fit(X, y) - self._t_plus_one = self._X.index.get_level_values('t').max() - logger.debug('Pipeline fitted on %d rows; next period is %s', len(X), self._t_plus_one) + self._t_plus_one = self._X.index.get_level_values("t").max() + logger.debug("Pipeline fitted on %d rows; next period is %s", len(X), self._t_plus_one) return self def predict(self): @@ -207,15 +208,15 @@ def predict(self): self._check_fitted() X = self._X.loc[[self._t_plus_one], :] y_pred = pd.DataFrame(self.estimator.predict(X), index=X.index) - y_pred.columns = ['crime_density'] - logger.debug('Predicted %d places for %s', len(y_pred), self._t_plus_one) - self._stseries = pd.concat([self._stseries, y_pred['crime_density']]).sort_index() - self._stseries.name = 'crime_density' + y_pred.columns = ["crime_density"] + logger.debug("Predicted %d places for %s", len(y_pred), self._t_plus_one) + self._stseries = pd.concat([self._stseries, y_pred["crime_density"]]).sort_index() + self._stseries.name = "crime_density" self._X = self.fextraction.transform(self._stseries) self._t_plus_one = self._t_plus_one + self._offset return y_pred - def evaluate(self, scoring='r2', cv=5): + def evaluate(self, scoring="r2", cv=5): """ Score the estimator with time series cross-validation. @@ -234,21 +235,25 @@ def evaluate(self, scoring='r2', cv=5): if scoring not in SCORERS: raise ValueError('invalid scoring. Try "r2" or "mse".') scorer = SCORERS[scoring] - timestamps = (self._X.index.get_level_values('t').unique() - .intersection(self._stseries.index.get_level_values('t').unique()) - .sort_values()) + timestamps = ( + self._X.index.get_level_values("t") + .unique() + .intersection(self._stseries.index.get_level_values("t").unique()) + .sort_values() + ) if not isinstance(cv, int) or cv >= len(timestamps): - raise ValueError('cv must be an integer lower than the number of periods.') + raise ValueError("cv must be an integer lower than the number of periods.") scores = [] for train_t, test_t in TimeSeriesSplit(cv).split(timestamps): - X_train = (self._X.loc[idx[timestamps[train_t], :], :] - .sample(frac=1, random_state=self.random_state)) + X_train = self._X.loc[idx[timestamps[train_t], :], :].sample( + frac=1, random_state=self.random_state + ) X_test = self._X.loc[idx[timestamps[test_t], :], :] y_train = self._stseries.loc[X_train.index] y_test = self._stseries.loc[X_test.index] self.estimator.fit(X_train, y_train) y_pred = self.estimator.predict(X_test) scores.append(scorer(y_test, y_pred)) - logger.debug('%s-fold CV %s scores: %s', cv, scoring, scores) + logger.debug("%s-fold CV %s scores: %s", cv, scoring, scores) self.fit(self._dataset) # back to normal return scores diff --git a/predspot/pipeline.py b/src/predspot/pipeline.py similarity index 63% rename from predspot/pipeline.py rename to src/predspot/pipeline.py index 589899d..6b51933 100644 --- a/predspot/pipeline.py +++ b/src/predspot/pipeline.py @@ -11,7 +11,7 @@ >>> predictions, pipeline = run_prediction_pipeline(crimes, study_area, grid_resolution=1) """ -__author__ = 'Adelson Araujo' +__author__ = "Adelson Araujo" import logging @@ -50,24 +50,31 @@ def generate_testdata(n_points, start_time, end_time, bounds=DEFAULT_BOUNDS, see """ rng = np.random.default_rng(seed) west, south, east, north = bounds - study_area = gpd.GeoDataFrame({'name': ['study_area']}, - geometry=[box(west, south, east, north)], crs='EPSG:4326') + study_area = gpd.GeoDataFrame( + {"name": ["study_area"]}, geometry=[box(west, south, east, north)], crs="EPSG:4326" + ) start, end = pd.Timestamp(start_time), pd.Timestamp(end_time) seconds = rng.integers(0, int((end - start).total_seconds()), n_points) - tags = rng.choice(['burglary', 'assault', 'drugs', 'homicide'], size=n_points, - p=np.array([1000, 100, 10, 1]) / 1111) - crimes = pd.DataFrame({ - 'tag': tags, - 't': start + pd.to_timedelta(seconds, unit='s'), - 'lon': rng.uniform(west, east, n_points), - 'lat': rng.uniform(south, north, n_points), - }) - logger.debug('Generated %d synthetic events', n_points) + tags = rng.choice( + ["burglary", "assault", "drugs", "homicide"], + size=n_points, + p=np.array([1000, 100, 10, 1]) / 1111, + ) + crimes = pd.DataFrame( + { + "tag": tags, + "t": start + pd.to_timedelta(seconds, unit="s"), + "lon": rng.uniform(west, east, n_points), + "lat": rng.uniform(south, north, n_points), + } + ) + logger.debug("Generated %d synthetic events", n_points) return crimes, study_area -def build_default_pipeline(study_area, tfreq='M', grid_resolution=1, lags=2, - bandwidth='silverman', random_state=None): +def build_default_pipeline( + study_area, tfreq="M", grid_resolution=1, lags=2, bandwidth="silverman", random_state=None +): """ Build the default Predspot pipeline: KDE mapping, seasonal/trend/diff features, quantile scaling, RFE feature selection and a random forest. @@ -86,25 +93,49 @@ def build_default_pipeline(study_area, tfreq='M', grid_resolution=1, lags=2, grid = crime_mapping.create_gridpoints(study_area, grid_resolution) return ml_modelling.PredictionPipeline( mapping=crime_mapping.KDE(tfreq=tfreq, grid=grid, bandwidth=bandwidth), - fextraction=PandasFeatureUnion([ - ('seasonal', feature_engineering.Seasonality(lags=lags, tfreq=tfreq)), - ('trend', feature_engineering.Trend(lags=lags, tfreq=tfreq)), - ('diff', feature_engineering.Diff(lags=lags, tfreq=tfreq)), - ]), - estimator=Pipeline([ - ('f_scaling', feature_engineering.FeatureScaling( - QuantileTransformer(n_quantiles=10, output_distribution='uniform'))), - ('f_selection', ml_modelling.FeatureSelection( - RFE(RandomForestRegressor(n_estimators=20, random_state=random_state)))), - ('model', ml_modelling.Model( - RandomForestRegressor(n_estimators=50, random_state=random_state))), - ]), + fextraction=PandasFeatureUnion( + [ + ("seasonal", feature_engineering.Seasonality(lags=lags, tfreq=tfreq)), + ("trend", feature_engineering.Trend(lags=lags, tfreq=tfreq)), + ("diff", feature_engineering.Diff(lags=lags, tfreq=tfreq)), + ] + ), + estimator=Pipeline( + [ + ( + "f_scaling", + feature_engineering.FeatureScaling( + QuantileTransformer(n_quantiles=10, output_distribution="uniform") + ), + ), + ( + "f_selection", + ml_modelling.FeatureSelection( + RFE(RandomForestRegressor(n_estimators=20, random_state=random_state)) + ), + ), + ( + "model", + ml_modelling.Model( + RandomForestRegressor(n_estimators=50, random_state=random_state) + ), + ), + ] + ), random_state=random_state, ) -def run_prediction_pipeline(crime_data, study_area, crime_tags=None, time_range=None, - tfreq='M', grid_resolution=1, lags=2, random_state=None): +def run_prediction_pipeline( + crime_data, + study_area, + crime_tags=None, + time_range=None, + tfreq="M", + grid_resolution=1, + lags=2, + random_state=None, +): """ Fit the default pipeline on crime data and forecast the next period. @@ -123,24 +154,29 @@ def run_prediction_pipeline(crime_data, study_area, crime_tags=None, time_range= tuple: ``(predictions, pipeline)`` โ€” the forecast for the next period and the fitted :class:`predspot.ml_modelling.PredictionPipeline`. """ - missing = [c for c in ('tag', 't', 'lat', 'lon') if c not in crime_data.columns] + missing = [c for c in ("tag", "t", "lat", "lon") if c not in crime_data.columns] if missing: - raise ValueError(f'Crime data must contain columns tag, t, lat, lon; missing {missing}') + raise ValueError(f"Crime data must contain columns tag, t, lat, lon; missing {missing}") if crime_tags: - crime_data = crime_data.loc[crime_data['tag'].isin(crime_tags)] + crime_data = crime_data.loc[crime_data["tag"].isin(crime_tags)] if time_range: - time_ix = pd.DatetimeIndex(pd.to_datetime(crime_data['t'])) + time_ix = pd.DatetimeIndex(pd.to_datetime(crime_data["t"])) crime_data = crime_data.iloc[time_ix.indexer_between_time(time_range[0], time_range[1])] dataset = dataset_preparation.Dataset(crimes=crime_data, study_area=study_area) - pipeline = build_default_pipeline(study_area, tfreq=tfreq, grid_resolution=grid_resolution, - lags=lags, random_state=random_state) + pipeline = build_default_pipeline( + study_area, + tfreq=tfreq, + grid_resolution=grid_resolution, + lags=lags, + random_state=random_state, + ) pipeline.fit(dataset) predictions = pipeline.predict() return predictions, pipeline -def evaluate_pipeline(pipeline, scoring='r2', cv=5): +def evaluate_pipeline(pipeline, scoring="r2", cv=5): """ Cross-validate a fitted pipeline; see :meth:`PredictionPipeline.evaluate`. @@ -153,5 +189,5 @@ def evaluate_pipeline(pipeline, scoring='r2', cv=5): list: One score per fold. """ scores = pipeline.evaluate(scoring=scoring, cv=cv) - logger.debug('Evaluation complete. Mean score: %.4f', np.mean(scores)) + logger.debug("Evaluation complete. Mean score: %.4f", np.mean(scores)) return scores diff --git a/predspot/utilities.py b/src/predspot/utilities.py similarity index 82% rename from predspot/utilities.py rename to src/predspot/utilities.py index 6045f68..1e67528 100644 --- a/predspot/utilities.py +++ b/src/predspot/utilities.py @@ -7,13 +7,12 @@ GeoJSON contour export for density maps. """ -__author__ = 'Adelson Araujo' +__author__ = "Adelson Araujo" import logging import numpy as np import pandas as pd -from geopandas import GeoDataFrame from sklearn.base import BaseEstimator, TransformerMixin logger = logging.getLogger(__name__) @@ -36,7 +35,7 @@ def __init__(self, transformer_list): def _iter(self): for name, transformer in self.transformer_list: - if transformer is None or transformer == 'drop': + if transformer is None or transformer == "drop": continue yield name, transformer @@ -46,8 +45,7 @@ def fit(self, X, y=None, **fit_params): return self def fit_transform(self, X, y=None, **fit_params): - outputs = [transformer.fit_transform(X, y, **fit_params) - for _, transformer in self._iter()] + outputs = [transformer.fit_transform(X, y, **fit_params) for _, transformer in self._iter()] return self.merge_dataframes_by_column(outputs) def transform(self, X): @@ -66,9 +64,9 @@ def merge_dataframes_by_column(outputs): pandas.DataFrame: The merged features without missing rows. """ if not outputs: - raise ValueError('PandasFeatureUnion has no transformers.') - logger.debug('Merging %d feature blocks', len(outputs)) - return pd.concat(outputs, axis='columns').dropna() + raise ValueError("PandasFeatureUnion has no transformers.") + logger.debug("Merging %d feature blocks", len(outputs)) + return pd.concat(outputs, axis="columns").dropna() def contour_geojson(y, bbox, resolution, cmin, cmax): @@ -94,14 +92,16 @@ def contour_geojson(y, bbox, resolution, cmin, cmax): try: import geojsoncontour except ImportError as exc: # pragma: no cover - optional dependency - raise ImportError('contour_geojson requires the optional dependency ' - '`geojsoncontour`: pip install predspot[contour]') from exc + raise ImportError( + "contour_geojson requires the optional dependency " + "`geojsoncontour`: pip install predspot[contour]" + ) from exc import matplotlib - matplotlib.use('Agg') + + matplotlib.use("Agg") import matplotlib.pyplot as plt - from predspot.crime_mapping import (KM_PER_DEG_LAT, KM_PER_DEG_LON, - _check_bbox, _wgs84_bounds) + from predspot.crime_mapping import KM_PER_DEG_LAT, KM_PER_DEG_LON, _check_bbox, _wgs84_bounds _check_bbox(bbox) b_w, b_s, b_e, b_n = _wgs84_bounds(bbox) @@ -113,8 +113,7 @@ def contour_geojson(y, bbox, resolution, cmin, cmax): Z = Z.reshape(lonv.shape) fig, axes = plt.subplots() - contourf = axes.contourf(lonv, latv, Z, levels=np.linspace(cmin, cmax, 25), - cmap='Spectral_r') + contourf = axes.contourf(lonv, latv, Z, levels=np.linspace(cmin, cmax, 25), cmap="Spectral_r") geojson = geojsoncontour.contourf_to_geojson(contourf=contourf, fill_opacity=0.5) plt.close(fig) return geojson diff --git a/tests/conftest.py b/tests/conftest.py index 0b5cafd..7079c3e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,13 +10,12 @@ BOUNDS = (-35.30, -5.90, -35.20, -5.80) -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def study_area(): - return gpd.GeoDataFrame({'name': ['test']}, - geometry=[box(*BOUNDS)], crs='EPSG:4326') + return gpd.GeoDataFrame({"name": ["test"]}, geometry=[box(*BOUNDS)], crs="EPSG:4326") -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def crimes(): """2 years of events: a uniform background plus one Gaussian hotspot.""" rng = np.random.default_rng(42) @@ -24,15 +23,18 @@ def crimes(): west, south, east, north = BOUNDS lon = np.concatenate([rng.uniform(west, east, n_bg), rng.normal(-35.23, 0.01, n_hot)]) lat = np.concatenate([rng.uniform(south, north, n_bg), rng.normal(-5.83, 0.01, n_hot)]) - start = pd.Timestamp('2019-01-01') + start = pd.Timestamp("2019-01-01") seconds = rng.integers(0, 730 * 24 * 3600, n_bg + n_hot) - return pd.DataFrame({ - 'tag': rng.choice(['burglary', 'assault'], n_bg + n_hot, p=[0.8, 0.2]), - 't': start + pd.to_timedelta(seconds, unit='s'), - 'lon': np.clip(lon, west, east), 'lat': np.clip(lat, south, north), - }) + return pd.DataFrame( + { + "tag": rng.choice(["burglary", "assault"], n_bg + n_hot, p=[0.8, 0.2]), + "t": start + pd.to_timedelta(seconds, unit="s"), + "lon": np.clip(lon, west, east), + "lat": np.clip(lat, south, north), + } + ) -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def dataset(crimes, study_area): return Dataset(crimes, study_area) diff --git a/tests/test_crime_mapping.py b/tests/test_crime_mapping.py index 257eda1..641de0b 100644 --- a/tests/test_crime_mapping.py +++ b/tests/test_crime_mapping.py @@ -1,3 +1,4 @@ +import geopandas as gpd import numpy as np import pandas as pd import pytest @@ -5,24 +6,26 @@ from predspot import crime_mapping as cm -@pytest.mark.parametrize('tfreq,expected', [('M', 'ME'), ('m', 'ME'), ('ME', 'ME'), - ('W', 'W'), ('D', 'D'), ('daily', 'D')]) +@pytest.mark.parametrize( + "tfreq,expected", + [("M", "ME"), ("m", "ME"), ("ME", "ME"), ("W", "W"), ("D", "D"), ("daily", "D")], +) def test_normalize_tfreq(tfreq, expected): assert cm.normalize_tfreq(tfreq) == expected def test_normalize_tfreq_invalid(): with pytest.raises(ValueError): - cm.normalize_tfreq('Y') + cm.normalize_tfreq("Y") def test_create_gridpoints(study_area): grid = cm.create_gridpoints(study_area, resolution=1) - assert grid.index.name == 'places' - assert {'lon', 'lat', 'geometry'} <= set(grid.columns) + assert grid.index.name == "places" + assert {"lon", "lat", "geometry"} <= set(grid.columns) # ~11 x 11 points for a ~10x10 km box at 1 km spacing assert 100 <= len(grid) <= 200 - assert grid.geometry.geom_type.eq('Point').all() + assert grid.geometry.geom_type.eq("Point").all() assert grid.crs == study_area.crs @@ -45,23 +48,23 @@ def test_create_gridpoints_errors(study_area): with pytest.raises(TypeError): cm.create_gridpoints(study_area.geometry, resolution=1) # a diamond: the corners of its bounding box fall outside the polygon - from shapely.geometry import Polygon import geopandas as gpd - sliver = gpd.GeoDataFrame(geometry=[Polygon([(-35.25, -5.90), (-35.20, -5.85), (-35.25, -5.80), (-35.30, -5.85)])], - crs='EPSG:4326') - with pytest.raises(ValueError, match='coarse'): + from shapely.geometry import Polygon + + diamond = Polygon([(-35.25, -5.90), (-35.20, -5.85), (-35.25, -5.80), (-35.30, -5.85)]) + sliver = gpd.GeoDataFrame(geometry=[diamond], crs="EPSG:4326") + with pytest.raises(ValueError, match="coarse"): cm.create_gridpoints(sliver, resolution=10000) def test_create_gridhexagonal(study_area): grid = cm.create_gridhexagonal(study_area, resolution=1) - assert grid.index.name == 'places' - assert grid.geometry.geom_type.eq('Polygon').all() - assert {'lon', 'lat'} <= set(grid.columns) + assert grid.index.name == "places" + assert grid.geometry.geom_type.eq("Polygon").all() + assert {"lon", "lat"} <= set(grid.columns) # every centroid must lie inside its own hexagon - inside = [geom.contains(pt) for geom, pt in - zip(grid.geometry, __import__('geopandas').points_from_xy(grid.lon, grid.lat))] - assert all(inside) + centroids = gpd.points_from_xy(grid.lon, grid.lat) + assert all(geom.contains(pt) for geom, pt in zip(grid.geometry, centroids, strict=True)) # equal-area hexagons: the study area (~100 km2) needs ~100-150 cells of 1 km2 assert 90 <= len(grid) <= 170 assert grid.geometry.union_all().covers(study_area.geometry.iloc[0]) @@ -69,91 +72,91 @@ def test_create_gridhexagonal(study_area): def test_create_gridsquares(study_area): grid = cm.create_gridsquares(study_area, resolution=1) - assert grid.index.name == 'places' - assert grid.geometry.geom_type.eq('Polygon').all() + assert grid.index.name == "places" + assert grid.geometry.geom_type.eq("Polygon").all() assert 100 <= len(grid) <= 150 assert grid.geometry.union_all().covers(study_area.geometry.iloc[0]) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def points_grid(study_area): return cm.create_gridpoints(study_area, resolution=1) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def hex_grid(study_area): return cm.create_gridhexagonal(study_area, resolution=1) def test_kde_transform(dataset, points_grid): - kde = cm.KDE(tfreq='M', grid=points_grid) + kde = cm.KDE(tfreq="M", grid=points_grid) st = kde.fit_transform(dataset.crimes) assert isinstance(st, pd.Series) - assert st.name == 'crime_density' - assert st.index.names == ['t', 'places'] - times = st.index.get_level_values('t').unique() + assert st.name == "crime_density" + assert st.index.names == ["t", "places"] + times = st.index.get_level_values("t").unique() assert len(times) == 24 # 2019-01 .. 2020-12 - assert (times == pd.date_range('2019-01-31', '2020-12-31', freq='ME')).all() + assert (times == pd.date_range("2019-01-31", "2020-12-31", freq="ME")).all() assert len(st) == 24 * len(points_grid) assert not st.isna().any() assert (st >= 0).all() assert kde.factor is not None and kde.factor > 0 # the hotspot around (-35.23, -5.83) must be denser than the far corner - month = st.xs(times[5], level='t') + month = st.xs(times[5], level="t") hot = ((points_grid.lon - (-35.23)).abs() < 0.005) & ((points_grid.lat - (-5.83)).abs() < 0.005) cold = (points_grid.lon < -35.29) & (points_grid.lat < -5.89) assert month[hot.values].mean() > month[cold.values].mean() def test_kde_bandwidth_options(dataset, points_grid): - fixed = cm.KDE(tfreq='M', grid=points_grid, bandwidth=0.3) + fixed = cm.KDE(tfreq="M", grid=points_grid, bandwidth=0.3) fixed.fit_transform(dataset.crimes) assert fixed.factor == 0.3 - scott = cm.KDE(tfreq='M', grid=points_grid, bandwidth='scott') + scott = cm.KDE(tfreq="M", grid=points_grid, bandwidth="scott") scott.fit_transform(dataset.crimes) assert scott.factor > 0 - auto = cm.KDE(tfreq='M', grid=points_grid, bandwidth='auto') - assert auto._bw_method == 'silverman' + auto = cm.KDE(tfreq="M", grid=points_grid, bandwidth="auto") + assert auto._bw_method == "silverman" with pytest.raises(ValueError): - cm.KDE(tfreq='M', grid=points_grid, bandwidth='gaussian') + cm.KDE(tfreq="M", grid=points_grid, bandwidth="gaussian") with pytest.raises(ValueError): - cm.KDE(tfreq='M', grid=points_grid, bandwidth=-1) + cm.KDE(tfreq="M", grid=points_grid, bandwidth=-1) def test_kde_start_end_time(dataset, points_grid): - kde = cm.KDE(tfreq='M', grid=points_grid, start_time='2018-06-01', end_time='2021-03-31') + kde = cm.KDE(tfreq="M", grid=points_grid, start_time="2018-06-01", end_time="2021-03-31") st = kde.fit_transform(dataset.crimes) - times = st.index.get_level_values('t').unique() - assert times.min() == pd.Timestamp('2018-06-30') - assert times.max() == pd.Timestamp('2021-03-31') - assert (st.xs(pd.Timestamp('2018-06-30'), level='t') == 0).all() + times = st.index.get_level_values("t").unique() + assert times.min() == pd.Timestamp("2018-06-30") + assert times.max() == pd.Timestamp("2021-03-31") + assert (st.xs(pd.Timestamp("2018-06-30"), level="t") == 0).all() def test_kde_weekly_and_daily(dataset, points_grid): - weekly = cm.KDE(tfreq='W', grid=points_grid).fit_transform(dataset.crimes) - assert 100 <= len(weekly.index.get_level_values('t').unique()) <= 106 - small = dataset.crimes[dataset.crimes['t'] < '2019-02-01'] - daily = cm.KDE(tfreq='D', grid=points_grid).fit_transform(small) - assert len(daily.index.get_level_values('t').unique()) == 31 + weekly = cm.KDE(tfreq="W", grid=points_grid).fit_transform(dataset.crimes) + assert 100 <= len(weekly.index.get_level_values("t").unique()) <= 106 + small = dataset.crimes[dataset.crimes["t"] < "2019-02-01"] + daily = cm.KDE(tfreq="D", grid=points_grid).fit_transform(small) + assert len(daily.index.get_level_values("t").unique()) == 31 def test_kde_few_points_gives_zeros(dataset, points_grid): two = dataset.crimes.iloc[:2] - st = cm.KDE(tfreq='M', grid=points_grid).fit_transform(two) + st = cm.KDE(tfreq="M", grid=points_grid).fit_transform(two) assert (st == 0).all() def test_kde_grid_validation(points_grid): - with pytest.raises(ValueError, match='lon'): - cm.KDE(tfreq='M', grid=points_grid.drop(columns=['lon'])) + with pytest.raises(ValueError, match="lon"): + cm.KDE(tfreq="M", grid=points_grid.drop(columns=["lon"])) def test_quadrat_count(dataset, hex_grid): - qc = cm.QuadratCount(tfreq='M', grid=hex_grid) + qc = cm.QuadratCount(tfreq="M", grid=hex_grid) st = qc.fit_transform(dataset.crimes) - assert st.name == 'crime_density' - assert st.index.names == ['t', 'places'] - assert len(st.index.get_level_values('t').unique()) == 24 + assert st.name == "crime_density" + assert st.index.names == ["t", "places"] + assert len(st.index.get_level_values("t").unique()) == 24 assert len(st) == 24 * len(hex_grid) assert (st >= 0).all() assert np.allclose(st, np.round(st)) @@ -163,10 +166,10 @@ def test_quadrat_count(dataset, hex_grid): def test_quadrat_count_squares(dataset, study_area): grid = cm.create_gridsquares(study_area, resolution=2) - st = cm.QuadratCount(tfreq='W', grid=grid).fit_transform(dataset.crimes) + st = cm.QuadratCount(tfreq="W", grid=grid).fit_transform(dataset.crimes) assert st.sum() == len(dataset.crimes) def test_quadrat_count_requires_polygons(points_grid): - with pytest.raises(ValueError, match='polygonal'): - cm.QuadratCount(tfreq='M', grid=points_grid) + with pytest.raises(ValueError, match="polygonal"): + cm.QuadratCount(tfreq="M", grid=points_grid) diff --git a/tests/test_dataset_preparation.py b/tests/test_dataset_preparation.py index 1e7505c..2deb72d 100644 --- a/tests/test_dataset_preparation.py +++ b/tests/test_dataset_preparation.py @@ -9,25 +9,25 @@ def test_dataset_builds_points_in_wgs84(crimes, study_area): ds = Dataset(crimes, study_area) assert isinstance(ds.crimes, gpd.GeoDataFrame) assert ds.crimes.crs.to_epsg() == 4326 - assert pd.api.types.is_datetime64_any_dtype(ds.crimes['t']) - assert ds.crimes.geometry.geom_type.eq('Point').all() - assert ds.shape == {'crimes': (len(crimes), 5), 'study_area': (1, 2)} - assert 'predspot.Dataset' in repr(ds) + assert pd.api.types.is_datetime64_any_dtype(ds.crimes["t"]) + assert ds.crimes.geometry.geom_type.eq("Point").all() + assert ds.shape == {"crimes": (len(crimes), 5), "study_area": (1, 2)} + assert "predspot.Dataset" in repr(ds) def test_dataset_does_not_mutate_input(crimes, study_area): before = crimes.copy() Dataset(crimes, study_area) pd.testing.assert_frame_equal(crimes, before) - assert 'geometry' not in crimes.columns + assert "geometry" not in crimes.columns def test_dataset_validation(crimes, study_area): with pytest.raises(TypeError): Dataset(crimes, study_area.geometry.iloc[0]) - with pytest.raises(ValueError, match='missing'): - Dataset(crimes.drop(columns=['lat']), study_area) - with pytest.raises(ValueError, match='CRS'): + with pytest.raises(ValueError, match="missing"): + Dataset(crimes.drop(columns=["lat"]), study_area) + with pytest.raises(ValueError, match="CRS"): Dataset(crimes, study_area.set_crs(None, allow_override=True)) @@ -41,6 +41,7 @@ def test_train_test_split(dataset): def test_plot(dataset): import matplotlib - matplotlib.use('Agg') + + matplotlib.use("Agg") ax = dataset.plot(crime_samples=50) assert ax is not None diff --git a/tests/test_feature_engineering.py b/tests/test_feature_engineering.py index a595e7e..b58f846 100644 --- a/tests/test_feature_engineering.py +++ b/tests/test_feature_engineering.py @@ -8,68 +8,81 @@ from predspot.utilities import PandasFeatureUnion -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def stseries(dataset, study_area): grid = cm.create_gridpoints(study_area, resolution=2) - return cm.KDE(tfreq='M', grid=grid).fit_transform(dataset.crimes) + return cm.KDE(tfreq="M", grid=grid).fit_transform(dataset.crimes) def _check_features(X, stseries, label, lags, first_time): - assert list(X.columns) == [f'{label}_{i}' for i in range(1, lags + 1)] - assert X.index.names == ['t', 'places'] - times = X.index.get_level_values('t').unique() - last_obs = stseries.index.get_level_values('t').max() + assert list(X.columns) == [f"{label}_{i}" for i in range(1, lags + 1)] + assert X.index.names == ["t", "places"] + times = X.index.get_level_values("t").unique() + last_obs = stseries.index.get_level_values("t").max() assert times.max() == last_obs + pd.offsets.MonthEnd(1) # next period row assert times.min() == first_time - n_places = stseries.index.get_level_values('places').nunique() + n_places = stseries.index.get_level_values("places").nunique() assert len(X) == len(times) * n_places assert not X.isna().any().any() def test_ar_features(stseries): - X = fe.AR(lags=3, tfreq='M').fit_transform(stseries) - _check_features(X, stseries, 'ar', 3, pd.Timestamp('2019-04-30')) + X = fe.AR(lags=3, tfreq="M").fit_transform(stseries) + _check_features(X, stseries, "ar", 3, pd.Timestamp("2019-04-30")) # ar_1 at t equals the series at t-1 - place = stseries.index.get_level_values('places')[0] - assert np.isclose(X.loc[(pd.Timestamp('2019-04-30'), place), 'ar_1'], - stseries.loc[(pd.Timestamp('2019-03-31'), place)]) + place = stseries.index.get_level_values("places")[0] + assert np.isclose( + X.loc[(pd.Timestamp("2019-04-30"), place), "ar_1"], + stseries.loc[(pd.Timestamp("2019-03-31"), place)], + ) def test_diff_features(stseries): - X = fe.Diff(lags=2, tfreq='M').fit_transform(stseries) - _check_features(X, stseries, 'diff', 2, pd.Timestamp('2019-04-30')) + X = fe.Diff(lags=2, tfreq="M").fit_transform(stseries) + _check_features(X, stseries, "diff", 2, pd.Timestamp("2019-04-30")) def test_seasonality_and_trend(stseries): - S = fe.Seasonality(lags=6, tfreq='M').fit_transform(stseries) - T = fe.Trend(lags=6, tfreq='M').fit_transform(stseries) - _check_features(S, stseries, 'seasonal', 6, pd.Timestamp('2019-07-31')) - _check_features(T, stseries, 'trend', 6, pd.Timestamp('2019-07-31')) + S = fe.Seasonality(lags=6, tfreq="M").fit_transform(stseries) + T = fe.Trend(lags=6, tfreq="M").fit_transform(stseries) + _check_features(S, stseries, "seasonal", 6, pd.Timestamp("2019-07-31")) + _check_features(T, stseries, "trend", 6, pd.Timestamp("2019-07-31")) assert not np.allclose(S.values, T.values) def test_tfreq_is_inferred_when_omitted(stseries): X = fe.AR(lags=2).fit_transform(stseries) - assert X.index.get_level_values('t').max() == pd.Timestamp('2021-01-31') + assert X.index.get_level_values("t").max() == pd.Timestamp("2021-01-31") def test_validation(stseries): with pytest.raises(ValueError): fe.AR(lags=1) - with pytest.raises(ValueError, match='lags'): - fe.AR(lags=30, tfreq='M').fit_transform(stseries) - with pytest.raises(ValueError, match='2 \\* lags'): - fe.Seasonality(lags=13, tfreq='M').fit_transform(stseries) + with pytest.raises(ValueError, match="lags"): + fe.AR(lags=30, tfreq="M").fit_transform(stseries) + with pytest.raises(ValueError, match="2 \\* lags"): + fe.Seasonality(lags=13, tfreq="M").fit_transform(stseries) def test_pandas_feature_union(stseries): - union = PandasFeatureUnion([('ar', fe.AR(lags=2, tfreq='M')), - ('seasonal', fe.Seasonality(lags=4, tfreq='M')), - ('skip', None)]) + union = PandasFeatureUnion( + [ + ("ar", fe.AR(lags=2, tfreq="M")), + ("seasonal", fe.Seasonality(lags=4, tfreq="M")), + ("skip", None), + ] + ) X = union.fit_transform(stseries) - assert list(X.columns) == ['ar_1', 'ar_2', 'seasonal_1', 'seasonal_2', 'seasonal_3', 'seasonal_4'] + assert list(X.columns) == [ + "ar_1", + "ar_2", + "seasonal_1", + "seasonal_2", + "seasonal_3", + "seasonal_4", + ] # rows are aligned on the intersection of the indexes (the STL warm-up wins) - assert X.index.get_level_values('t').min() == pd.Timestamp('2019-05-31') + assert X.index.get_level_values("t").min() == pd.Timestamp("2019-05-31") assert not X.isna().any().any() pd.testing.assert_frame_equal(union.transform(stseries), X) with pytest.raises(ValueError): @@ -77,7 +90,7 @@ def test_pandas_feature_union(stseries): def test_feature_scaling(stseries): - X = fe.AR(lags=2, tfreq='M').fit_transform(stseries) + X = fe.AR(lags=2, tfreq="M").fit_transform(stseries) scaled = fe.FeatureScaling(StandardScaler()).fit_transform(X) assert isinstance(scaled, pd.DataFrame) assert scaled.index.equals(X.index) and list(scaled.columns) == list(X.columns) diff --git a/tests/test_ml_modelling.py b/tests/test_ml_modelling.py index 7ea914e..0a1479f 100644 --- a/tests/test_ml_modelling.py +++ b/tests/test_ml_modelling.py @@ -15,22 +15,30 @@ def make_pipeline(grid, mapping_cls=cm.KDE, estimator=None): return ml.PredictionPipeline( - mapping=mapping_cls(tfreq='M', grid=grid), - fextraction=PandasFeatureUnion([ - ('seasonal', fe.Seasonality(lags=4, tfreq='M')), - ('trend', fe.Trend(lags=4, tfreq='M')), - ('diff', fe.Diff(lags=4, tfreq='M')), - ]), - estimator=estimator or Pipeline([ - ('f_scaling', fe.FeatureScaling(QuantileTransformer(n_quantiles=10))), - ('f_selection', ml.FeatureSelection(RFE(RandomForestRegressor(n_estimators=5, random_state=0)))), - ('model', ml.Model(RandomForestRegressor(n_estimators=10, random_state=0))), - ]), + mapping=mapping_cls(tfreq="M", grid=grid), + fextraction=PandasFeatureUnion( + [ + ("seasonal", fe.Seasonality(lags=4, tfreq="M")), + ("trend", fe.Trend(lags=4, tfreq="M")), + ("diff", fe.Diff(lags=4, tfreq="M")), + ] + ), + estimator=estimator + or Pipeline( + [ + ("f_scaling", fe.FeatureScaling(QuantileTransformer(n_quantiles=10))), + ( + "f_selection", + ml.FeatureSelection(RFE(RandomForestRegressor(n_estimators=5, random_state=0))), + ), + ("model", ml.Model(RandomForestRegressor(n_estimators=10, random_state=0))), + ] + ), random_state=0, ) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def fitted(dataset, study_area): grid = cm.create_gridpoints(study_area, resolution=2) return make_pipeline(grid).fit(dataset) @@ -38,37 +46,37 @@ def fitted(dataset, study_area): def test_fit_predict(fitted): n_places = len(fitted.grid) - assert fitted.next_time == pd.Timestamp('2021-01-31') + assert fitted.next_time == pd.Timestamp("2021-01-31") pred = fitted.predict() - assert list(pred.columns) == ['crime_density'] + assert list(pred.columns) == ["crime_density"] assert len(pred) == n_places - assert pred.index.get_level_values('t').unique().tolist() == [pd.Timestamp('2021-01-31')] - assert (pred['crime_density'] >= 0).all() + assert pred.index.get_level_values("t").unique().tolist() == [pd.Timestamp("2021-01-31")] + assert (pred["crime_density"] >= 0).all() # the forecast is appended to the series and the horizon moves forward - assert fitted.stseries.index.get_level_values('t').max() == pd.Timestamp('2021-01-31') - assert fitted.next_time == pd.Timestamp('2021-02-28') + assert fitted.stseries.index.get_level_values("t").max() == pd.Timestamp("2021-01-31") + assert fitted.next_time == pd.Timestamp("2021-02-28") pred2 = fitted.predict() - assert pred2.index.get_level_values('t').unique().tolist() == [pd.Timestamp('2021-02-28')] + assert pred2.index.get_level_values("t").unique().tolist() == [pd.Timestamp("2021-02-28")] def test_feature_importances(fitted): fi = fitted.feature_importances - assert list(fi.columns) == ['importance'] - assert np.isclose(fi['importance'].sum(), 1) + assert list(fi.columns) == ["importance"] + assert np.isclose(fi["importance"].sum(), 1) assert set(fi.index) <= set(fitted.features.columns) def test_evaluate(dataset, study_area): grid = cm.create_gridpoints(study_area, resolution=2) pipe = make_pipeline(grid).fit(dataset) - scores = pipe.evaluate('r2', cv=3) + scores = pipe.evaluate("r2", cv=3) assert len(scores) == 3 - mse = pipe.evaluate('mse', cv=3) + mse = pipe.evaluate("mse", cv=3) assert all(s >= 0 for s in mse) with pytest.raises(ValueError): - pipe.evaluate('mae') + pipe.evaluate("mae") with pytest.raises(ValueError): - pipe.evaluate('r2', cv=100) + pipe.evaluate("r2", cv=100) # evaluate refits on the full data: predictions still work afterwards assert len(pipe.predict()) == len(grid) @@ -79,7 +87,7 @@ def test_plain_sklearn_estimator(dataset, study_area): pred = pipe.predict() assert len(pred) == len(grid) with pytest.raises(AttributeError): - pipe.feature_importances + _ = pipe.feature_importances def test_quadrat_count_pipeline(dataset, study_area): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 83da5be..be7a53a 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -5,24 +5,25 @@ def test_generate_testdata(): - crimes, area = pipeline.generate_testdata(500, '2019-01-01', '2019-12-31', seed=1) - assert list(crimes.columns) == ['tag', 't', 'lon', 'lat'] + crimes, area = pipeline.generate_testdata(500, "2019-01-01", "2019-12-31", seed=1) + assert list(crimes.columns) == ["tag", "t", "lon", "lat"] assert len(crimes) == 500 assert area.crs.to_epsg() == 4326 - assert crimes['t'].between('2019-01-01', '2019-12-31').all() - again, _ = pipeline.generate_testdata(500, '2019-01-01', '2019-12-31', seed=1) + assert crimes["t"].between("2019-01-01", "2019-12-31").all() + again, _ = pipeline.generate_testdata(500, "2019-01-01", "2019-12-31", seed=1) pd.testing.assert_frame_equal(crimes, again) def test_run_prediction_pipeline(crimes, study_area): pred, pipe = pipeline.run_prediction_pipeline( - crimes, study_area, crime_tags=['burglary'], grid_resolution=2, random_state=0) + crimes, study_area, crime_tags=["burglary"], grid_resolution=2, random_state=0 + ) assert len(pred) == len(pipe.grid) - assert pred.index.get_level_values('t').unique().tolist() == [pd.Timestamp('2021-01-31')] - scores = pipeline.evaluate_pipeline(pipe, 'r2', cv=2) + assert pred.index.get_level_values("t").unique().tolist() == [pd.Timestamp("2021-01-31")] + scores = pipeline.evaluate_pipeline(pipe, "r2", cv=2) assert len(scores) == 2 def test_run_prediction_pipeline_validation(crimes, study_area): with pytest.raises(ValueError): - pipeline.run_prediction_pipeline(crimes.drop(columns=['tag']), study_area) + pipeline.run_prediction_pipeline(crimes.drop(columns=["tag"]), study_area) diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 88457e1..a4ba5fc 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -7,9 +7,9 @@ def test_contour_geojson(dataset, study_area): - pytest.importorskip('geojsoncontour') + pytest.importorskip("geojsoncontour") grid = cm.create_gridpoints(study_area, resolution=1) - st = cm.KDE(tfreq='M', grid=grid).fit_transform(dataset.crimes) - month = st.xs(st.index.get_level_values('t')[0], level='t') + st = cm.KDE(tfreq="M", grid=grid).fit_transform(dataset.crimes) + month = st.xs(st.index.get_level_values("t")[0], level="t") geojson = contour_geojson(month, study_area, 1, cmin=0, cmax=month.max()) - assert json.loads(geojson)['type'] == 'FeatureCollection' + assert json.loads(geojson)["type"] == "FeatureCollection"