Import logic from TaxData repo that creates and tests generation of growfactor.csv file

.gitignore

@@ -9,6 +9,8 @@
 tmd/storage/output/cached_files
 tmd/storage/output/tax_expenditures
 !tmd/storage/input/*.csv
+!tmd/growfactors/data/*.csv
+!tmd/growfactors/docs/*.csv
 !tmd/areas/targets/*.csv
 !tmd/areas/prepare/data/soi_cds/*.csv
 !tmd/areas/prepare/recipes/*.csv

Makefile

@@ -11,6 +11,7 @@ install:
 .PHONY=clean
 clean:
 	rm -f tmd/storage/output/tmd*
+	rm -f tmd/storage/output/growfactors.csv
 	rm -f tmd/storage/output/cached*
 	rm -f tmd/storage/output/preimpute_tmd.csv.gz
 	rm -f tmd/storage/output/make_data_*.log
@@ -39,8 +40,10 @@ tmd/storage/output/tmd_weights.csv.gz:
 	    | tee tmd/storage/output/make_data_weights.log
 
 tmd/storage/output/tmd_growfactors.csv:
+	python -u tmd/create_raw_growth_factors.py 2>&1 \
+	    | tee tmd/storage/output/make_data_raw_growfactors.log
 	python -u tmd/create_taxcalc_growth_factors.py 2>&1 \
-	    | tee tmd/storage/output/make_data_growfactors.log
+	    | tee tmd/storage/output/make_data_tmd_growfactors.log
 
 tmd/storage/output/cached_files:
 	python -u tmd/create_taxcalc_cached_files.py 2>&1 \
@@ -61,6 +64,7 @@ tmd_files: tmd/storage/output/tmd.csv.gz \
 test: tmd_files
 	pytest . -v -n4 \
 	    --ignore=tests/national_targets_pipeline \
+	    --ignore=tests/growfactors_pipeline \
 	    --ignore=tests/test_fingerprint.py \
 	    --ignore=tests/test_state_weight_results.py \
 	    --ignore=tests/test_cd_crosswalk.py \
@@ -69,6 +73,8 @@ test: tmd_files
 
 .PHONY=data
 data: install clean format lint tmd_files test warnings
+	diff tmd/storage/output/growfactors.csv tmd/storage/output/old_growfactors.csv
+	diff tmd/storage/output/tmd_growfactors.csv tmd/storage/output/old_tmd_growfactors.csv
 
 .PHONY=warnings
 warnings:

setup.py

@@ -24,4 +24,13 @@
         "pytest",
         "pytest-xdist",
     ],
+    extras_require={
+        # Needed only by the annual growth-factors source-data refresh
+        # scripts in tmd/growfactors/update, which are never run by
+        # `make data`.  See tmd/growfactors/docs/annual_update.md.
+        "growfactors-update": [
+            "requests_html",
+            "jinja2",
+        ],
+    },
 )

tests/growfactors_pipeline/test_growfactors_pipeline.py

@@ -0,0 +1,69 @@
+"""
+Reproducibility tests for the growth factors pipeline.
+
+Each test re-runs a pipeline stage into a temporary directory and
+compares the result with the committed artifact, which catches source
+data in tmd/growfactors/data being edited without the downstream
+artifacts being regenerated.
+
+These tests are excluded from the default `make test` run (they rebuild
+both pipeline stages); run them with:
+    make -C tmd/growfactors test
+"""
+
+import pandas as pd
+
+from tmd.growfactors import DATA_FOLDER
+from tmd.growfactors import build_growfactors, build_stage1_factors
+
+STAGE1_FILE = DATA_FOLDER / "Stage_I_factors.csv"
+
+
+def read_text(path):
+    """
+    Return contents of path with any CRLF line endings normalized, so
+    that the historical line endings of a committed file do not cause a
+    spurious mismatch.
+    """
+    return path.read_text().replace("\r\n", "\n")
+
+
+def test_stage1_factors_reproducible(tmp_path, monkeypatch):
+    """
+    Check that build_stage1_factors reproduces the committed
+    data/Stage_I_factors.csv from the committed source data.
+    """
+    rebuilt = tmp_path / "Stage_I_factors.csv"
+    monkeypatch.setattr(build_stage1_factors, "OUTFILE", rebuilt)
+    build_stage1_factors.create_stage1_factors()
+    assert read_text(rebuilt) == read_text(STAGE1_FILE)
+
+
+def test_growfactors_reproducible(tmp_path, monkeypatch):
+    """
+    Check that build_growfactors reproduces the committed growfactors
+    file for the current data vintage.
+    """
+    vintage = build_growfactors.DEFAULT_VINTAGE
+    committed = build_growfactors.growfactors_path(vintage)
+    rebuilt = tmp_path / committed.name
+    monkeypatch.setattr(
+        build_growfactors, "growfactors_path", lambda _vintage: rebuilt
+    )
+    build_growfactors.create_growfactors_file(vintage)
+    assert read_text(rebuilt) == read_text(committed)
+
+
+def test_growfactors_have_no_missing_values():
+    """
+    Check that no growth factor is missing in any year covered by the
+    Stage_I_factors.csv file.  Benefit growth rates stop several years
+    before the last stage 1 year, so the benefit factors must be padded
+    rather than left undefined.
+    """
+    vintage = build_growfactors.DEFAULT_VINTAGE
+    committed = build_growfactors.growfactors_path(vintage)
+    gfdf = pd.read_csv(committed, index_col="YEAR")
+    stage1 = pd.read_csv(STAGE1_FILE, index_col="YEAR")
+    assert gfdf.index.max() == stage1.index.max()
+    assert gfdf.isnull().sum().sum() == 0

tests/test_growfactors.py

@@ -0,0 +1,47 @@
+"""
+Test contents of the raw growth factors file that is read by the
+tmd/create_raw_growth_factors.py script during `make data`.
+
+Ported from the TaxData repository (tests/test_growfactors.py).
+"""
+
+import pandas as pd
+import pytest
+
+from tmd.create_raw_growth_factors import INPUT_FILE
+from tmd.imputation_assumptions import TAXYEAR
+from tmd.storage import STORAGE_FOLDER
+
+FIRST_TAXCALC_POLICY_YEAR = 2013
+MIN_VALUE = 0.15
+MAX_VALUE = 8.70
+
+
+@pytest.fixture(scope="module", name="growfactors")
+def fixture_growfactors():
+    """
+    Return DataFrame containing the raw growth factors used for TAXYEAR.
+    """
+    path = STORAGE_FOLDER / "input" / INPUT_FILE[TAXYEAR]
+    return pd.read_csv(path, index_col="YEAR")
+
+
+def test_growfactor_start_year(growfactors):
+    """
+    Check that the growth factors file can support Tax-Calculator Policy
+    needs, which begin in FIRST_TAXCALC_POLICY_YEAR.
+    """
+    assert growfactors.index.min() <= FIRST_TAXCALC_POLICY_YEAR
+
+
+def test_growfactor_values(growfactors):
+    """
+    Check that each growth factor value is in a plausible min,max range,
+    that every first-year value is one, and that no values are missing.
+    """
+    first_year = growfactors.index.min()
+    for fname in growfactors:
+        assert growfactors[fname][first_year] == 1.0
+        assert growfactors[fname].min() >= MIN_VALUE
+        assert growfactors[fname].max() <= MAX_VALUE
+    assert growfactors.isnull().sum().sum() == 0

tmd/create_raw_growth_factors.py

@@ -0,0 +1,30 @@
+"""
+Construct raw growfactors.csv, a Tax-Calculator-style GrowFactors file.
+"""
+
+import pandas as pd
+from tmd.storage import STORAGE_FOLDER
+from tmd.imputation_assumptions import TAXYEAR
+
+INPUT_FILE = {
+    2021: "taxdata25_growfactors.csv",
+    2022: "taxdata26_growfactors.csv",
+}
+INFILE = STORAGE_FOLDER / "input" / INPUT_FILE[TAXYEAR]
+OUTFILE = STORAGE_FOLDER / "output" / "growfactors.csv"
+
+
+def create_raw_factors_file():
+    """
+    Create Tax-Calculator-style raw growth factors file.
+    """
+    # read PUF-factors from INFILE
+    gfdf = pd.read_csv(INFILE)
+
+    # write gfdf to CSV-formatted file
+    gfdf.YEAR = gfdf.YEAR.astype(int)
+    gfdf.to_csv(OUTFILE, index=False, float_format="%.6f")
+
+
+if __name__ == "__main__":
+    create_raw_factors_file()

tmd/create_taxcalc_growth_factors.py

@@ -19,11 +19,7 @@
 ASOCSEC_INDEX = 14
 AUCOMP_INDEX = 15
 
-INPUT_FILE = {
-    2021: "taxdata25_growfactors.csv",
-    2022: "taxdata26_growfactors.csv",
-}
-INFILE = STORAGE_FOLDER / "input" / INPUT_FILE[TAXYEAR]
+INFILE = STORAGE_FOLDER / "output" / "growfactors.csv"
 OUTFILE = STORAGE_FOLDER / "output" / "tmd_growfactors.csv"
 
 

tmd/growfactors/Makefile

@@ -0,0 +1,71 @@
+# Growth factors pipeline: Census/CBO/IRS/SOI source data -> growfactors CSV
+#
+# This pipeline is pre-processing that runs *before* `make data`.  It
+# produces committed artifacts (data/Stage_I_factors.csv and
+# tmd/storage/input/taxdata<VINTAGE>_growfactors.csv) that `make data`
+# then consumes via tmd/create_raw_growth_factors.py.  Most users never
+# need to run it; re-run it only when refreshing the source data for a
+# new vintage.
+#
+# Usage from repo root:
+#  make -C tmd/growfactors stage1  # stage 1: source data -> Stage_I_factors.csv
+#  make -C tmd/growfactors factors # stages 1+2: -> storage/input/*_growfactors.csv
+#  make -C tmd/growfactors test    # pipeline tests only
+#  make -C tmd/growfactors all     # all stages + pipeline tests
+#
+# The update-* targets below FETCH FROM LIVE WEBSITES (CBO, BLS, SSA,
+# IRS).  They are never invoked by `all`, by `factors`, or by `make
+# data`.  Run them by hand once a year, inspect the resulting diff in
+# data/, and commit it as its own pull request.
+# See the docs/annual_update.md file for details.
+
+ROOT := $(realpath $(CURDIR)/../..)
+INPUT := $(ROOT)/tmd/storage/input
+
+# Vintage of the growfactors file this pipeline currently produces.
+VINTAGE ?= 26
+GROWFACTORS := $(INPUT)/taxdata$(VINTAGE)_growfactors.csv
+
+STAGE1_DEPS := build_stage1_factors.py \
+               data/CBO_baseline.csv \
+               data/IRS_return_projection.csv \
+               data/NC-EST2014-AGESEX-RES.csv \
+               data/NP2014_D1.csv \
+               data/SOI_estimates.csv \
+               data/US-EST00INT-ALLDATA.csv \
+               data/benefitprograms.csv
+
+.PHONY: all stage1 factors test clean update-cbo update-soi
+
+data/Stage_I_factors.csv: $(STAGE1_DEPS)
+	python -m tmd.growfactors.build_stage1_factors
+
+stage1: data/Stage_I_factors.csv
+
+$(GROWFACTORS): build_growfactors.py \
+                data/Stage_I_factors.csv \
+                data/benefit_growth_rates.csv
+	python -m tmd.growfactors.build_growfactors --vintage $(VINTAGE)
+
+factors: $(GROWFACTORS)
+
+test:
+	python -m pytest $(ROOT)/tests/growfactors_pipeline -v
+	python -m pytest $(ROOT)/tests/test_growfactors.py -v
+
+all: factors test
+
+# Removes only regenerable pipeline artifacts, never data/ source files.
+clean:
+	rm -f data/Stage_I_factors.csv
+
+# ---- annual source-data refresh: NETWORK ACCESS, run by hand only ----
+# Requires the optional extras:  pip install -e ".[growfactors-update]"
+
+update-cbo:
+	python -m tmd.growfactors.update.update_cbo
+
+update-soi:
+	@test -n "$(YEAR)" || { echo "usage: make update-soi YEAR=2019 SOIPATH=/path/to/soi/files"; exit 1; }
+	@test -n "$(SOIPATH)" || { echo "usage: make update-soi YEAR=2019 SOIPATH=/path/to/soi/files"; exit 1; }
+	python -m tmd.growfactors.update.update_soi $(YEAR) $(SOIPATH)