diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..dc026ef --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +version: 2 + +updates: + # The package pins no runtime dependencies beyond numpy/scipy, but the test + # and CI toolchain moves. Weekly PRs keep the reproduction honest against + # current versions rather than the ones that happened to be installed on the + # day of release. + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + day: monday + open-pull-requests-limit: 3 + commit-message: + prefix: deps + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: monday + open-pull-requests-limit: 3 + commit-message: + prefix: ci diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce15c7f..662695f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,13 @@ on: push: branches: [master] pull_request: + # A package whose claim is reproducibility should keep proving it. The + # weekly run re-asserts the paper's numbers against whatever numpy, scipy + # and Python resolve to that week, so drift surfaces here rather than in a + # reader's environment. + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 0000000..ef5cb51 --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,34 @@ +name: Draft PDF + +# Compiles paper.md with the Open Journals (inara) toolchain and uploads the +# rendered paper.pdf as a build artifact. This is the JOSS-documented preview +# workflow: it proves the paper compiles before submission. It does not contact +# JOSS and does not submit anything. + +on: + push: + paths: + - 'paper.md' + - 'paper.bib' + - '.github/workflows/draft-pdf.yml' + workflow_dispatch: + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + paper-path: paper.md + + - name: Upload rendered paper + uses: actions/upload-artifact@v4 + with: + name: paper + path: paper.pdf diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c7f2ddb..5ff2845 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -47,7 +47,7 @@ jobs: permissions: id-token: write # required for Trusted Publishing steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: dist path: dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5be9b7e..bea616b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,38 @@ # Changelog +## Unreleased + +Screen-result export. **No change to package behavior:** +`tests/test_reproduces_paper.py` is untouched and the demo reproduces the same +numbers as 0.1.1. + +- `--export PATH` writes a `pit-screen-results` v1.0 record — per-signal summary + statistics and the screen settings that produced the verdicts. Fully offline + (local file, no network call) and free of clocks, so the same screen exports + byte-identical bytes. +- `docs/results-schema.md`: the schema as a standalone versioned interchange + spec, free for other tools to adopt. +- New module `pit_release_gate.results`: `summarize_signal`, `screen_config`, + `build_results`, `validate_results`, `write_results`. Totals are derived, so + they cannot disagree with the per-signal rows. +- Still no telemetry of any kind, now enforced structurally rather than by + policy: no module in the package imports a transport, and a test asserts that + over every module — no background thread, no `atexit` hook, nothing to opt + out of. + +## 0.1.1 — 2026-08-16 + +Documentation and submission materials only. **No change to package behavior:** +the test suite, including `tests/test_reproduces_paper.py`, is untouched and the +demo reproduces the same numbers as 0.1.0. + +- `paper.md` and `paper.bib`: software paper prepared for submission to the + Journal of Open Source Software. +- README: added *Statement of need*, *API overview*, and *Community guidelines* + sections. +- CI: `draft-pdf.yml` builds a preview PDF of `paper.md` with the Open Journals + toolchain and uploads it as a build artifact. + ## 0.1.0 — 2026-08-15 First public release. diff --git a/README.md b/README.md index 0bd57e9..565e9ce 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,10 @@ [![CI](https://github.com/MaxWellApexLab/pit-release-gate/actions/workflows/ci.yml/badge.svg)](https://github.com/MaxWellApexLab/pit-release-gate/actions/workflows/ci.yml) [![codecov](https://codecov.io/gh/MaxWellApexLab/pit-release-gate/branch/master/graph/badge.svg)](https://codecov.io/gh/MaxWellApexLab/pit-release-gate) [![PyPI](https://img.shields.io/pypi/v/pit-release-gate)](https://pypi.org/project/pit-release-gate/) +[![Downloads](https://img.shields.io/pypi/dm/pit-release-gate)](https://pypi.org/project/pit-release-gate/) [![Python](https://img.shields.io/pypi/pyversions/pit-release-gate)](https://pypi.org/project/pit-release-gate/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) +[![PIT Hygiene](https://img.shields.io/badge/PIT%20Hygiene-pledged-2ea44f)](https://github.com/MaxWellApexLab/pit-hygiene) Completeness-aware release control for staggered-arrival cross-sectional data. @@ -20,6 +22,24 @@ correlation fitted honestly on prior *completed* periods — and grades the requ completeness per signal, so benign signals release early and susceptible signals are withheld until enough of the cross-section has arrived to suppress the bias. +## Statement of need + +Point-in-time discipline in ML pipelines currently rests on tooling that answers one +question: *was this value readable at time t?* Feature-store as-of joins, bitemporal +and vintage-aware storage, and purged or embargoed cross-validation all enforce +read-time correctness, and they do it well. + +None of them answers a second question: given that every value read was legitimately +readable, was the *set* of entities that had reported by t a selected sample? An +as-of join over an incomplete cross-section is a correct join over a biased sample. +The two failures need different remedies — the first is fixed by timestamp hygiene, +the second only by waiting or by an explicit correction. Researchers building +cross-sectional signals on staggered-arrival panels have had no routine, per-signal +screen for the second. `pit-release-gate` is that screen, plus the release controller +that acts on it: one `fit_trailing` call per signal, so reporting a susceptibility +estimate alongside a released signal costs about as much as reporting a standard +error. + ## Install ```bash @@ -56,6 +76,21 @@ values, and per-entity filing-arrival times, then call `ReleaseController.decide(store, t)` at each evaluation time — it returns `WITHHOLD`, `REWEIGHT_RELEASE`, or `RELEASE` plus the released values. +## API overview + +Five public components, all importable from the top-level `pit_release_gate` package: + +| component | what it does | +|---|---| +| [`AsOfDataStore`](src/pit_release_gate/store.py) | Holds one period's as-filed records for a cross-sectional group: design matrix, signal values, and a filing-arrival time per entity. | +| [`CompletenessMonitor`](src/pit_release_gate/monitor.py) | Reports the arrived fraction at an evaluation time, plus a composition-shift gauge for the arrived subset. | +| [`SusceptibilityGate`](src/pit_release_gate/gate.py) | Estimates ρ̂, the partial correlation between filing latency and the complete-cross-section residual given observables. `fit_trailing` enforces the honest-estimation contract: prior *completed* periods only. | +| [`PropensityReweighter`](src/pit_release_gate/reweight.py) | Inverse-filing-propensity weights. Included to make a negative result executable: reweighting on observables corrects composition, but cannot remove selection on the disturbance. | +| [`ReleaseController`](src/pit_release_gate/controller.py) | Maps \|ρ̂\| to a required completeness `φ_req = min(1, φ_min + κ·\|ρ̂\|)` and returns `WITHHOLD` / `REWEIGHT_RELEASE` / `RELEASE` at each evaluation time. | + +`make_group`, `run_demo` and `demo` ([`simulate.py`](src/pit_release_gate/simulate.py)) +generate and run the known-ground-truth worked example described below. + ## The known-ground-truth demo The package ships a self-contained worked example with a *planted* leakage strength, @@ -80,6 +115,44 @@ A sensitivity sweep of the policy slope κ shows the timeliness–bias dial: κ = 2.0 → 100% (bias exactly 0). The demo is deterministic (fixed seed), and `tests/test_reproduces_paper.py` asserts these numbers. +## Export your screen result + +```bash +pit-release-gate --export results.json +``` + +writes a `pit-screen-results` v1.0 record: per screened signal, how many periods +were screened, how many the measure flagged, mean and max ρ̂, the required +completeness that was assigned, and the verdict (`benign` / `susceptible`), plus +the five settings that produced those verdicts. It is the file a *screened with* +badge should point at. The format is a standalone versioned interchange spec — +[`docs/results-schema.md`](docs/results-schema.md) — that any other tool is free +to emit or consume. + +**`--export` is fully offline**: it writes a local file and makes no network +call. Records carry summary statistics only — never input rows, file paths, +usernames, hostnames, or any environment detail beyond the tool version — and no +clock is read while building one, so the same screen always exports byte-identical +bytes. + +Where the record goes afterwards is entirely your business — commit it next to +your badge, publish it, or keep it. This tool does not send it anywhere. + +**No telemetry — structurally, not merely by default.** No module in this +package imports a transport at all, and a test walks every module in the package +and fails the suite if one ever does. There is no background thread, no `atexit` +hook, no anonymous usage counter, and nothing to opt out of. + +Programmatic use, for screens on your own data: + +```python +from pit_release_gate import build_results, screen_config, summarize_signal, validate_results + +sig = summarize_signal("accruals", rhos=[...], phi_reqs=[...], rho_threshold=0.10) +record = build_results([sig], screen_config(0.10, 0.35, 1.0, trailing_k=8, min_entities=6)) +assert validate_results(record) == [] +``` + ## Papers The method and its evaluation are developed in three public papers: @@ -95,6 +168,31 @@ The method and its evaluation are developed in three public papers: This package is the reference implementation of paper 3's release controller; its demo reproduces the paper's controlled experiment. +## Badge + +If you have run the susceptibility screen on your own data — whatever the result — you +are welcome to say so: + +```markdown +[![screened with pit-release-gate](https://img.shields.io/badge/screened%20with-pit--release--gate-blue)](https://github.com/MaxWellApexLab/pit-release-gate) +``` + +The badge reads **screened with**, not *passed* — it states that the screen was run, the +same way a formatter badge states that the formatter was run. A benign result and a +susceptible result are equally worth badging; the second one arguably more, because it +means the screen found something and your pipeline now waits for it. + +**Make it point at something.** A badge is worth reading only if there is evidence behind +it. Commit your screen output — `pit-release-gate --export results.json` produces exactly +that file: which signals came out benign, which came out susceptible, and the required +completeness each was assigned — and link the badge at it rather +than at this repo. A worked example is the OSAP screen in the +[PIT audit registry](https://github.com/MaxWellApexLab/pit-audit-registry/blob/main/audits/2026-08_osap/report.md). + +**Related:** the [PIT Hygiene pledge](https://github.com/MaxWellApexLab/pit-hygiene) is a +broader, tool-neutral statement about how a staggered-arrival pipeline is built; this badge +is the narrower statement that this particular screen was run. + ## Cite this See [`CITATION.cff`](CITATION.cff). If you use this software, please cite paper 3: @@ -110,6 +208,15 @@ See [`CITATION.cff`](CITATION.cff). If you use this software, please cite paper } ``` +## Community guidelines + +- **Report a bug or request a feature:** open an issue at + [github.com/MaxWellApexLab/pit-release-gate/issues](https://github.com/MaxWellApexLab/pit-release-gate/issues). +- **Contribute:** see [CONTRIBUTING.md](CONTRIBUTING.md) for the development setup, + test requirements, and pull-request process. +- **Get help:** open an issue with a minimal reproducing example, or email + maxwellapexlab@proton.me. + ## License MIT — see [LICENSE](LICENSE). diff --git a/docs/results-schema.md b/docs/results-schema.md new file mode 100644 index 0000000..e2fb704 --- /dev/null +++ b/docs/results-schema.md @@ -0,0 +1,214 @@ +# `pit-screen-results` — screen-result interchange format + +**Schema name:** `pit-screen-results` +**Version:** `1.0` +**Media type:** `application/json` +**Reference implementation:** [`src/pit_release_gate/results.py`](../src/pit_release_gate/results.py) +**Status:** stable + +A `pit-screen-results` record states what an incomplete-cross-section +susceptibility screen found: for each signal that was screened, how many periods +it was screened over, how susceptible it looked, how much completeness the +release controller therefore required, and whether the verdict was **benign** or +**susceptible**. It is the evidence a *screened with* badge should point at. + +The format is deliberately small. It is not a report format, not a results +database, and not a telemetry envelope. + +## Design rules + +These three rules are what make the record safe to commit to a public +repository or hand to a third party, and they are binding on any producer: + +1. **Summary statistics only.** A record carries per-signal aggregates. + It must never carry input rows, entity identifiers, residuals, file paths, + usernames, hostnames, or any environment detail beyond the producing tool's + name and version. +2. **Offline by construction.** Writing a record is local file I/O. In the + reference implementation the module that builds and writes records imports + no transport machinery at all; sending one is a separate, explicitly + requested act (see [Submission envelope](#submission-envelope-non-normative)). +3. **No clocks.** A producer must not read the system clock while building a + record: two runs of the same screen must produce byte-identical bytes. If a + date belongs in the record, the caller passes it in (`date`, below). + +## Top-level fields + +| field | type | required | meaning | +|---|---|---|---| +| `schema` | string | yes | Always `"pit-screen-results"`. Identifies the format, not the producer. | +| `schema_version` | string | yes | `"1.0"` for this document. | +| `tool` | string | yes | Name of the producing tool, e.g. `"pit-release-gate"`. Any tool may emit this format under its own name. | +| `tool_version` | string | yes | Version of the producing tool. The only environment detail permitted anywhere in the record. | +| `config` | object | yes | The screen settings that determine the verdicts. See below. | +| `signals` | array of objects | yes | One entry per screened signal, non-empty. See below. | +| `totals` | object | yes | Aggregates over `signals`, derived — never independently asserted. | +| `date` | string | no | Caller-supplied date (ISO 8601 `YYYY-MM-DD` recommended). Absent unless the caller passed one; producers must not fill it in from a clock. | + +## `config` + +The five settings a reader needs in order to interpret a verdict. All are +required. + +| field | type | meaning | +|---|---|---| +| `rho_threshold` | number | A signal is susceptible when \|ρ̂\| exceeds this. | +| `phi_min` | number | Completeness floor: the earliest completeness at which anything is released. | +| `kappa` | number | Slope of the graded requirement `phi_req = min(1, phi_min + kappa·\|ρ̂\|)`. | +| `trailing_k` | integer | Number of prior **completed** periods the susceptibility estimate was fitted on. | +| `min_entities` | integer | Minimum arrived entity count below which nothing is released. | + +## `signals[]` + +| field | type | meaning | +|---|---|---| +| `name` | string | Caller-chosen signal name. Must not encode a path, a file name, or an identity. | +| `periods_screened` | integer ≥ 0 | Number of periods the signal was screened over. | +| `periods_flagged` | integer | Of those, how many had a *per-period* \|ρ̂\| above `rho_threshold`. Must be ≤ `periods_screened`. | +| `mean_rho` | number | Mean of the per-period susceptibility estimates (signed). | +| `max_abs_rho` | number | Largest per-period \|ρ̂\| observed. | +| `mean_phi_req` | number | Mean required completeness the controller assigned across those periods. | +| `verdict` | string | `"benign"` or `"susceptible"` — no other value is valid. | + +**`periods_flagged` is a noise gauge, not the verdict.** A single period's ρ̂ is +a small-sample estimate and will cross the threshold now and then on a perfectly +benign signal; that is exactly why a screen pools over `trailing_k` completed +periods before deciding. In the worked example below, `clean` has 3 of 5 periods +flagged and is still — correctly — `benign`: its pooled estimate is far under the +threshold, while `max_abs_rho = 0.164` records how noisy a single period was. +Read `verdict` for the decision, `periods_flagged` and `max_abs_rho` for how +stable that decision was. + +## `totals` + +Derived from `signals`; a producer computes them rather than accepting them, and +a consumer may treat a mismatch as a corrupt record. + +| field | type | meaning | +|---|---|---| +| `signal_cycles` | integer | Sum of `periods_screened` over all signals — the total screening work the record represents. | +| `signals_benign` | integer | Number of signals with `verdict == "benign"`. | +| `signals_susceptible` | integer | Number with `verdict == "susceptible"`. | + +## Validity + +A record is valid when all of the following hold. The reference implementation +is `validate_results(obj) -> list[str]`, which returns one human-readable string +per problem and an empty list for a valid record. + +1. The record is a JSON object with `schema == "pit-screen-results"`. +2. `schema_version` is present and known to the reader (`"1.0"`). +3. Every required top-level, `config`, `signals[]`, and `totals` field is present. +4. Every `verdict` is `"benign"` or `"susceptible"`. +5. `totals.signal_cycles` equals the sum of `periods_screened` over `signals`, + and the two verdict counts equal the corresponding counts in `signals`. +6. For every signal, `0 <= periods_flagged <= periods_screened`. + +Validation is a check on the record, not on the science: a record can be +perfectly valid and report a thoroughly susceptible pipeline. That is the point. + +## Worked example + +Produced by `pit-release-gate --train 3 --eval 5 --export results.json` +(reduced settings, so the file fits here): + +```json +{ + "schema": "pit-screen-results", + "schema_version": "1.0", + "tool": "pit-release-gate", + "tool_version": "0.1.1", + "config": { + "rho_threshold": 0.1, + "phi_min": 0.35, + "kappa": 1.0, + "trailing_k": 3, + "min_entities": 6 + }, + "signals": [ + { + "name": "clean", + "periods_screened": 5, + "periods_flagged": 3, + "mean_rho": -0.018502844357097172, + "max_abs_rho": 0.16437940321670516, + "mean_phi_req": 0.40870844204483336, + "verdict": "benign" + }, + { + "name": "composition", + "periods_screened": 5, + "periods_flagged": 1, + "mean_rho": 0.054842113923429116, + "max_abs_rho": 0.1916906690936213, + "mean_phi_req": 0.3622517271310478, + "verdict": "benign" + }, + { + "name": "mild_leak", + "periods_screened": 5, + "periods_flagged": 5, + "mean_rho": -0.5050242025985237, + "max_abs_rho": 0.6073065398308837, + "mean_phi_req": 0.8244168498651367, + "verdict": "susceptible" + }, + { + "name": "strong_leak", + "periods_screened": 5, + "periods_flagged": 5, + "mean_rho": -0.868283463437835, + "max_abs_rho": 0.8934707972913439, + "mean_phi_req": 1.0, + "verdict": "susceptible" + } + ], + "totals": { + "signal_cycles": 20, + "signals_benign": 2, + "signals_susceptible": 2 + } +} +``` + +How to read it: four signals were screened over five periods each (20 signal +cycles). Two came out benign and release at roughly the completeness floor +(`mean_phi_req` ≈ 0.36–0.41). `mild_leak` is susceptible and is held until 82% +of its cross-section has arrived; `strong_leak` is held to the +deadline-complete cross-section (`mean_phi_req == 1.0`). + +## Transport (deliberately unspecified) + +This spec describes a **file format**, not a protocol. `pit-release-gate` writes +a record with `--export` and does nothing else with it: the package contains no +submission path and imports no transport at all. Where a record travels — a +commit next to a badge, an artifact in CI, an attachment, an endpoint of your own +— is the emitter's choice and outside this document. + +If you build something that receives records, validate them the way §"Validity +rules" describes and treat everything in a record as publishable. A record +carries summary statistics only: it is designed so that publishing one leaks +nothing about the data it was computed from. + +## Versioning + +`schema_version` is `MAJOR.MINOR`. A **minor** bump only adds optional fields; +a reader for `1.0` may ignore fields it does not know and keep working. A +**major** bump may remove or repurpose fields, and readers should refuse a major +version they do not know rather than guess. The schema name never changes +meaning: a document identified as `pit-screen-results` always means a screen +result in the sense described here. + +## Adoption + +**This format is free for any tool to emit or consume**, with no attribution +requirement, no coordination with this project, and no compatibility obligation +in either direction. A screen result is more useful when it is comparable across +tools, so put your own name in `tool`, keep `schema` and `schema_version` as +specified, and the record will read the same everywhere. Extensions are welcome +under a namespaced key of your own (e.g. `"x_yourtool"`); a `1.0` reader must +ignore what it does not recognize. + +If you extend the format in a way you think belongs in the core schema, open an +issue at +[github.com/MaxWellApexLab/pit-release-gate](https://github.com/MaxWellApexLab/pit-release-gate/issues). diff --git a/paper.bib b/paper.bib new file mode 100644 index 0000000..42bf9bc --- /dev/null +++ b/paper.bib @@ -0,0 +1,84 @@ +% paper.bib — pit-release-gate (JOSS) +% +% PROVENANCE: every field below was taken verbatim from the resolved DOI record +% on 2026-08-16. figshare entries from api.datacite.org; the rest from +% api.crossref.org. Nothing in this file was written from memory. +% 10.6084/m9.figshare.32952482 -> DataCite, resourceTypeGeneral "Preprint" +% 10.6084/m9.figshare.33061955 -> DataCite, resourceTypeGeneral "Preprint" +% 10.6084/m9.figshare.33158615 -> DataCite, resourceTypeGeneral "Preprint" +% 10.1145/2382577.2382579 -> Crossref +% 10.1016/j.patter.2023.100804 -> Crossref +% 10.2307/1912352 -> Crossref +% +% Two deliberate fidelity notes (do NOT "fix" these without re-resolving the DOI): +% * The figshare records register the first creator as "Kuan Ta Wu" (no hyphen) +% and carry no ORCIDs. That is what the DOI resolves to, so that is what is +% recorded here. See the submission-day checklist for the figshare metadata fix. +% * Crossref stores only the first page (153) for Heckman (1979); the record does +% not carry an end page, so none is invented here. + +@misc{wu2026a, + author = {Wu, Kuan Ta and Wu, Kuan-I}, + title = {{Correct-by-Construction Factor Computation: A Verifiably Point-in-Time Engine for Tradeable Signals}}, + year = {2026}, + publisher = {figshare}, + note = {Preprint}, + doi = {10.6084/m9.figshare.32952482}, + url = {https://doi.org/10.6084/m9.figshare.32952482} +} + +@misc{wu2026b, + author = {Wu, Kuan Ta and Wu, Kuan-I}, + title = {{Measuring Incomplete-Cross-Section Leakage: A Matched Placebo, a Susceptibility Screen, and Evidence from Taiwan and US As-Filed Data that the Channel Is Benign and Correctable}}, + year = {2026}, + publisher = {figshare}, + note = {Preprint}, + doi = {10.6084/m9.figshare.33061955}, + url = {https://doi.org/10.6084/m9.figshare.33061955} +} + +@misc{wu2026c, + author = {Wu, Kuan Ta and Wu, Kuan-I}, + title = {{Susceptibility-Graded Release Control: Preventing Incomplete-Cross-Section Leakage in Financial Machine-Learning Pipelines without a Blanket Timeliness Penalty}}, + year = {2026}, + publisher = {figshare}, + note = {Preprint}, + doi = {10.6084/m9.figshare.33158615}, + url = {https://doi.org/10.6084/m9.figshare.33158615} +} + +@article{kaufman2012, + author = {Kaufman, Shachar and Rosset, Saharon and Perlich, Claudia and Stitelman, Ori}, + title = {{Leakage in data mining: Formulation, detection, and avoidance}}, + journal = {ACM Transactions on Knowledge Discovery from Data}, + volume = {6}, + number = {4}, + pages = {1--21}, + year = {2012}, + publisher = {Association for Computing Machinery (ACM)}, + doi = {10.1145/2382577.2382579} +} + +@article{kapoor2023, + author = {Kapoor, Sayash and Narayanan, Arvind}, + title = {{Leakage and the reproducibility crisis in machine-learning-based science}}, + journal = {Patterns}, + volume = {4}, + number = {9}, + pages = {100804}, + year = {2023}, + publisher = {Elsevier BV}, + doi = {10.1016/j.patter.2023.100804} +} + +@article{heckman1979, + author = {Heckman, James J.}, + title = {{Sample Selection Bias as a Specification Error}}, + journal = {Econometrica}, + volume = {47}, + number = {1}, + pages = {153}, + year = {1979}, + publisher = {JSTOR}, + doi = {10.2307/1912352} +} diff --git a/paper.md b/paper.md new file mode 100644 index 0000000..5c3231a --- /dev/null +++ b/paper.md @@ -0,0 +1,115 @@ +--- +title: 'pit-release-gate: completeness-aware release control for staggered-arrival cross-sectional data' +tags: + - Python + - data leakage + - point-in-time data + - machine learning pipelines + - quantitative finance +authors: + - name: Kuan-Ta Wu + orcid: 0009-0006-0529-8709 + affiliation: 1 +affiliations: + - name: Max Well Apex LLC, NY, United States + index: 1 +date: 1 September 2026 +bibliography: paper.bib +--- + +# Summary + +Many empirical cross-sections are assembled from records that arrive on staggered +dates. Companies filing quarterly financial statements are the canonical case: a +fiscal period ends on a single date, but individual filings land over the following +weeks, up to a statutory deadline. Any same-period cross-sectional signal computed +before the last filer has arrived is therefore estimated from an incomplete +cross-section. Incompleteness is harmless when arrival timing is unrelated to what +the signal measures. It is not harmless when filing timing depends on the very +disturbance the signal is meant to capture: the early-arriving subset is then +selected on the estimand itself, and the released signal carries a systematic bias. +The standard defence — wait for the deadline before releasing anything — removes +the bias but charges a timeliness penalty to every signal, including those that +were never at risk. + +`pit-release-gate` replaces that blanket rule with a per-signal measurement. It +estimates each signal's susceptibility to this bias — a disturbance-conditional +partial correlation, fitted only on prior *completed* periods — and converts the +estimate into the cross-sectional completeness that the signal must reach before it +may be released. Benign signals release as soon as a minimum completeness floor is +met; susceptible signals are withheld until enough of the cross-section has arrived +to suppress the bias, up to the deadline. + +# Statement of need + +Point-in-time discipline in machine-learning pipelines currently rests on tooling +that answers one question: *was this value readable at time $t$?* Feature-store +as-of joins, bitemporal and vintage-aware storage, and purged or embargoed +cross-validation all enforce read-time correctness, and they do it well. + +None of them answers a second question: given that every value read was legitimately +readable, was the *set* of entities that had reported by $t$ a selected sample? An +as-of join over an incomplete cross-section is a correct join over a biased sample. +The distinction matters because the two failures need different remedies — the +first is fixed by timestamp hygiene, the second only by waiting or by an explicit +correction. Existing leakage taxonomies [@kaufman2012] and recent surveys of leakage +in machine-learning-based science [@kapoor2023] name this family of problems, and +the mechanism is a selection problem in the classical sense [@heckman1979], but +researchers have had no routine, per-signal screen they can execute inside a +pipeline. + +`pit-release-gate` provides one. The susceptibility measure, the grading rule that +turns it into a release threshold, and their evaluation on as-filed data are +developed in three publicly available preprints [@wu2026a; @wu2026b; @wu2026c]; this +package is the reference implementation of the release controller of the third. It +is aimed at researchers who build cross-sectional signals on staggered-arrival +panels — most immediately in empirical accounting and quantitative finance, but the +same arrival structure appears wherever administrative records backfill after a +reporting period closes. The design goal is that the screen costs one function call +per signal, so that reporting a susceptibility estimate alongside a released signal +becomes as ordinary as reporting a standard error. + +# Functionality + +The public API has five components: + +- `AsOfDataStore` holds the as-filed records for one period and cross-sectional + group: a design matrix, the constructed signal values, and a filing-arrival time + per entity. +- `CompletenessMonitor` reports the arrived fraction at an evaluation time, plus a + composition-shift gauge for the arrived subset. +- `SusceptibilityGate` estimates $\hat{\rho}$, the partial correlation between + filing latency and the complete-cross-section residual, conditional on + observables. Its `fit_trailing` method enforces an honest-estimation contract: + $\hat{\rho}$ is fitted on prior completed periods only, never on the period being + gated, whose cross-section is by definition still incomplete. +- `PropensityReweighter` supplies inverse-filing-propensity weights. It is included + to make a negative result executable: reweighting on observables corrects + composition shift but cannot remove selection on the disturbance, which is why the + controller grades on completeness rather than on reweighting. +- `ReleaseController` maps $|\hat{\rho}|$ to a required completeness, + $\varphi_{\mathrm{req}} = \min(1,\ \varphi_{\min} + \kappa|\hat{\rho}|)$, and + returns at each evaluation time one of `WITHHOLD`, `REWEIGHT_RELEASE` or + `RELEASE`, together with the released values. + +The package ships a self-contained worked example in which the strength of selection +on the disturbance is *planted*, so the correct decision is known exactly and no +licensed data is required. Running `pit-release-gate` compares five release policies +across four signal types and prints the resulting timeliness and bias. The run is +deterministic under a fixed seed and the test suite asserts its headline numbers, so +a reader can confirm in one command that the gate releases benign signals at roughly +a third of the cross-section, while holding a strongly selected signal to the +complete cross-section, where its bias is exactly zero. + +Installation is `pip install pit-release-gate`. The package requires Python 3.10 or +later and depends only on NumPy, pandas and SciPy; it is MIT licensed and tested on +Linux and Windows against Python 3.10 and 3.13. + +# Acknowledgements + +The methodology preprints cited above are co-authored with Kuan-I Wu; the software +described here was designed and written solely by the author. The author used +AI-assisted drafting tools in preparing this manuscript; all technical content, +experiments, and claims were designed, executed, and verified by the author. + +# References diff --git a/pyproject.toml b/pyproject.toml index 11f6233..2a705bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pit-release-gate" -version = "0.1.0" +version = "0.1.1" description = "Completeness-aware release control for staggered-arrival cross-sectional data: a susceptibility-graded gate that blocks incomplete-cross-section leakage without a blanket timeliness penalty" readme = "README.md" requires-python = ">=3.10" diff --git a/src/pit_release_gate/__init__.py b/src/pit_release_gate/__init__.py index f02846e..a112bce 100644 --- a/src/pit_release_gate/__init__.py +++ b/src/pit_release_gate/__init__.py @@ -18,16 +18,40 @@ make_group one synthetic staggered-arrival cross-section (known truth) run_demo the full known-ground-truth worked example (returns dict) demo same, console-table form +build_results assemble a pit-screen-results record (summary stats only) +validate_results check such a record; returns a list of problems +badge_snippet README badge markdown for a completed screen run """ +# defined first: results.tool_version() reads it while the submodules below +# are still importing +__version__ = "0.1.1" + from .controller import ReleaseController, ReleaseDecision from .gate import SusceptibilityGate from .monitor import CompletenessMonitor +from .results import ( + SCHEMA, + SCHEMA_VERSION, + build_results, + screen_config, + summarize_signal, + validate_results, + write_results, +) from .reweight import PropensityReweighter -from .simulate import DEMO_POLICIES, DEMO_SIGNALS, SEED, demo, main, make_group, run_demo +from .simulate import ( + DEMO_POLICIES, + DEMO_SIGNALS, + SEED, + badge_snippet, + demo, + main, + make_group, + results_from_demo, + run_demo, +) from .store import AsOfDataStore -__version__ = "0.1.0" - __all__ = [ "AsOfDataStore", "CompletenessMonitor", @@ -39,8 +63,17 @@ "run_demo", "demo", "main", + "badge_snippet", "SEED", "DEMO_SIGNALS", "DEMO_POLICIES", + "SCHEMA", + "SCHEMA_VERSION", + "summarize_signal", + "screen_config", + "build_results", + "validate_results", + "write_results", + "results_from_demo", "__version__", ] diff --git a/src/pit_release_gate/controller.py b/src/pit_release_gate/controller.py index 070a15d..6cded86 100644 --- a/src/pit_release_gate/controller.py +++ b/src/pit_release_gate/controller.py @@ -10,6 +10,11 @@ from .reweight import PropensityReweighter from .store import AsOfDataStore +#: Minimum number of arrived entities before anything is released; below it a +#: cross-section is too small for the residualization to mean much. Named here +#: so a screen record can report the floor it ran under. +MIN_ENTITIES = 6 + def _ols_resid(X: np.ndarray, y: np.ndarray, w: np.ndarray = None) -> np.ndarray: if w is None: @@ -70,11 +75,11 @@ def decide(self, store: AsOfDataStore, t: float, policy: str = 'gated') -> Relea # fixed baseline policies (for comparison / fallback configs) ---- if policy == 'naive': - if comp < self.phi_min or m.sum() < 6: + if comp < self.phi_min or m.sum() < MIN_ENTITIES: return ReleaseDecision('WITHHOLD', t, comp, policy=policy) return self._emit(store, t, m, None, 'RELEASE', policy) if policy == 'threshold': - if comp < self.phi_high or m.sum() < 6: + if comp < self.phi_high or m.sum() < MIN_ENTITIES: return ReleaseDecision('WITHHOLD', t, comp, policy=policy) return self._emit(store, t, m, None, 'RELEASE', policy) if policy == 'deadline': @@ -82,7 +87,7 @@ def decide(self, store: AsOfDataStore, t: float, policy: str = 'gated') -> Relea return ReleaseDecision('WITHHOLD', t, comp, policy=policy) return self._emit(store, t, m, None, 'RELEASE', policy) if policy == 'reweight': - if comp < self.phi_min or m.sum() < 6: + if comp < self.phi_min or m.sum() < MIN_ENTITIES: return ReleaseDecision('WITHHOLD', t, comp, policy=policy) w = self.reweighter.weights(store, t) return self._emit(store, t, m, w, 'REWEIGHT_RELEASE', policy) @@ -95,7 +100,7 @@ def decide(self, store: AsOfDataStore, t: float, policy: str = 'gated') -> Relea # bias, up to the statutory deadline (completeness=1.0). rho = self.gate.rho_hat(store) phi_req = self.required_completeness(rho) - if (comp >= phi_req or t >= 1.0) and m.sum() >= 6: + if (comp >= phi_req or t >= 1.0) and m.sum() >= MIN_ENTITIES: return self._emit(store, t, m, None, 'RELEASE', f'gated(phi_req={phi_req:.2f})') return ReleaseDecision('WITHHOLD', t, comp, policy=f'gated(phi_req={phi_req:.2f})') diff --git a/src/pit_release_gate/results.py b/src/pit_release_gate/results.py new file mode 100644 index 0000000..456650e --- /dev/null +++ b/src/pit_release_gate/results.py @@ -0,0 +1,256 @@ +"""The ``pit-screen-results`` record: build it, validate it, write it. + +A screen result is a *summary*: per signal, how many periods were screened, +how many of them the susceptibility measure flagged, the mean and maximum +susceptibility, the required completeness the controller assigned, and the +verdict. Nothing else. No input rows, no file paths, no identities, no +environment details beyond the tool version -- so a record is safe to commit +next to the badge, or to hand to a third party. + +This module is deliberately **offline**: it imports no transport machinery, +so nothing that builds or writes a record can send it anywhere. Neither does +anything else in this package -- there is no submission path at all, and a +record goes where you put it and nowhere else. + +It is also deliberately **timeless**: no clock is read here, so two runs of +the same screen produce byte-identical records. A caller that wants a date +in the record passes one in explicitly (``build_results(..., date=...)``). + +The schema is documented as a standalone interchange format in +``docs/results-schema.md``; other tools are free to emit it. +""" +from __future__ import annotations + +import json +from pathlib import Path + +#: Self-identifying name of the interchange format (not of the producing tool). +SCHEMA = 'pit-screen-results' +SCHEMA_VERSION = '1.0' +KNOWN_SCHEMA_VERSIONS = ('1.0',) + +#: The tool that produces the record in this package. +TOOL = 'pit-release-gate' + +#: The only two verdicts a screened signal can carry. +VERDICTS = ('benign', 'susceptible') + +_TOP_FIELDS = ('schema', 'schema_version', 'tool', 'tool_version', + 'config', 'signals', 'totals') +_CONFIG_FIELDS = ('rho_threshold', 'phi_min', 'kappa', 'trailing_k', 'min_entities') +_SIGNAL_FIELDS = ('name', 'periods_screened', 'periods_flagged', 'mean_rho', + 'max_abs_rho', 'mean_phi_req', 'verdict') +_TOTALS_FIELDS = ('signal_cycles', 'signals_benign', 'signals_susceptible') + + +def tool_version() -> str: + """Version of the code that actually ran. + + The in-tree ``__version__`` is preferred over installed distribution + metadata, because a source checkout on ``PYTHONPATH`` can shadow an older + installed wheel; the metadata is the fallback. Never a hardcoded literal. + """ + try: + from . import __version__ + return str(__version__) + except ImportError: # pragma: no cover - only if the package is half-built + from importlib.metadata import PackageNotFoundError, version + try: + return version(TOOL) + except PackageNotFoundError: + return 'unknown' + + +def screen_config(rho_threshold: float, phi_min: float, kappa: float, + trailing_k: int, min_entities: int) -> dict: + """The screen settings that determine a verdict, in record form. + + These five are what another party needs in order to read a verdict: + the susceptibility threshold, the completeness floor and slope of + ``phi_req = min(1, phi_min + kappa*|rho_hat|)``, how many prior completed + periods the estimate was fitted on, and the minimum arrived count below + which the controller releases nothing. + """ + return { + 'rho_threshold': float(rho_threshold), + 'phi_min': float(phi_min), + 'kappa': float(kappa), + 'trailing_k': int(trailing_k), + 'min_entities': int(min_entities), + } + + +def summarize_signal(name: str, rhos, phi_reqs, rho_threshold: float = 0.10, + susceptible: bool = None) -> dict: + """Reduce one signal's per-period screen to the record's summary row. + + ``rhos`` are the per-period susceptibility estimates and ``phi_reqs`` the + required completeness the controller assigned in each of those periods; + they must be the same length, and that length is ``periods_screened``. + A period is *flagged* when ``|rho|`` exceeds ``rho_threshold``, so + ``periods_flagged <= periods_screened`` by construction. + + The verdict defaults to the same test applied to the mean susceptibility; + a caller that gates on a frozen trailing estimate should pass its own + verdict as ``susceptible`` so the record states what the screen actually + decided. + """ + rhos = [float(r) for r in rhos] + phi_reqs = [float(p) for p in phi_reqs] + if len(rhos) != len(phi_reqs): + raise ValueError(f'{name}: got {len(rhos)} rho values but ' + f'{len(phi_reqs)} required-completeness values') + if not rhos: + raise ValueError(f'{name}: no screened periods') + mean_rho = sum(rhos) / len(rhos) + if susceptible is None: + susceptible = abs(mean_rho) > rho_threshold + return { + 'name': str(name), + 'periods_screened': len(rhos), + 'periods_flagged': sum(1 for r in rhos if abs(r) > rho_threshold), + 'mean_rho': mean_rho, + 'max_abs_rho': max(abs(r) for r in rhos), + 'mean_phi_req': sum(phi_reqs) / len(phi_reqs), + 'verdict': VERDICTS[1] if susceptible else VERDICTS[0], + } + + +def build_results(signals, config: dict, tool: str = TOOL, + version: str = None, date: str = None) -> dict: + """Assemble a complete ``pit-screen-results`` record. + + ``signals`` is a list of :func:`summarize_signal` rows. The totals are + derived here rather than accepted from the caller, so they cannot + disagree with the rows. ``date`` is optional and purely caller-supplied + -- this module never reads a clock. + """ + signals = [dict(s) for s in signals] + record = { + 'schema': SCHEMA, + 'schema_version': SCHEMA_VERSION, + 'tool': str(tool), + 'tool_version': version or tool_version(), + 'config': dict(config), + 'signals': signals, + 'totals': { + 'signal_cycles': sum(int(s['periods_screened']) for s in signals), + 'signals_benign': sum(1 for s in signals if s['verdict'] == VERDICTS[0]), + 'signals_susceptible': sum(1 for s in signals if s['verdict'] == VERDICTS[1]), + }, + } + if date is not None: + record['date'] = str(date) + return record + + +def validate_results(obj) -> list[str]: + """Return a list of human-readable problems with ``obj``; empty == valid. + + Written so that a reader of the format -- not just this package -- can + check a record before relying on it, and so that any receiver of the + format can refuse something malformed. + """ + problems: list[str] = [] + if not isinstance(obj, dict): + return [f'record must be a JSON object, got {type(obj).__name__}'] + + version = obj.get('schema_version') + if version is None: + problems.append('missing required field: schema_version') + elif version not in KNOWN_SCHEMA_VERSIONS: + problems.append(f'unknown schema_version {version!r} ' + f'(known: {", ".join(KNOWN_SCHEMA_VERSIONS)})') + if obj.get('schema') != SCHEMA: + problems.append(f'schema must be {SCHEMA!r}, got {obj.get("schema")!r}') + for field in _TOP_FIELDS: + if field not in obj: + if field != 'schema_version': # already reported above + problems.append(f'missing required field: {field}') + + config = obj.get('config') + if config is not None and not isinstance(config, dict): + problems.append('config must be a JSON object') + elif isinstance(config, dict): + for field in _CONFIG_FIELDS: + if field not in config: + problems.append(f'missing required config field: {field}') + + signals = obj.get('signals') + cycles = 0 + if signals is not None and not isinstance(signals, list): + problems.append('signals must be a list') + elif isinstance(signals, list): + if not signals: + problems.append('signals is empty: nothing was screened') + for i, sig in enumerate(signals): + where = f'signals[{i}]' + if not isinstance(sig, dict): + problems.append(f'{where} must be a JSON object') + continue + where = f'signals[{i}] ({sig.get("name", "unnamed")})' + for field in _SIGNAL_FIELDS: + if field not in sig: + problems.append(f'{where}: missing required field: {field}') + if sig.get('verdict') not in VERDICTS and 'verdict' in sig: + problems.append(f'{where}: verdict must be one of ' + f'{", ".join(VERDICTS)}, got {sig["verdict"]!r}') + screened, flagged = sig.get('periods_screened'), sig.get('periods_flagged') + if isinstance(screened, int) and isinstance(flagged, int): + cycles += screened + if flagged > screened: + problems.append(f'{where}: periods_flagged ({flagged}) exceeds ' + f'periods_screened ({screened})') + if flagged < 0 or screened < 0: + problems.append(f'{where}: periods_screened and periods_flagged ' + f'must not be negative') + elif screened is not None or flagged is not None: + problems.append(f'{where}: periods_screened and periods_flagged ' + f'must be integers') + + totals = obj.get('totals') + if totals is not None and not isinstance(totals, dict): + problems.append('totals must be a JSON object') + elif isinstance(totals, dict): + for field in _TOTALS_FIELDS: + if field not in totals: + problems.append(f'missing required totals field: {field}') + if isinstance(signals, list) and 'signal_cycles' in totals: + if totals['signal_cycles'] != cycles: + problems.append(f'totals.signal_cycles ({totals["signal_cycles"]}) ' + f'does not equal the sum over signals ({cycles})') + if isinstance(signals, list): + benign = sum(1 for s in signals + if isinstance(s, dict) and s.get('verdict') == VERDICTS[0]) + suspect = sum(1 for s in signals + if isinstance(s, dict) and s.get('verdict') == VERDICTS[1]) + if totals.get('signals_benign') != benign: + problems.append(f'totals.signals_benign ({totals.get("signals_benign")}) ' + f'does not equal the number of benign signals ({benign})') + if totals.get('signals_susceptible') != suspect: + problems.append(f'totals.signals_susceptible ' + f'({totals.get("signals_susceptible")}) does not equal ' + f'the number of susceptible signals ({suspect})') + return problems + + +def dumps_results(obj) -> str: + """Canonical JSON text of a record -- exactly what ``--export`` writes.""" + return json.dumps(obj, indent=2, ensure_ascii=False) + + +def write_results(obj, path) -> Path: + """Write a validated record to ``path`` as JSON. Local file I/O only. + + Raises ``ValueError`` rather than writing a record that would not survive + :func:`validate_results`. + """ + problems = validate_results(obj) + if problems: + raise ValueError('refusing to write an invalid ' + SCHEMA + ' record: ' + + '; '.join(problems)) + path = Path(path) + if path.parent and not path.parent.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(dumps_results(obj) + '\n', encoding='utf-8', newline='\n') + return path diff --git a/src/pit_release_gate/simulate.py b/src/pit_release_gate/simulate.py index 516193f..c37b5c2 100644 --- a/src/pit_release_gate/simulate.py +++ b/src/pit_release_gate/simulate.py @@ -15,13 +15,25 @@ import pandas as pd from scipy.stats import rankdata -from .controller import ReleaseController, ReleaseDecision, _ols_resid +from .controller import MIN_ENTITIES, ReleaseController, ReleaseDecision, _ols_resid from .gate import SusceptibilityGate +from .results import ( + SCHEMA, + SCHEMA_VERSION, + build_results, + screen_config, + summarize_signal, + write_results, +) from .reweight import PropensityReweighter from .store import AsOfDataStore SEED = 20260601 +#: Susceptibility threshold the demo screen runs at (|rho_hat| above it is +#: susceptible). Named so the exported screen record cannot drift from it. +DEMO_RHO_THRESHOLD = 0.10 + #: The four planted-truth signal configurations reported in the demo: #: (key, label, c_a, c_x) DEMO_SIGNALS = [ @@ -113,10 +125,17 @@ def run_demo(n_train=10, n_eval=60, verbose=True) -> dict: for key, name, c_a, c_x in DEMO_SIGNALS: # --- honest trailing estimation on prior completed periods --- - gate = SusceptibilityGate(threshold=0.10) + gate = SusceptibilityGate(threshold=DEMO_RHO_THRESHOLD) train = [make_group(n=120, c_a=c_a, c_x=c_x, rng=rng) for _ in range(n_train)] rho_tr = gate.fit_trailing(train) ctrl = ReleaseController(gate=gate) + # the settings that produced the verdicts, recorded for --export + # (identical for every signal; the kappa sweep below is separate) + results['config'] = screen_config(rho_threshold=gate.threshold, + phi_min=ctrl.phi_min, + kappa=ctrl.suscept_slope, + trailing_k=n_train, + min_entities=MIN_ENTITIES) # --- evaluation on fresh periods, gated with the FROZEN estimate --- agg = {p: {'comp': [], 'flip': [], 'bias': [], 'act': []} for p in policies} rho_realized = [] @@ -133,8 +152,11 @@ def run_demo(n_train=10, n_eval=60, verbose=True) -> dict: sig = {'label': name, 'c_a': c_a, 'c_x': c_x, 'rho_trailing': rho_tr, + 'rho_realized': [float(r) for r in rho_realized], 'rho_realized_mean': float(np.mean(rho_realized)), 'rho_realized_std': float(np.std(rho_realized)), + # constant across periods: the gate runs on the FROZEN estimate + 'phi_req': ctrl.required_completeness(rho_tr), 'susceptible': bool(gate.is_susceptible(rho_tr)), 'policies': {}} for p in policies: @@ -164,7 +186,7 @@ def run_demo(n_train=10, n_eval=60, verbose=True) -> dict: if verbose: print("\n" + "-" * 96) print("Sensitivity: required-completeness slope kappa on Mild-leak (c_a=0.3, c_x=0.7)") - gate = SusceptibilityGate(threshold=0.10) + gate = SusceptibilityGate(threshold=DEMO_RHO_THRESHOLD) train = [make_group(n=120, c_a=0.3, c_x=0.7, rng=rng) for _ in range(n_train)] rho_tr = gate.fit_trailing(train) for kappa in [0.5, 1.0, 2.0]: @@ -212,17 +234,110 @@ def demo(n_train=10, n_eval=60): return run_demo(n_train=n_train, n_eval=n_eval, verbose=True) +def results_from_demo(run: dict, date: str = None) -> dict: + """Reduce a :func:`run_demo` result to a ``pit-screen-results`` record. + + Only summary statistics survive the reduction: per signal, the number of + screened periods, how many of them the susceptibility measure flagged, + the mean and maximum |rho_hat|, the required completeness the controller + assigned, and the verdict. The simulated cross-sections themselves stay + on this machine. + + ``date`` is optional and caller-supplied; nothing here reads a clock, so + the same screen always reduces to the same record. + """ + threshold = run['config']['rho_threshold'] + signals = [ + summarize_signal( + key, + rhos=sig['rho_realized'], + phi_reqs=[sig['phi_req']] * len(sig['rho_realized']), + rho_threshold=threshold, + # the verdict the screen actually acted on: the frozen trailing + # estimate, not the ex-post realized ones summarized above + susceptible=sig['susceptible'], + ) + for key, sig in run['signals'].items() + ] + return build_results(signals, run['config'], date=date) + + +BADGE_MARKDOWN = ( + '[![screened with pit-release-gate]' + '(https://img.shields.io/badge/screened%20with-pit--release--gate-blue)]' + '(https://github.com/MaxWellApexLab/pit-release-gate)' +) + + +def badge_snippet(results) -> str: + """README badge markdown for a completed screen, with a rho_hat summary. + + Pure formatting over an existing ``run_demo`` result -- it reads the + result dict and returns a string. It performs no estimation and changes + no numerical behavior. + + The badge states that the screen was RUN. It is deliberately not a + pass/fail claim: a susceptible verdict is as worth reporting as a + benign one. + """ + try: + from . import __version__ as version + except ImportError: # pragma: no cover + version = '' + + sigs = results['signals'] + keys = [k for k, *_ in DEMO_SIGNALS if k in sigs] + n_susc = sum(bool(sigs[k]['susceptible']) for k in keys) + rhos = ' | '.join(f"{k} {sigs[k]['rho_trailing']:+.3f}" for k in keys) + + rule = '-' * 72 + return '\n'.join([ + rule, + 'Badge snippet (paste into your README):', + '', + BADGE_MARKDOWN, + '', + f'', + '', + 'The badge states that the screen was RUN, not that anything passed.', + 'Point it at your own screen output to make it worth clicking.', + rule, + ]) + + def main(argv=None): ap = argparse.ArgumentParser( prog='pit-release-gate', description='Run the self-contained known-ground-truth demo of the ' - 'completeness-aware release controller.') + 'completeness-aware release controller.', + epilog='This tool never reports anything, anywhere. --export writes a ' + 'local file and makes no network call; the package opens no ' + 'socket at all.') ap.add_argument('--train', type=int, default=10, help='number of prior completed periods used to fit rho_hat (default 10)') ap.add_argument('--eval', dest='n_eval', type=int, default=60, help='number of fresh evaluation periods (default 60)') + ap.add_argument('--export', metavar='PATH', + help=f'write the screen result to PATH as a {SCHEMA} ' + f'v{SCHEMA_VERSION} JSON record (fully offline)') + ap.add_argument('--badge', action='store_true', + help='after the demo, print a README badge snippet recording ' + 'that the screen was run (does not change the demo output)') a = ap.parse_args(argv) - run_demo(n_train=a.train, n_eval=a.n_eval, verbose=True) + + run = run_demo(n_train=a.train, n_eval=a.n_eval, verbose=True) + if a.badge: + print() + print(badge_snippet(run)) + if not a.export: + return + + record = results_from_demo(run) + path = write_results(record, a.export) + print(f'\nwrote {SCHEMA} v{SCHEMA_VERSION} to {path} ' + f'(local file only -- no network call was made)') if __name__ == '__main__': diff --git a/tests/test_cli_and_monitor.py b/tests/test_cli_and_monitor.py index e5ac233..fb10769 100644 --- a/tests/test_cli_and_monitor.py +++ b/tests/test_cli_and_monitor.py @@ -6,7 +6,8 @@ import numpy as np -from pit_release_gate import AsOfDataStore, CompletenessMonitor, make_group +from pit_release_gate import (AsOfDataStore, CompletenessMonitor, badge_snippet, + make_group, run_demo) def test_cli_demo_runs_and_prints_verdicts(): @@ -31,6 +32,43 @@ def test_demo_main_in_process(capsys): assert token in text +def test_badge_snippet_reports_the_screen_result(): + r = run_demo(n_train=2, n_eval=4, verbose=False) + text = badge_snippet(r) + + # the markdown a user actually pastes + assert "img.shields.io/badge/screened%20with-pit--release--gate-blue" in text + assert "github.com/MaxWellApexLab/pit-release-gate" in text + + # the summary comment carries every signal's frozen rho_hat + for key in ("clean", "composition", "mild_leak", "strong_leak"): + assert key in text + # demo plants two benign and two susceptible signals + assert "2 benign, 2 susceptible" in text + + +def test_badge_snippet_makes_no_pass_fail_claim(): + text = badge_snippet(run_demo(n_train=2, n_eval=4, verbose=False)).lower() + assert "screened with" in text + assert "not that anything passed" in text + for forbidden in ("certif", "approv", "endors", "official", "trusted"): + assert forbidden not in text, f"badge output must not claim {forbidden!r}" + + +def test_badge_flag_does_not_change_demo_output(capsys): + from pit_release_gate.simulate import main + + main(["--train", "2", "--eval", "4"]) + plain = capsys.readouterr().out + main(["--train", "2", "--eval", "4", "--badge"]) + badged = capsys.readouterr().out + + # the badge is strictly appended: the demo output is byte-identical + assert badged.startswith(plain) + assert "Badge snippet" in badged[len(plain):] + assert "Badge snippet" not in plain + + def test_completeness_monitor_fraction_and_shift(): rng = np.random.default_rng(7) store = make_group(c_a=0.0, c_x=1.0, rng=rng) diff --git a/tests/test_export.py b/tests/test_export.py new file mode 100644 index 0000000..b508561 --- /dev/null +++ b/tests/test_export.py @@ -0,0 +1,287 @@ +"""Tests for the ``pit-screen-results`` record and the ``--export`` path. + +One property matters more than the schema details and is asserted from +several directions here: **this package never talks to the network.** The +urlopen entry point and the socket constructor are monkeypatched to raise, +and every path must still succeed; a structural test then walks every module +in the package and fails if any of them so much as imports a transport. + +There is deliberately no submission path on this branch. The ``--submit`` +half of this work is parked on the ``submit-cli`` branch until a receiving +endpoint exists -- a command pointing at an endpoint that is not there must +not ship. +""" +import getpass +import json +import os +import re +import socket +import sys +import urllib.request +from pathlib import Path + +import pytest + +import pit_release_gate +from pit_release_gate import results as results_mod +from pit_release_gate.results import ( + SCHEMA, + SCHEMA_VERSION, + TOOL, + VERDICTS, + build_results, + screen_config, + summarize_signal, + validate_results, + write_results, +) +from pit_release_gate.simulate import main + + +# --------------------------------------------------------------------------- +# no test in this file may reach the real network +# --------------------------------------------------------------------------- +def _exploding(*a, **k): + raise AssertionError("network call attempted") + + +@pytest.fixture +def no_network(monkeypatch): + """Any attempt to open a socket or a URL blows up the test.""" + monkeypatch.setattr(urllib.request, "urlopen", _exploding) + monkeypatch.setattr(socket, "socket", _exploding) + return None + + +def _tiny_record(**kw): + """A small valid record built through the public API (no demo run).""" + sig = summarize_signal("alpha", rhos=[0.01, -0.02, 0.30], + phi_reqs=[0.35, 0.35, 0.35], rho_threshold=0.10) + cfg = screen_config(rho_threshold=0.10, phi_min=0.35, kappa=1.0, + trailing_k=4, min_entities=6) + return build_results([sig], cfg, **kw) + + +# --------------------------------------------------------------------------- +# 1. the record: summary statistics, totals consistent by construction +# --------------------------------------------------------------------------- +def test_signal_summary_reports_only_summary_statistics(): + sig = summarize_signal("alpha", rhos=[0.01, -0.02, 0.30], + phi_reqs=[0.35, 0.35, 0.35], rho_threshold=0.10) + assert set(sig) == {"name", "periods_screened", "periods_flagged", "mean_rho", + "max_abs_rho", "mean_phi_req", "verdict"} + assert sig["periods_screened"] == 3 + assert sig["periods_flagged"] == 1 # only |0.30| exceeds 0.10 + assert sig["max_abs_rho"] == pytest.approx(0.30) + assert sig["mean_rho"] == pytest.approx((0.01 - 0.02 + 0.30) / 3) + assert sig["mean_phi_req"] == pytest.approx(0.35) + assert sig["verdict"] in VERDICTS + assert sig["periods_flagged"] <= sig["periods_screened"] + + +def test_verdict_follows_the_threshold_and_can_be_overridden(): + benign = summarize_signal("a", [0.01, 0.0], [0.35, 0.35], rho_threshold=0.10) + leaky = summarize_signal("b", [-0.9, -0.8], [1.0, 1.0], rho_threshold=0.10) + assert benign["verdict"] == "benign" + assert leaky["verdict"] == "susceptible" + # the screen's own frozen verdict wins when the caller supplies it + forced = summarize_signal("c", [0.01, 0.0], [0.35, 0.35], + rho_threshold=0.10, susceptible=True) + assert forced["verdict"] == "susceptible" + + +def test_build_results_totals_are_consistent_by_construction(): + sigs = [ + summarize_signal("a", [0.01, 0.0, 0.0], [0.35] * 3, rho_threshold=0.10), + summarize_signal("b", [-0.9, -0.8], [1.0, 1.0], rho_threshold=0.10), + ] + rec = build_results(sigs, screen_config(0.10, 0.35, 1.0, 4, 6)) + assert rec["schema"] == SCHEMA == "pit-screen-results" + assert rec["schema_version"] == SCHEMA_VERSION == "1.0" + assert rec["tool"] == TOOL == "pit-release-gate" + assert rec["tool_version"] == pit_release_gate.__version__ + assert rec["config"] == {"rho_threshold": 0.10, "phi_min": 0.35, "kappa": 1.0, + "trailing_k": 4, "min_entities": 6} + assert rec["totals"] == {"signal_cycles": 5, "signals_benign": 1, + "signals_susceptible": 1} + assert validate_results(rec) == [] + + +def test_record_carries_no_timestamp_unless_the_caller_passes_a_date(): + a = _tiny_record() + b = _tiny_record() + assert a == b # reproducible: nothing time-varying inside + assert json.dumps(a) == json.dumps(b) + assert "date" not in a + assert not [k for k in a if "time" in k.lower() or "date" in k.lower()] + dated = _tiny_record(date="2026-08-16") + assert dated["date"] == "2026-08-16" + + +# --------------------------------------------------------------------------- +# 2. the validator +# --------------------------------------------------------------------------- +def test_validate_accepts_a_built_record(): + assert validate_results(_tiny_record()) == [] + + +@pytest.mark.parametrize("mutate, needle", [ + (lambda r: r.pop("schema_version"), "schema_version"), + (lambda r: r.update(schema_version="9.9"), "9.9"), + (lambda r: r.pop("tool_version"), "tool_version"), + (lambda r: r.pop("config"), "config"), + (lambda r: r["config"].pop("kappa"), "kappa"), + (lambda r: r["signals"][0].pop("mean_rho"), "mean_rho"), + (lambda r: r["signals"][0].update(verdict="probably fine"), "verdict"), + (lambda r: r["signals"][0].update(periods_flagged=99), "periods_flagged"), + (lambda r: r["totals"].update(signal_cycles=999), "signal_cycles"), + (lambda r: r.update(signals="not a list"), "signals"), +]) +def test_validate_reports_each_kind_of_problem(mutate, needle): + rec = _tiny_record() + mutate(rec) + problems = validate_results(rec) + assert problems, "validator missed a broken record" + assert any(needle in p for p in problems), problems + + +def test_validate_rejects_non_mapping(): + assert validate_results([1, 2, 3]) + assert validate_results(None) + + +# --------------------------------------------------------------------------- +# 3. --export is fully offline +# --------------------------------------------------------------------------- +def test_export_writes_a_valid_results_json(tmp_path, capsys): + path = tmp_path / "results.json" + main(["--train", "3", "--eval", "4", "--export", str(path)]) + capsys.readouterr() + + rec = json.loads(path.read_text(encoding="utf-8")) + assert validate_results(rec) == [] + assert rec["schema"] == "pit-screen-results" + assert rec["schema_version"] == "1.0" + assert rec["tool"] == "pit-release-gate" + assert rec["tool_version"] == pit_release_gate.__version__ + assert rec["config"]["trailing_k"] == 3 + + names = [s["name"] for s in rec["signals"]] + assert names == ["clean", "composition", "mild_leak", "strong_leak"] + by_name = {s["name"]: s for s in rec["signals"]} + assert by_name["clean"]["verdict"] == "benign" + assert by_name["mild_leak"]["verdict"] == "susceptible" + assert by_name["strong_leak"]["verdict"] == "susceptible" + # the susceptible signals are graded to a higher required completeness + assert by_name["strong_leak"]["mean_phi_req"] > by_name["clean"]["mean_phi_req"] + + for s in rec["signals"]: + assert s["periods_screened"] == 4 + assert s["periods_flagged"] <= s["periods_screened"] + assert rec["totals"]["signal_cycles"] == 16 + assert rec["totals"]["signals_benign"] + rec["totals"]["signals_susceptible"] == 4 + + +def test_export_performs_no_network_io(tmp_path, capsys, no_network): + """The red line: --export must not touch the network at all.""" + path = tmp_path / "offline.json" + main(["--train", "2", "--eval", "2", "--export", str(path)]) + capsys.readouterr() + assert validate_results(json.loads(path.read_text(encoding="utf-8"))) == [] + + +def test_results_module_has_no_network_machinery(): + # the module that builds and writes the record cannot import a transport + assert not hasattr(results_mod, "urllib") + assert not hasattr(results_mod, "socket") + src = Path(results_mod.__file__).read_text(encoding="utf-8") + assert "urllib" not in src + assert "atexit" not in src + + +def _imported_modules(path: Path) -> set: + src = path.read_text(encoding="utf-8") + return {m.lstrip(".").split(".")[0] + for m in re.findall(r"^\s*(?:from|import)\s+([\w.]+)", src, re.M)} + + +def test_no_module_in_the_package_can_reach_the_network(): + """Structural guard on the no-telemetry red line. + + Not "no module reports by default" -- *no module can*. Nothing in the + package may import a transport, register an exit hook, or spawn a worker + that could report in the background. This is asserted over every module, + so adding a phone-home later fails the suite rather than shipping. + """ + pkg = Path(pit_release_gate.__file__).parent + modules = sorted(pkg.glob("*.py")) + assert len(modules) >= 8 + for py in modules: + imported = _imported_modules(py) + assert "atexit" not in imported, f"{py.name} registers an exit hook" + assert not imported & {"threading", "multiprocessing", "concurrent", + "asyncio", "subprocess"}, f"{py.name} spawns work" + assert not imported & {"urllib", "socket", "http", "ssl", "requests", + "smtplib", "ftplib"}, f"{py.name} imports a transport" + + +def test_the_package_ships_no_submission_path(): + """The --submit half stays parked until a receiving endpoint exists.""" + pkg = Path(pit_release_gate.__file__).parent + assert not (pkg / "submit.py").exists() + assert not hasattr(pit_release_gate, "submit_results") + help_text = Path(pkg / "simulate.py").read_text(encoding="utf-8") + for flag in ("'--submit'", "'--contact'", "'--dry-run'"): + assert flag not in help_text, f"{flag} is still wired into the CLI" + + +def test_the_export_path_never_reads_a_clock(): + """No timestamps generated inside the library: a date must be passed in.""" + pkg = Path(pit_release_gate.__file__).parent + for name in ("results.py", "simulate.py"): + imported = _imported_modules(pkg / name) + assert not imported & {"time", "datetime", "calendar"}, f"{name} reads a clock" + + +def test_export_is_byte_identical_across_runs(tmp_path, capsys): + a, b = tmp_path / "a.json", tmp_path / "b.json" + main(["--train", "2", "--eval", "2", "--export", str(a)]) + main(["--train", "2", "--eval", "2", "--export", str(b)]) + capsys.readouterr() + assert a.read_bytes() == b.read_bytes() + + +def test_exported_payload_carries_no_identifying_details(tmp_path, capsys): + path = tmp_path / "results.json" + main(["--train", "2", "--eval", "2", "--export", str(path)]) + capsys.readouterr() + text = path.read_text(encoding="utf-8") + for secret in (getpass.getuser(), socket.gethostname(), os.getcwd(), + str(Path.home()), sys.executable, str(path)): + if secret and len(secret) > 3: + assert secret not in text, f"payload leaks {secret!r}" + for banned in ("path", "user", "host", "cwd", "platform", "python"): + assert banned not in text.lower() + + +# --------------------------------------------------------------------------- +# 4. the submission flags are not wired in on this branch +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("argv", [ + ["--submit", "https://example.invalid/submit"], + ["--contact", "a@b.org"], + ["--dry-run"], +]) +def test_cli_does_not_accept_submission_flags(argv, capsys, no_network): + """argparse must reject them outright -- they are not shipped.""" + with pytest.raises(SystemExit) as exc: + main(["--train", "2", "--eval", "2", *argv]) + assert exc.value.code != 0 + assert "unrecognized arguments" in capsys.readouterr().err + + +def test_plain_cli_run_writes_no_file_and_opens_no_socket(tmp_path, capsys, no_network): + main(["--train", "2", "--eval", "2"]) + out = capsys.readouterr().out + assert "gated" in out + assert list(tmp_path.iterdir()) == []