diff --git a/.coverage b/.coverage index 0fe9c8d..ceb6058 100644 Binary files a/.coverage and b/.coverage differ diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..dc026ef --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +version: 2 + +updates: + # The package pins no runtime dependencies beyond numpy/scipy, but the test + # and CI toolchain moves. Weekly PRs keep the reproduction honest against + # current versions rather than the ones that happened to be installed on the + # day of release. + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + day: monday + open-pull-requests-limit: 3 + commit-message: + prefix: deps + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: monday + open-pull-requests-limit: 3 + commit-message: + prefix: ci diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce15c7f..662695f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,13 @@ on: push: branches: [master] pull_request: + # A package whose claim is reproducibility should keep proving it. The + # weekly run re-asserts the paper's numbers against whatever numpy, scipy + # and Python resolve to that week, so drift surfaces here rather than in a + # reader's environment. + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 0000000..ef5cb51 --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,34 @@ +name: Draft PDF + +# Compiles paper.md with the Open Journals (inara) toolchain and uploads the +# rendered paper.pdf as a build artifact. This is the JOSS-documented preview +# workflow: it proves the paper compiles before submission. It does not contact +# JOSS and does not submit anything. + +on: + push: + paths: + - 'paper.md' + - 'paper.bib' + - '.github/workflows/draft-pdf.yml' + workflow_dispatch: + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + paper-path: paper.md + + - name: Upload rendered paper + uses: actions/upload-artifact@v4 + with: + name: paper + path: paper.pdf diff --git a/CHANGELOG.md b/CHANGELOG.md index 5be9b7e..decac7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,58 @@ # Changelog +## Unreleased + +Screen your own panel. **No change to package behavior:** +`tests/test_reproduces_paper.py` is untouched and the demo reproduces the same +numbers as 0.1.1. + +- `screen_dataframe(table, value=..., trailing_k=...)` screens a long table + directly and returns a `pit-screen-results` record. The table may be a + pandas or polars DataFrame, a pyarrow Table, or a dict of arrays — columns + are read by duck typing, so no dataframe library is imported and none is + required. Datetime arrival columns are accepted. +- `stores_from_frame` exposes the per-period stores the screen builds, for + callers who want to inspect or gate them individually. +- CLI: `--csv PATH --value COLUMN` screens a user panel instead of running the + demo, with `--period/--arrival/--size/--trailing-k/--threshold` and the + existing `--export` and `--badge`. CSV reading uses the standard library. +- The verdict convention is documented, including its noise floor: a signal is + susceptible if any screened period crossed the threshold, which on small + panels can fire on sampling noise. + +## 0.1.2 — unreleased (earlier work) + +Screen-result export. **No change to package behavior:** +`tests/test_reproduces_paper.py` is untouched and the demo reproduces the same +numbers as 0.1.1. + +- `--export PATH` writes a `pit-screen-results` v1.0 record — per-signal summary + statistics and the screen settings that produced the verdicts. Fully offline + (local file, no network call) and free of clocks, so the same screen exports + byte-identical bytes. +- `docs/results-schema.md`: the schema as a standalone versioned interchange + spec, free for other tools to adopt. +- New module `pit_release_gate.results`: `summarize_signal`, `screen_config`, + `build_results`, `validate_results`, `write_results`. Totals are derived, so + they cannot disagree with the per-signal rows. +- Still no telemetry of any kind, now enforced structurally rather than by + policy: no module in the package imports a transport, and a test asserts that + over every module — no background thread, no `atexit` hook, nothing to opt + out of. + +## 0.1.1 — 2026-08-16 + +Documentation and submission materials only. **No change to package behavior:** +the test suite, including `tests/test_reproduces_paper.py`, is untouched and the +demo reproduces the same numbers as 0.1.0. + +- `paper.md` and `paper.bib`: software paper prepared for submission to the + Journal of Open Source Software. +- README: added *Statement of need*, *API overview*, and *Community guidelines* + sections. +- CI: `draft-pdf.yml` builds a preview PDF of `paper.md` with the Open Journals + toolchain and uploads it as a build artifact. + ## 0.1.0 — 2026-08-15 First public release. diff --git a/README.md b/README.md index 0bd57e9..2964ff6 100644 --- a/README.md +++ b/README.md @@ -3,22 +3,50 @@ [![CI](https://github.com/MaxWellApexLab/pit-release-gate/actions/workflows/ci.yml/badge.svg)](https://github.com/MaxWellApexLab/pit-release-gate/actions/workflows/ci.yml) [![codecov](https://codecov.io/gh/MaxWellApexLab/pit-release-gate/branch/master/graph/badge.svg)](https://codecov.io/gh/MaxWellApexLab/pit-release-gate) [![PyPI](https://img.shields.io/pypi/v/pit-release-gate)](https://pypi.org/project/pit-release-gate/) -[![Python](https://img.shields.io/pypi/pyversions/pit-release-gate)](https://pypi.org/project/pit-release-gate/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) +[![PIT Hygiene](https://img.shields.io/badge/PIT%20Hygiene-pledged-2ea44f)](https://github.com/MaxWellApexLab/pit-hygiene) Completeness-aware release control for staggered-arrival cross-sectional data. -When the entities of a cross-section report on staggered dates — companies filing -financial statements are the canonical case — any same-period cross-sectional signal -computed before the last filer arrives is estimated from an incomplete, and possibly -*selectively* incomplete, cross-section. If filing timing depends on the very -disturbance the signal measures, releasing early produces a systematic bias -(incomplete-cross-section leakage), while a blanket wait-for-the-deadline rule removes -the bias at a timeliness cost paid by every signal, biased or not. `pit-release-gate` -measures each signal's susceptibility to this bias — a disturbance-conditional partial -correlation fitted honestly on prior *completed* periods — and grades the required -completeness per signal, so benign signals release early and susceptible signals are -withheld until enough of the cross-section has arrived to suppress the bias. +**[Audit registry](https://github.com/MaxWellApexLab/pit-audit-registry) · +[Pledge](https://github.com/MaxWellApexLab/pit-hygiene) · +[Papers](#papers) · +[Result schema](docs/results-schema.md)** + +Rebuilt from as-filed SEC EDGAR filings, on observed filing dates, **7 of 14 +standard fundamental signals are contaminated** by incomplete-cross-section +leakage — [measured, reproducibly, in the companion registry](https://github.com/MaxWellApexLab/pit-audit-registry/blob/main/methodology/2026-08_sec-edgar/report.md). +This tool measures each signal's susceptibility *before* release and withholds +only the signals that need it. The screen costs one `fit_trailing` call per +signal. + +![Known-ground-truth demo: naive release is biased exactly when the leak is strong; the gate routes that signal to the deadline and its bias is exactly zero, while benign signals still release at ~36% completeness](https://raw.githubusercontent.com/MaxWellApexLab/pit-release-gate/master/docs/assets/demo_bias.png) + +*The shipped fixed-seed demo, where the right answer is planted: naive early +release carries a systematic bias of −0.386 on the strong-leak signal; the gate +routes it to the deadline (bias exactly 0.0) while releasing the two benign +signals at 36–39% completeness. `tests/test_reproduces_paper.py` pins these +numbers; [`tools/make_readme_chart.py`](tools/make_readme_chart.py) redraws +this figure from the live demo.* + +**You need this if:** + +- you build **same-period cross-sectional signals** — industry-adjusted ratios, + cross-sectional ranks, peer medians — on entities that report on their own + schedule; +- you **rebuild panels from as-filed sources** (EDGAR `companyfacts`, raw + filings) instead of using a vendor's curated release; +- you own a **feature-store pipeline** where an as-of join reads whatever has + arrived by *t*; +- you want a **per-signal susceptibility number** published next to every + released signal, the way a standard error is. + +**Not for you if** you are hunting look-ahead bugs in a backtest engine, or your +data has no arrival times — that is a different failure mode +([scope statement](https://github.com/MaxWellApexLab/pit-audit-registry#what-gets-screened)). + +**No telemetry — structurally, not merely by default:** no module in this +package imports a transport, and [a test enforces that over every module](tests/test_export.py). ## Install @@ -56,6 +84,81 @@ values, and per-entity filing-arrival times, then call `ReleaseController.decide(store, t)` at each evaluation time — it returns `WITHHOLD`, `REWEIGHT_RELEASE`, or `RELEASE` plus the released values. +## Screen your own panel + +One long table — one row per (entity, period) — and one call. The table can be +a pandas DataFrame, a polars DataFrame, a pyarrow Table, or a dict of arrays: +columns are read by duck typing, so the screen itself depends on no dataframe +library. + +```python +from pit_release_gate import screen_dataframe + +record = screen_dataframe( + panel, # columns: period, arrival, value, size + value=["accruals", "roa"], # one signal or several + trailing_k=5, # fit on 5 prior COMPLETED periods, then freeze +) +for s in record["signals"]: + print(s["name"], s["verdict"], s["periods_flagged"], "/", s["periods_screened"]) +``` + +Or without writing any Python: + +```bash +pit-release-gate --csv panel.csv --value accruals --value roa --export results.json +``` + +```text +screened panel.csv: 2 signal(s), trailing_k=5, threshold=0.1 + signal periods flagged mean rho max |rho| phi_req verdict + accruals 7 7 -0.8721 0.8841 1.000 susceptible + roa 7 0 +0.0268 0.0564 0.377 benign + totals: 1 benign, 1 susceptible, 14 signal-cycles screened +``` + +This is the same frozen protocol the [published audits](https://github.com/MaxWellApexLab/pit-audit-registry) +use: the estimate applied to a period is never fitted on that period, and the +first `trailing_k` periods are used for fitting only. The output is a +`pit-screen-results` v1.0 record — the file a *screened with* badge should +point at. + +**One caveat, stated up front.** A verdict fires if any single screened period +crosses the threshold, so it inherits that period's sampling noise +(roughly `1 / sqrt(trailing_k x entities_per_period)`). On a small panel, +a reading just over the threshold may be noise. Establish your noise floor +first — screen a signal you expect to be unexposed, or shuffle arrival order +within periods and re-screen — before treating a marginal verdict as a finding. + +## What this catches that your current tools don't + +| guarantee | as-of join / bitemporal store | purged & embargoed CV | **pit-release-gate** | +|---|---|---|---| +| No value was read before it was available | ✅ | — | assumed as input (bring your arrival times) | +| Train and test don't overlap through time | — | ✅ | — | +| The **set of entities** present at *t* was not selected on the disturbance | ❌ | ❌ | **✅ measured per signal (ρ̂), gated per signal** | + +A point-in-time-correct join over an incomplete cross-section is a correct join +over a biased sample. [Statement of need](#statement-of-need) has the full +argument; the +[as-of join methodology page](https://github.com/MaxWellApexLab/pit-audit-registry/blob/main/methodology/2026-08_feast-pit-join/report.md) +has the measured demonstration. + +## API overview + +Five public components, all importable from the top-level `pit_release_gate` package: + +| component | what it does | +|---|---| +| [`AsOfDataStore`](src/pit_release_gate/store.py) | Holds one period's as-filed records for a cross-sectional group: design matrix, signal values, and a filing-arrival time per entity. | +| [`CompletenessMonitor`](src/pit_release_gate/monitor.py) | Reports the arrived fraction at an evaluation time, plus a composition-shift gauge for the arrived subset. | +| [`SusceptibilityGate`](src/pit_release_gate/gate.py) | Estimates ρ̂, the partial correlation between filing latency and the complete-cross-section residual given observables. `fit_trailing` enforces the honest-estimation contract: prior *completed* periods only. | +| [`PropensityReweighter`](src/pit_release_gate/reweight.py) | Inverse-filing-propensity weights. Included to make a negative result executable: reweighting on observables corrects composition, but cannot remove selection on the disturbance. | +| [`ReleaseController`](src/pit_release_gate/controller.py) | Maps \|ρ̂\| to a required completeness `φ_req = min(1, φ_min + κ·\|ρ̂\|)` and returns `WITHHOLD` / `REWEIGHT_RELEASE` / `RELEASE` at each evaluation time. | + +`make_group`, `run_demo` and `demo` ([`simulate.py`](src/pit_release_gate/simulate.py)) +generate and run the known-ground-truth worked example described below. + ## The known-ground-truth demo The package ships a self-contained worked example with a *planted* leakage strength, @@ -65,6 +168,16 @@ so the right answer is known exactly and no licensed data is needed: pit-release-gate # or: python -m pit_release_gate ``` +Real output, abridged to the signal the gate exists for: + +```text +Strong-leak (c_a=1.0, c_x=0.7) rho_trailing=-0.868 (fitted ex ante) (SUSCEPTIBLE -> wait) + policy comp% [95%CI] flip% [95%CI] biasB(signed) [CI] route + naive 35 ± 0.0 63.6 ± 2.8 -0.386 ±0.037 naive + deadline 100 ± 0.0 0.0 ± 0.0 +0.000 ±0.000 deadline + gated 100 ± 0.0 0.0 ± 0.0 +0.000 ±0.000 gated(phi_req=1.00) <-- gated +``` + It compares five release policies (`naive`, `threshold`, `reweight`, `deadline`, `gated`) on four signal types. Headline behavior: @@ -80,6 +193,67 @@ A sensitivity sweep of the policy slope κ shows the timeliness–bias dial: κ = 2.0 → 100% (bias exactly 0). The demo is deterministic (fixed seed), and `tests/test_reproduces_paper.py` asserts these numbers. +## Statement of need + +When the entities of a cross-section report on staggered dates — companies filing +financial statements are the canonical case — any same-period cross-sectional signal +computed before the last filer arrives is estimated from an incomplete, and possibly +*selectively* incomplete, cross-section. If filing timing depends on the very +disturbance the signal measures, releasing early produces a systematic bias +(incomplete-cross-section leakage), while a blanket wait-for-the-deadline rule removes +the bias at a timeliness cost paid by every signal, biased or not. + +Point-in-time discipline in ML pipelines currently rests on tooling that answers one +question: *was this value readable at time t?* Feature-store as-of joins, bitemporal +and vintage-aware storage, and purged or embargoed cross-validation all enforce +read-time correctness, and they do it well. + +None of them answers a second question: given that every value read was legitimately +readable, was the *set* of entities that had reported by t a selected sample? An +as-of join over an incomplete cross-section is a correct join over a biased sample. +The two failures need different remedies — the first is fixed by timestamp hygiene, +the second only by waiting or by an explicit correction. Researchers building +cross-sectional signals on staggered-arrival panels have had no routine, per-signal +screen for the second. `pit-release-gate` is that screen, plus the release controller +that acts on it: one `fit_trailing` call per signal, so reporting a susceptibility +estimate alongside a released signal costs about as much as reporting a standard +error. + +## Export your screen result + +```bash +pit-release-gate --export results.json +``` + +writes a `pit-screen-results` v1.0 record: per screened signal, how many periods +were screened, how many the measure flagged, mean and max ρ̂, the required +completeness that was assigned, and the verdict (`benign` / `susceptible`), plus +the five settings that produced those verdicts. It is the file a *screened with* +badge should point at. The format is a standalone versioned interchange spec — +[`docs/results-schema.md`](docs/results-schema.md) — that any other tool is free +to emit or consume. + +**`--export` is fully offline**: it writes a local file and makes no network +call. Records carry summary statistics only — never input rows, file paths, +usernames, hostnames, or any environment detail beyond the tool version — and no +clock is read while building one, so the same screen always exports byte-identical +bytes. + +Where the record goes afterwards is entirely your business — commit it next to +your badge, publish it, or keep it. This tool does not send it anywhere. There is +no background thread, no `atexit` hook, no anonymous usage counter, and nothing +to opt out of. + +Programmatic use, for screens on your own data: + +```python +from pit_release_gate import build_results, screen_config, summarize_signal, validate_results + +sig = summarize_signal("accruals", rhos=[...], phi_reqs=[...], rho_threshold=0.10) +record = build_results([sig], screen_config(0.10, 0.35, 1.0, trailing_k=8, min_entities=6)) +assert validate_results(record) == [] +``` + ## Papers The method and its evaluation are developed in three public papers: @@ -95,6 +269,31 @@ The method and its evaluation are developed in three public papers: This package is the reference implementation of paper 3's release controller; its demo reproduces the paper's controlled experiment. +## Badge + +If you have run the susceptibility screen on your own data — whatever the result — you +are welcome to say so: + +```markdown +[![screened with pit-release-gate](https://img.shields.io/badge/screened%20with-pit--release--gate-blue)](https://github.com/MaxWellApexLab/pit-release-gate) +``` + +The badge reads **screened with**, not *passed* — it states that the screen was run, the +same way a formatter badge states that the formatter was run. A benign result and a +susceptible result are equally worth badging; the second one arguably more, because it +means the screen found something and your pipeline now waits for it. + +**Make it point at something.** A badge is worth reading only if there is evidence behind +it. Commit your screen output — `pit-release-gate --export results.json` produces exactly +that file: which signals came out benign, which came out susceptible, and the required +completeness each was assigned — and link the badge at it rather +than at this repo. A worked example is the OSAP screen in the +[PIT audit registry](https://github.com/MaxWellApexLab/pit-audit-registry/blob/main/audits/2026-08_osap/report.md). + +**Related:** the [PIT Hygiene pledge](https://github.com/MaxWellApexLab/pit-hygiene) is a +broader, tool-neutral statement about how a staggered-arrival pipeline is built; this badge +is the narrower statement that this particular screen was run. + ## Cite this See [`CITATION.cff`](CITATION.cff). If you use this software, please cite paper 3: @@ -110,6 +309,15 @@ See [`CITATION.cff`](CITATION.cff). If you use this software, please cite paper } ``` +## Community guidelines + +- **Report a bug or request a feature:** open an issue at + [github.com/MaxWellApexLab/pit-release-gate/issues](https://github.com/MaxWellApexLab/pit-release-gate/issues). +- **Contribute:** see [CONTRIBUTING.md](CONTRIBUTING.md) for the development setup, + test requirements, and pull-request process. +- **Get help:** open an issue with a minimal reproducing example, or email + maxwellapexlab@proton.me. + ## License MIT — see [LICENSE](LICENSE). diff --git a/docs/assets/demo_bias.png b/docs/assets/demo_bias.png new file mode 100644 index 0000000..6598180 Binary files /dev/null and b/docs/assets/demo_bias.png differ diff --git a/docs/results-schema.md b/docs/results-schema.md new file mode 100644 index 0000000..e2fb704 --- /dev/null +++ b/docs/results-schema.md @@ -0,0 +1,214 @@ +# `pit-screen-results` — screen-result interchange format + +**Schema name:** `pit-screen-results` +**Version:** `1.0` +**Media type:** `application/json` +**Reference implementation:** [`src/pit_release_gate/results.py`](../src/pit_release_gate/results.py) +**Status:** stable + +A `pit-screen-results` record states what an incomplete-cross-section +susceptibility screen found: for each signal that was screened, how many periods +it was screened over, how susceptible it looked, how much completeness the +release controller therefore required, and whether the verdict was **benign** or +**susceptible**. It is the evidence a *screened with* badge should point at. + +The format is deliberately small. It is not a report format, not a results +database, and not a telemetry envelope. + +## Design rules + +These three rules are what make the record safe to commit to a public +repository or hand to a third party, and they are binding on any producer: + +1. **Summary statistics only.** A record carries per-signal aggregates. + It must never carry input rows, entity identifiers, residuals, file paths, + usernames, hostnames, or any environment detail beyond the producing tool's + name and version. +2. **Offline by construction.** Writing a record is local file I/O. In the + reference implementation the module that builds and writes records imports + no transport machinery at all; sending one is a separate, explicitly + requested act (see [Submission envelope](#submission-envelope-non-normative)). +3. **No clocks.** A producer must not read the system clock while building a + record: two runs of the same screen must produce byte-identical bytes. If a + date belongs in the record, the caller passes it in (`date`, below). + +## Top-level fields + +| field | type | required | meaning | +|---|---|---|---| +| `schema` | string | yes | Always `"pit-screen-results"`. Identifies the format, not the producer. | +| `schema_version` | string | yes | `"1.0"` for this document. | +| `tool` | string | yes | Name of the producing tool, e.g. `"pit-release-gate"`. Any tool may emit this format under its own name. | +| `tool_version` | string | yes | Version of the producing tool. The only environment detail permitted anywhere in the record. | +| `config` | object | yes | The screen settings that determine the verdicts. See below. | +| `signals` | array of objects | yes | One entry per screened signal, non-empty. See below. | +| `totals` | object | yes | Aggregates over `signals`, derived — never independently asserted. | +| `date` | string | no | Caller-supplied date (ISO 8601 `YYYY-MM-DD` recommended). Absent unless the caller passed one; producers must not fill it in from a clock. | + +## `config` + +The five settings a reader needs in order to interpret a verdict. All are +required. + +| field | type | meaning | +|---|---|---| +| `rho_threshold` | number | A signal is susceptible when \|ρ̂\| exceeds this. | +| `phi_min` | number | Completeness floor: the earliest completeness at which anything is released. | +| `kappa` | number | Slope of the graded requirement `phi_req = min(1, phi_min + kappa·\|ρ̂\|)`. | +| `trailing_k` | integer | Number of prior **completed** periods the susceptibility estimate was fitted on. | +| `min_entities` | integer | Minimum arrived entity count below which nothing is released. | + +## `signals[]` + +| field | type | meaning | +|---|---|---| +| `name` | string | Caller-chosen signal name. Must not encode a path, a file name, or an identity. | +| `periods_screened` | integer ≥ 0 | Number of periods the signal was screened over. | +| `periods_flagged` | integer | Of those, how many had a *per-period* \|ρ̂\| above `rho_threshold`. Must be ≤ `periods_screened`. | +| `mean_rho` | number | Mean of the per-period susceptibility estimates (signed). | +| `max_abs_rho` | number | Largest per-period \|ρ̂\| observed. | +| `mean_phi_req` | number | Mean required completeness the controller assigned across those periods. | +| `verdict` | string | `"benign"` or `"susceptible"` — no other value is valid. | + +**`periods_flagged` is a noise gauge, not the verdict.** A single period's ρ̂ is +a small-sample estimate and will cross the threshold now and then on a perfectly +benign signal; that is exactly why a screen pools over `trailing_k` completed +periods before deciding. In the worked example below, `clean` has 3 of 5 periods +flagged and is still — correctly — `benign`: its pooled estimate is far under the +threshold, while `max_abs_rho = 0.164` records how noisy a single period was. +Read `verdict` for the decision, `periods_flagged` and `max_abs_rho` for how +stable that decision was. + +## `totals` + +Derived from `signals`; a producer computes them rather than accepting them, and +a consumer may treat a mismatch as a corrupt record. + +| field | type | meaning | +|---|---|---| +| `signal_cycles` | integer | Sum of `periods_screened` over all signals — the total screening work the record represents. | +| `signals_benign` | integer | Number of signals with `verdict == "benign"`. | +| `signals_susceptible` | integer | Number with `verdict == "susceptible"`. | + +## Validity + +A record is valid when all of the following hold. The reference implementation +is `validate_results(obj) -> list[str]`, which returns one human-readable string +per problem and an empty list for a valid record. + +1. The record is a JSON object with `schema == "pit-screen-results"`. +2. `schema_version` is present and known to the reader (`"1.0"`). +3. Every required top-level, `config`, `signals[]`, and `totals` field is present. +4. Every `verdict` is `"benign"` or `"susceptible"`. +5. `totals.signal_cycles` equals the sum of `periods_screened` over `signals`, + and the two verdict counts equal the corresponding counts in `signals`. +6. For every signal, `0 <= periods_flagged <= periods_screened`. + +Validation is a check on the record, not on the science: a record can be +perfectly valid and report a thoroughly susceptible pipeline. That is the point. + +## Worked example + +Produced by `pit-release-gate --train 3 --eval 5 --export results.json` +(reduced settings, so the file fits here): + +```json +{ + "schema": "pit-screen-results", + "schema_version": "1.0", + "tool": "pit-release-gate", + "tool_version": "0.1.1", + "config": { + "rho_threshold": 0.1, + "phi_min": 0.35, + "kappa": 1.0, + "trailing_k": 3, + "min_entities": 6 + }, + "signals": [ + { + "name": "clean", + "periods_screened": 5, + "periods_flagged": 3, + "mean_rho": -0.018502844357097172, + "max_abs_rho": 0.16437940321670516, + "mean_phi_req": 0.40870844204483336, + "verdict": "benign" + }, + { + "name": "composition", + "periods_screened": 5, + "periods_flagged": 1, + "mean_rho": 0.054842113923429116, + "max_abs_rho": 0.1916906690936213, + "mean_phi_req": 0.3622517271310478, + "verdict": "benign" + }, + { + "name": "mild_leak", + "periods_screened": 5, + "periods_flagged": 5, + "mean_rho": -0.5050242025985237, + "max_abs_rho": 0.6073065398308837, + "mean_phi_req": 0.8244168498651367, + "verdict": "susceptible" + }, + { + "name": "strong_leak", + "periods_screened": 5, + "periods_flagged": 5, + "mean_rho": -0.868283463437835, + "max_abs_rho": 0.8934707972913439, + "mean_phi_req": 1.0, + "verdict": "susceptible" + } + ], + "totals": { + "signal_cycles": 20, + "signals_benign": 2, + "signals_susceptible": 2 + } +} +``` + +How to read it: four signals were screened over five periods each (20 signal +cycles). Two came out benign and release at roughly the completeness floor +(`mean_phi_req` ≈ 0.36–0.41). `mild_leak` is susceptible and is held until 82% +of its cross-section has arrived; `strong_leak` is held to the +deadline-complete cross-section (`mean_phi_req == 1.0`). + +## Transport (deliberately unspecified) + +This spec describes a **file format**, not a protocol. `pit-release-gate` writes +a record with `--export` and does nothing else with it: the package contains no +submission path and imports no transport at all. Where a record travels — a +commit next to a badge, an artifact in CI, an attachment, an endpoint of your own +— is the emitter's choice and outside this document. + +If you build something that receives records, validate them the way §"Validity +rules" describes and treat everything in a record as publishable. A record +carries summary statistics only: it is designed so that publishing one leaks +nothing about the data it was computed from. + +## Versioning + +`schema_version` is `MAJOR.MINOR`. A **minor** bump only adds optional fields; +a reader for `1.0` may ignore fields it does not know and keep working. A +**major** bump may remove or repurpose fields, and readers should refuse a major +version they do not know rather than guess. The schema name never changes +meaning: a document identified as `pit-screen-results` always means a screen +result in the sense described here. + +## Adoption + +**This format is free for any tool to emit or consume**, with no attribution +requirement, no coordination with this project, and no compatibility obligation +in either direction. A screen result is more useful when it is comparable across +tools, so put your own name in `tool`, keep `schema` and `schema_version` as +specified, and the record will read the same everywhere. Extensions are welcome +under a namespaced key of your own (e.g. `"x_yourtool"`); a `1.0` reader must +ignore what it does not recognize. + +If you extend the format in a way you think belongs in the core schema, open an +issue at +[github.com/MaxWellApexLab/pit-release-gate](https://github.com/MaxWellApexLab/pit-release-gate/issues). diff --git a/paper.bib b/paper.bib new file mode 100644 index 0000000..42bf9bc --- /dev/null +++ b/paper.bib @@ -0,0 +1,84 @@ +% paper.bib — pit-release-gate (JOSS) +% +% PROVENANCE: every field below was taken verbatim from the resolved DOI record +% on 2026-08-16. figshare entries from api.datacite.org; the rest from +% api.crossref.org. Nothing in this file was written from memory. +% 10.6084/m9.figshare.32952482 -> DataCite, resourceTypeGeneral "Preprint" +% 10.6084/m9.figshare.33061955 -> DataCite, resourceTypeGeneral "Preprint" +% 10.6084/m9.figshare.33158615 -> DataCite, resourceTypeGeneral "Preprint" +% 10.1145/2382577.2382579 -> Crossref +% 10.1016/j.patter.2023.100804 -> Crossref +% 10.2307/1912352 -> Crossref +% +% Two deliberate fidelity notes (do NOT "fix" these without re-resolving the DOI): +% * The figshare records register the first creator as "Kuan Ta Wu" (no hyphen) +% and carry no ORCIDs. That is what the DOI resolves to, so that is what is +% recorded here. See the submission-day checklist for the figshare metadata fix. +% * Crossref stores only the first page (153) for Heckman (1979); the record does +% not carry an end page, so none is invented here. + +@misc{wu2026a, + author = {Wu, Kuan Ta and Wu, Kuan-I}, + title = {{Correct-by-Construction Factor Computation: A Verifiably Point-in-Time Engine for Tradeable Signals}}, + year = {2026}, + publisher = {figshare}, + note = {Preprint}, + doi = {10.6084/m9.figshare.32952482}, + url = {https://doi.org/10.6084/m9.figshare.32952482} +} + +@misc{wu2026b, + author = {Wu, Kuan Ta and Wu, Kuan-I}, + title = {{Measuring Incomplete-Cross-Section Leakage: A Matched Placebo, a Susceptibility Screen, and Evidence from Taiwan and US As-Filed Data that the Channel Is Benign and Correctable}}, + year = {2026}, + publisher = {figshare}, + note = {Preprint}, + doi = {10.6084/m9.figshare.33061955}, + url = {https://doi.org/10.6084/m9.figshare.33061955} +} + +@misc{wu2026c, + author = {Wu, Kuan Ta and Wu, Kuan-I}, + title = {{Susceptibility-Graded Release Control: Preventing Incomplete-Cross-Section Leakage in Financial Machine-Learning Pipelines without a Blanket Timeliness Penalty}}, + year = {2026}, + publisher = {figshare}, + note = {Preprint}, + doi = {10.6084/m9.figshare.33158615}, + url = {https://doi.org/10.6084/m9.figshare.33158615} +} + +@article{kaufman2012, + author = {Kaufman, Shachar and Rosset, Saharon and Perlich, Claudia and Stitelman, Ori}, + title = {{Leakage in data mining: Formulation, detection, and avoidance}}, + journal = {ACM Transactions on Knowledge Discovery from Data}, + volume = {6}, + number = {4}, + pages = {1--21}, + year = {2012}, + publisher = {Association for Computing Machinery (ACM)}, + doi = {10.1145/2382577.2382579} +} + +@article{kapoor2023, + author = {Kapoor, Sayash and Narayanan, Arvind}, + title = {{Leakage and the reproducibility crisis in machine-learning-based science}}, + journal = {Patterns}, + volume = {4}, + number = {9}, + pages = {100804}, + year = {2023}, + publisher = {Elsevier BV}, + doi = {10.1016/j.patter.2023.100804} +} + +@article{heckman1979, + author = {Heckman, James J.}, + title = {{Sample Selection Bias as a Specification Error}}, + journal = {Econometrica}, + volume = {47}, + number = {1}, + pages = {153}, + year = {1979}, + publisher = {JSTOR}, + doi = {10.2307/1912352} +} diff --git a/paper.md b/paper.md new file mode 100644 index 0000000..5c3231a --- /dev/null +++ b/paper.md @@ -0,0 +1,115 @@ +--- +title: 'pit-release-gate: completeness-aware release control for staggered-arrival cross-sectional data' +tags: + - Python + - data leakage + - point-in-time data + - machine learning pipelines + - quantitative finance +authors: + - name: Kuan-Ta Wu + orcid: 0009-0006-0529-8709 + affiliation: 1 +affiliations: + - name: Max Well Apex LLC, NY, United States + index: 1 +date: 1 September 2026 +bibliography: paper.bib +--- + +# Summary + +Many empirical cross-sections are assembled from records that arrive on staggered +dates. Companies filing quarterly financial statements are the canonical case: a +fiscal period ends on a single date, but individual filings land over the following +weeks, up to a statutory deadline. Any same-period cross-sectional signal computed +before the last filer has arrived is therefore estimated from an incomplete +cross-section. Incompleteness is harmless when arrival timing is unrelated to what +the signal measures. It is not harmless when filing timing depends on the very +disturbance the signal is meant to capture: the early-arriving subset is then +selected on the estimand itself, and the released signal carries a systematic bias. +The standard defence — wait for the deadline before releasing anything — removes +the bias but charges a timeliness penalty to every signal, including those that +were never at risk. + +`pit-release-gate` replaces that blanket rule with a per-signal measurement. It +estimates each signal's susceptibility to this bias — a disturbance-conditional +partial correlation, fitted only on prior *completed* periods — and converts the +estimate into the cross-sectional completeness that the signal must reach before it +may be released. Benign signals release as soon as a minimum completeness floor is +met; susceptible signals are withheld until enough of the cross-section has arrived +to suppress the bias, up to the deadline. + +# Statement of need + +Point-in-time discipline in machine-learning pipelines currently rests on tooling +that answers one question: *was this value readable at time $t$?* Feature-store +as-of joins, bitemporal and vintage-aware storage, and purged or embargoed +cross-validation all enforce read-time correctness, and they do it well. + +None of them answers a second question: given that every value read was legitimately +readable, was the *set* of entities that had reported by $t$ a selected sample? An +as-of join over an incomplete cross-section is a correct join over a biased sample. +The distinction matters because the two failures need different remedies — the +first is fixed by timestamp hygiene, the second only by waiting or by an explicit +correction. Existing leakage taxonomies [@kaufman2012] and recent surveys of leakage +in machine-learning-based science [@kapoor2023] name this family of problems, and +the mechanism is a selection problem in the classical sense [@heckman1979], but +researchers have had no routine, per-signal screen they can execute inside a +pipeline. + +`pit-release-gate` provides one. The susceptibility measure, the grading rule that +turns it into a release threshold, and their evaluation on as-filed data are +developed in three publicly available preprints [@wu2026a; @wu2026b; @wu2026c]; this +package is the reference implementation of the release controller of the third. It +is aimed at researchers who build cross-sectional signals on staggered-arrival +panels — most immediately in empirical accounting and quantitative finance, but the +same arrival structure appears wherever administrative records backfill after a +reporting period closes. The design goal is that the screen costs one function call +per signal, so that reporting a susceptibility estimate alongside a released signal +becomes as ordinary as reporting a standard error. + +# Functionality + +The public API has five components: + +- `AsOfDataStore` holds the as-filed records for one period and cross-sectional + group: a design matrix, the constructed signal values, and a filing-arrival time + per entity. +- `CompletenessMonitor` reports the arrived fraction at an evaluation time, plus a + composition-shift gauge for the arrived subset. +- `SusceptibilityGate` estimates $\hat{\rho}$, the partial correlation between + filing latency and the complete-cross-section residual, conditional on + observables. Its `fit_trailing` method enforces an honest-estimation contract: + $\hat{\rho}$ is fitted on prior completed periods only, never on the period being + gated, whose cross-section is by definition still incomplete. +- `PropensityReweighter` supplies inverse-filing-propensity weights. It is included + to make a negative result executable: reweighting on observables corrects + composition shift but cannot remove selection on the disturbance, which is why the + controller grades on completeness rather than on reweighting. +- `ReleaseController` maps $|\hat{\rho}|$ to a required completeness, + $\varphi_{\mathrm{req}} = \min(1,\ \varphi_{\min} + \kappa|\hat{\rho}|)$, and + returns at each evaluation time one of `WITHHOLD`, `REWEIGHT_RELEASE` or + `RELEASE`, together with the released values. + +The package ships a self-contained worked example in which the strength of selection +on the disturbance is *planted*, so the correct decision is known exactly and no +licensed data is required. Running `pit-release-gate` compares five release policies +across four signal types and prints the resulting timeliness and bias. The run is +deterministic under a fixed seed and the test suite asserts its headline numbers, so +a reader can confirm in one command that the gate releases benign signals at roughly +a third of the cross-section, while holding a strongly selected signal to the +complete cross-section, where its bias is exactly zero. + +Installation is `pip install pit-release-gate`. The package requires Python 3.10 or +later and depends only on NumPy, pandas and SciPy; it is MIT licensed and tested on +Linux and Windows against Python 3.10 and 3.13. + +# Acknowledgements + +The methodology preprints cited above are co-authored with Kuan-I Wu; the software +described here was designed and written solely by the author. The author used +AI-assisted drafting tools in preparing this manuscript; all technical content, +experiments, and claims were designed, executed, and verified by the author. + +# References diff --git a/pyproject.toml b/pyproject.toml index 11f6233..957335f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "pit-release-gate" -version = "0.1.0" -description = "Completeness-aware release control for staggered-arrival cross-sectional data: a susceptibility-graded gate that blocks incomplete-cross-section leakage without a blanket timeliness penalty" +version = "0.1.1" +description = "Screen quant/ML feature pipelines for look-ahead bias from late-arriving data: measure each signal's leakage susceptibility and gate release until the cross-section is safe" readme = "README.md" requires-python = ">=3.10" license = "MIT" @@ -23,6 +23,11 @@ keywords = [ "selection-bias", "quantitative-finance", "ml-pipelines", + "lookahead-bias", + "quantitative-finance", + "backtesting", + "feature-store", + "data-snooping", ] classifiers = [ "Development Status :: 4 - Beta", @@ -46,10 +51,13 @@ dependencies = [ Homepage = "https://github.com/MaxWellApexLab/pit-release-gate" Repository = "https://github.com/MaxWellApexLab/pit-release-gate" Paper = "https://doi.org/10.6084/m9.figshare.33158615" +Changelog = "https://github.com/MaxWellApexLab/pit-release-gate/blob/master/CHANGELOG.md" +"Audit Registry" = "https://github.com/MaxWellApexLab/pit-audit-registry" +"Issue Tracker" = "https://github.com/MaxWellApexLab/pit-release-gate/issues" [project.optional-dependencies] dev = [ - "pytest>=7.0", + "pytest>=9.1.1", "pytest-cov>=4.0", ] diff --git a/src/pit_release_gate/__init__.py b/src/pit_release_gate/__init__.py index f02846e..2345091 100644 --- a/src/pit_release_gate/__init__.py +++ b/src/pit_release_gate/__init__.py @@ -9,6 +9,8 @@ Public API ---------- +screen_dataframe screen a long table (pandas/polars/dict) -> results record +stores_from_frame the per-period stores that screen builds, for inspection AsOfDataStore staggered-arrival records for one (period, group) CompletenessMonitor arrived-fraction and composition-shift gauges PropensityReweighter optional IPW composition-correction module @@ -18,18 +20,45 @@ make_group one synthetic staggered-arrival cross-section (known truth) run_demo the full known-ground-truth worked example (returns dict) demo same, console-table form +build_results assemble a pit-screen-results record (summary stats only) +validate_results check such a record; returns a list of problems +badge_snippet README badge markdown for a completed screen run """ +# defined first: results.tool_version() reads it while the submodules below +# are still importing +__version__ = "0.1.1" + from .controller import ReleaseController, ReleaseDecision +from .frame import screen_dataframe, stores_from_frame from .gate import SusceptibilityGate from .monitor import CompletenessMonitor +from .results import ( + SCHEMA, + SCHEMA_VERSION, + build_results, + screen_config, + summarize_signal, + validate_results, + write_results, +) from .reweight import PropensityReweighter -from .simulate import DEMO_POLICIES, DEMO_SIGNALS, SEED, demo, main, make_group, run_demo +from .simulate import ( + DEMO_POLICIES, + DEMO_SIGNALS, + SEED, + badge_snippet, + demo, + main, + make_group, + results_from_demo, + run_demo, +) from .store import AsOfDataStore -__version__ = "0.1.0" - __all__ = [ "AsOfDataStore", + "screen_dataframe", + "stores_from_frame", "CompletenessMonitor", "PropensityReweighter", "SusceptibilityGate", @@ -39,8 +68,17 @@ "run_demo", "demo", "main", + "badge_snippet", "SEED", "DEMO_SIGNALS", "DEMO_POLICIES", + "SCHEMA", + "SCHEMA_VERSION", + "summarize_signal", + "screen_config", + "build_results", + "validate_results", + "write_results", + "results_from_demo", "__version__", ] diff --git a/src/pit_release_gate/controller.py b/src/pit_release_gate/controller.py index 070a15d..6cded86 100644 --- a/src/pit_release_gate/controller.py +++ b/src/pit_release_gate/controller.py @@ -10,6 +10,11 @@ from .reweight import PropensityReweighter from .store import AsOfDataStore +#: Minimum number of arrived entities before anything is released; below it a +#: cross-section is too small for the residualization to mean much. Named here +#: so a screen record can report the floor it ran under. +MIN_ENTITIES = 6 + def _ols_resid(X: np.ndarray, y: np.ndarray, w: np.ndarray = None) -> np.ndarray: if w is None: @@ -70,11 +75,11 @@ def decide(self, store: AsOfDataStore, t: float, policy: str = 'gated') -> Relea # fixed baseline policies (for comparison / fallback configs) ---- if policy == 'naive': - if comp < self.phi_min or m.sum() < 6: + if comp < self.phi_min or m.sum() < MIN_ENTITIES: return ReleaseDecision('WITHHOLD', t, comp, policy=policy) return self._emit(store, t, m, None, 'RELEASE', policy) if policy == 'threshold': - if comp < self.phi_high or m.sum() < 6: + if comp < self.phi_high or m.sum() < MIN_ENTITIES: return ReleaseDecision('WITHHOLD', t, comp, policy=policy) return self._emit(store, t, m, None, 'RELEASE', policy) if policy == 'deadline': @@ -82,7 +87,7 @@ def decide(self, store: AsOfDataStore, t: float, policy: str = 'gated') -> Relea return ReleaseDecision('WITHHOLD', t, comp, policy=policy) return self._emit(store, t, m, None, 'RELEASE', policy) if policy == 'reweight': - if comp < self.phi_min or m.sum() < 6: + if comp < self.phi_min or m.sum() < MIN_ENTITIES: return ReleaseDecision('WITHHOLD', t, comp, policy=policy) w = self.reweighter.weights(store, t) return self._emit(store, t, m, w, 'REWEIGHT_RELEASE', policy) @@ -95,7 +100,7 @@ def decide(self, store: AsOfDataStore, t: float, policy: str = 'gated') -> Relea # bias, up to the statutory deadline (completeness=1.0). rho = self.gate.rho_hat(store) phi_req = self.required_completeness(rho) - if (comp >= phi_req or t >= 1.0) and m.sum() >= 6: + if (comp >= phi_req or t >= 1.0) and m.sum() >= MIN_ENTITIES: return self._emit(store, t, m, None, 'RELEASE', f'gated(phi_req={phi_req:.2f})') return ReleaseDecision('WITHHOLD', t, comp, policy=f'gated(phi_req={phi_req:.2f})') diff --git a/src/pit_release_gate/frame.py b/src/pit_release_gate/frame.py new file mode 100644 index 0000000..ad13b07 --- /dev/null +++ b/src/pit_release_gate/frame.py @@ -0,0 +1,186 @@ +"""Screen a tabular panel directly, without building stores by hand. + +`screen_dataframe` takes one long table — an entity, a period, a filing +arrival time, one or more signal values, and a conditioning covariate — and +runs the same frozen protocol the published audits use: fit the +susceptibility estimate on prior *completed* periods, freeze it, apply it +forward, and report per-signal verdicts as a `pit-screen-results` record. + +The table can be anything that hands over a column: a pandas DataFrame, a +polars DataFrame, a pyarrow Table, or a plain dict of arrays. Nothing is +imported from any dataframe library — columns are read by duck typing and +converted to numpy, so the screen itself carries no dataframe dependency. +""" +from __future__ import annotations + +import numpy as np + +from .controller import ReleaseController +from .gate import SusceptibilityGate +from .results import build_results, screen_config, summarize_signal +from .store import AsOfDataStore + +__all__ = ["screen_dataframe", "stores_from_frame"] + + +def _column(data, name): + """One column as a 1-D numpy array, from any column-addressable table.""" + try: + col = data[name] + except (KeyError, IndexError, TypeError) as exc: + raise KeyError(f"column {name!r} not found in the table") from exc + for attr in ("to_numpy", "__array__"): # pandas / polars / pyarrow / numpy + if hasattr(col, attr): + arr = np.asarray(col.to_numpy() if attr == "to_numpy" else col) + break + else: + arr = np.asarray(col) + return arr.reshape(-1) + + +def _as_float(arr, name): + """Numeric view of a column, accepting datetimes for arrival times.""" + if np.issubdtype(arr.dtype, np.datetime64): + return arr.astype("datetime64[s]").astype(np.float64) + if np.issubdtype(arr.dtype, np.timedelta64): + return arr.astype("timedelta64[s]").astype(np.float64) + try: + return arr.astype(np.float64) + except (TypeError, ValueError) as exc: + raise TypeError(f"column {name!r} is not numeric (dtype {arr.dtype})") from exc + + +def _ols_resid(X, y): + beta, *_ = np.linalg.lstsq(X, y, rcond=None) + return y - X @ beta + + +def stores_from_frame(data, *, period="period", arrival="arrival", + value="value", size="size", min_entities=6): + """Build one `AsOfDataStore` per period, oldest first. + + Rows with a missing value, size or arrival are dropped for that signal, + and periods left with fewer than `min_entities` rows are skipped — + a cross-section too small to residualize honestly is not screened + rather than screened badly. + """ + per = _column(data, period) + arr = _as_float(_column(data, arrival), arrival) + val = _as_float(_column(data, value), value) + siz = _as_float(_column(data, size), size) + if not (len(per) == len(arr) == len(val) == len(siz)): + raise ValueError("columns have different lengths") + + stores, kept = [], [] + for key in _ordered_unique(per): + m = (per == key) & np.isfinite(arr) & np.isfinite(val) & np.isfinite(siz) + if m.sum() < min_entities: + continue + a, v, s = arr[m], val[m], siz[m] + + span = a.max() - a.min() + # arrival on [0, 1] within the period, 1 = the last filer (the deadline) + norm = np.ones_like(a) if span <= 0 else (a - a.min()) / span + sd = s.std() + s_std = np.zeros_like(s) if sd < 1e-12 else (s - s.mean()) / sd + X = np.column_stack([np.ones(m.sum()), s_std]) + + stores.append(AsOfDataStore( + X=X, y=v, arrival=norm, size=s_std, + # the estimand: the residual the COMPLETE cross-section implies + truth_resid=_ols_resid(X, v), + )) + kept.append(key) + return stores, kept + + +def _ordered_unique(a): + """Unique period keys, sorted — periods must be screened in time order.""" + return sorted(set(a.tolist())) + + +def screen_dataframe(data, *, period="period", arrival="arrival", + value="value", size="size", trailing_k=5, + rho_threshold=0.10, phi_min=0.35, kappa=1.0, + min_entities=6, date=None) -> dict: + """Screen one or more signals in a long table for incomplete-cross-section + leakage, and return a `pit-screen-results` v1.0 record. + + Parameters + ---------- + data + A long table: one row per (entity, period). Any object whose columns + are addressable by name — pandas, polars, pyarrow, or a dict of + arrays. + period, arrival, size + Column names. `period` groups the cross-section, `arrival` is when + that entity's record became available (a date or any increasing + number), `size` is the observable the screen conditions on. + value + The signal column, or a list of them to screen several at once. + trailing_k + How many prior completed periods the estimate is fitted on before + it is frozen and applied forward. The first `trailing_k` periods + are therefore used for fitting only and are not screened. + rho_threshold, phi_min, kappa, min_entities + Screen settings, recorded in the returned record so a reader can + tell which settings produced the verdicts. + + Returns + ------- + dict + A validated `pit-screen-results` record. Write it with + `write_results`, publish it, and point a *screened with* badge at it. + + Notes + ----- + A signal's verdict is ``susceptible`` if the frozen estimate exceeded the + threshold in **any** screened period — the convention the published + audit reports use, because a channel that opens in one year is not + closed by averaging it against years where it did not. + + Because the verdict fires on any single period, it inherits that period's + sampling noise: the standard error of the estimate is roughly + ``1 / sqrt(trailing_k * entities_per_period)``, so on small panels a + reading just over the threshold may be noise rather than a channel. + Establish the noise floor for your panel — screen a signal you have + reason to believe is unexposed, or shuffle arrival order within periods + and re-screen — before treating a marginal verdict as a finding. The + published audit reports do this and state the floor they measured. + + The screen is honest by construction: the estimate applied to a period is + never fitted on that period. Nothing is sent anywhere; this function + performs no network I/O. + """ + names = [value] if isinstance(value, str) else list(value) + if not names: + raise ValueError("no signal column given") + if trailing_k < 1: + raise ValueError("trailing_k must be at least 1") + + controller = ReleaseController(phi_min=phi_min, suscept_slope=kappa) + signals = [] + for name in names: + stores, kept = stores_from_frame( + data, period=period, arrival=arrival, value=name, + size=size, min_entities=min_entities) + if len(stores) <= trailing_k: + raise ValueError( + f"{name}: {len(stores)} usable periods, need more than " + f"trailing_k={trailing_k} so at least one period can be screened " + f"(periods with fewer than {min_entities} entities are skipped)") + + rhos, phi_reqs = [], [] + for i in range(trailing_k, len(stores)): + gate = SusceptibilityGate(threshold=rho_threshold) + rho = gate.fit_trailing(stores[i - trailing_k:i]) # frozen before use + rhos.append(rho) + phi_reqs.append(controller.required_completeness(rho)) + + signals.append(summarize_signal( + name, rhos, phi_reqs, rho_threshold=rho_threshold, + susceptible=any(abs(r) > rho_threshold for r in rhos))) + + config = screen_config(rho_threshold, phi_min, kappa, + trailing_k=trailing_k, min_entities=min_entities) + return build_results(signals, config, date=date) diff --git a/src/pit_release_gate/results.py b/src/pit_release_gate/results.py new file mode 100644 index 0000000..456650e --- /dev/null +++ b/src/pit_release_gate/results.py @@ -0,0 +1,256 @@ +"""The ``pit-screen-results`` record: build it, validate it, write it. + +A screen result is a *summary*: per signal, how many periods were screened, +how many of them the susceptibility measure flagged, the mean and maximum +susceptibility, the required completeness the controller assigned, and the +verdict. Nothing else. No input rows, no file paths, no identities, no +environment details beyond the tool version -- so a record is safe to commit +next to the badge, or to hand to a third party. + +This module is deliberately **offline**: it imports no transport machinery, +so nothing that builds or writes a record can send it anywhere. Neither does +anything else in this package -- there is no submission path at all, and a +record goes where you put it and nowhere else. + +It is also deliberately **timeless**: no clock is read here, so two runs of +the same screen produce byte-identical records. A caller that wants a date +in the record passes one in explicitly (``build_results(..., date=...)``). + +The schema is documented as a standalone interchange format in +``docs/results-schema.md``; other tools are free to emit it. +""" +from __future__ import annotations + +import json +from pathlib import Path + +#: Self-identifying name of the interchange format (not of the producing tool). +SCHEMA = 'pit-screen-results' +SCHEMA_VERSION = '1.0' +KNOWN_SCHEMA_VERSIONS = ('1.0',) + +#: The tool that produces the record in this package. +TOOL = 'pit-release-gate' + +#: The only two verdicts a screened signal can carry. +VERDICTS = ('benign', 'susceptible') + +_TOP_FIELDS = ('schema', 'schema_version', 'tool', 'tool_version', + 'config', 'signals', 'totals') +_CONFIG_FIELDS = ('rho_threshold', 'phi_min', 'kappa', 'trailing_k', 'min_entities') +_SIGNAL_FIELDS = ('name', 'periods_screened', 'periods_flagged', 'mean_rho', + 'max_abs_rho', 'mean_phi_req', 'verdict') +_TOTALS_FIELDS = ('signal_cycles', 'signals_benign', 'signals_susceptible') + + +def tool_version() -> str: + """Version of the code that actually ran. + + The in-tree ``__version__`` is preferred over installed distribution + metadata, because a source checkout on ``PYTHONPATH`` can shadow an older + installed wheel; the metadata is the fallback. Never a hardcoded literal. + """ + try: + from . import __version__ + return str(__version__) + except ImportError: # pragma: no cover - only if the package is half-built + from importlib.metadata import PackageNotFoundError, version + try: + return version(TOOL) + except PackageNotFoundError: + return 'unknown' + + +def screen_config(rho_threshold: float, phi_min: float, kappa: float, + trailing_k: int, min_entities: int) -> dict: + """The screen settings that determine a verdict, in record form. + + These five are what another party needs in order to read a verdict: + the susceptibility threshold, the completeness floor and slope of + ``phi_req = min(1, phi_min + kappa*|rho_hat|)``, how many prior completed + periods the estimate was fitted on, and the minimum arrived count below + which the controller releases nothing. + """ + return { + 'rho_threshold': float(rho_threshold), + 'phi_min': float(phi_min), + 'kappa': float(kappa), + 'trailing_k': int(trailing_k), + 'min_entities': int(min_entities), + } + + +def summarize_signal(name: str, rhos, phi_reqs, rho_threshold: float = 0.10, + susceptible: bool = None) -> dict: + """Reduce one signal's per-period screen to the record's summary row. + + ``rhos`` are the per-period susceptibility estimates and ``phi_reqs`` the + required completeness the controller assigned in each of those periods; + they must be the same length, and that length is ``periods_screened``. + A period is *flagged* when ``|rho|`` exceeds ``rho_threshold``, so + ``periods_flagged <= periods_screened`` by construction. + + The verdict defaults to the same test applied to the mean susceptibility; + a caller that gates on a frozen trailing estimate should pass its own + verdict as ``susceptible`` so the record states what the screen actually + decided. + """ + rhos = [float(r) for r in rhos] + phi_reqs = [float(p) for p in phi_reqs] + if len(rhos) != len(phi_reqs): + raise ValueError(f'{name}: got {len(rhos)} rho values but ' + f'{len(phi_reqs)} required-completeness values') + if not rhos: + raise ValueError(f'{name}: no screened periods') + mean_rho = sum(rhos) / len(rhos) + if susceptible is None: + susceptible = abs(mean_rho) > rho_threshold + return { + 'name': str(name), + 'periods_screened': len(rhos), + 'periods_flagged': sum(1 for r in rhos if abs(r) > rho_threshold), + 'mean_rho': mean_rho, + 'max_abs_rho': max(abs(r) for r in rhos), + 'mean_phi_req': sum(phi_reqs) / len(phi_reqs), + 'verdict': VERDICTS[1] if susceptible else VERDICTS[0], + } + + +def build_results(signals, config: dict, tool: str = TOOL, + version: str = None, date: str = None) -> dict: + """Assemble a complete ``pit-screen-results`` record. + + ``signals`` is a list of :func:`summarize_signal` rows. The totals are + derived here rather than accepted from the caller, so they cannot + disagree with the rows. ``date`` is optional and purely caller-supplied + -- this module never reads a clock. + """ + signals = [dict(s) for s in signals] + record = { + 'schema': SCHEMA, + 'schema_version': SCHEMA_VERSION, + 'tool': str(tool), + 'tool_version': version or tool_version(), + 'config': dict(config), + 'signals': signals, + 'totals': { + 'signal_cycles': sum(int(s['periods_screened']) for s in signals), + 'signals_benign': sum(1 for s in signals if s['verdict'] == VERDICTS[0]), + 'signals_susceptible': sum(1 for s in signals if s['verdict'] == VERDICTS[1]), + }, + } + if date is not None: + record['date'] = str(date) + return record + + +def validate_results(obj) -> list[str]: + """Return a list of human-readable problems with ``obj``; empty == valid. + + Written so that a reader of the format -- not just this package -- can + check a record before relying on it, and so that any receiver of the + format can refuse something malformed. + """ + problems: list[str] = [] + if not isinstance(obj, dict): + return [f'record must be a JSON object, got {type(obj).__name__}'] + + version = obj.get('schema_version') + if version is None: + problems.append('missing required field: schema_version') + elif version not in KNOWN_SCHEMA_VERSIONS: + problems.append(f'unknown schema_version {version!r} ' + f'(known: {", ".join(KNOWN_SCHEMA_VERSIONS)})') + if obj.get('schema') != SCHEMA: + problems.append(f'schema must be {SCHEMA!r}, got {obj.get("schema")!r}') + for field in _TOP_FIELDS: + if field not in obj: + if field != 'schema_version': # already reported above + problems.append(f'missing required field: {field}') + + config = obj.get('config') + if config is not None and not isinstance(config, dict): + problems.append('config must be a JSON object') + elif isinstance(config, dict): + for field in _CONFIG_FIELDS: + if field not in config: + problems.append(f'missing required config field: {field}') + + signals = obj.get('signals') + cycles = 0 + if signals is not None and not isinstance(signals, list): + problems.append('signals must be a list') + elif isinstance(signals, list): + if not signals: + problems.append('signals is empty: nothing was screened') + for i, sig in enumerate(signals): + where = f'signals[{i}]' + if not isinstance(sig, dict): + problems.append(f'{where} must be a JSON object') + continue + where = f'signals[{i}] ({sig.get("name", "unnamed")})' + for field in _SIGNAL_FIELDS: + if field not in sig: + problems.append(f'{where}: missing required field: {field}') + if sig.get('verdict') not in VERDICTS and 'verdict' in sig: + problems.append(f'{where}: verdict must be one of ' + f'{", ".join(VERDICTS)}, got {sig["verdict"]!r}') + screened, flagged = sig.get('periods_screened'), sig.get('periods_flagged') + if isinstance(screened, int) and isinstance(flagged, int): + cycles += screened + if flagged > screened: + problems.append(f'{where}: periods_flagged ({flagged}) exceeds ' + f'periods_screened ({screened})') + if flagged < 0 or screened < 0: + problems.append(f'{where}: periods_screened and periods_flagged ' + f'must not be negative') + elif screened is not None or flagged is not None: + problems.append(f'{where}: periods_screened and periods_flagged ' + f'must be integers') + + totals = obj.get('totals') + if totals is not None and not isinstance(totals, dict): + problems.append('totals must be a JSON object') + elif isinstance(totals, dict): + for field in _TOTALS_FIELDS: + if field not in totals: + problems.append(f'missing required totals field: {field}') + if isinstance(signals, list) and 'signal_cycles' in totals: + if totals['signal_cycles'] != cycles: + problems.append(f'totals.signal_cycles ({totals["signal_cycles"]}) ' + f'does not equal the sum over signals ({cycles})') + if isinstance(signals, list): + benign = sum(1 for s in signals + if isinstance(s, dict) and s.get('verdict') == VERDICTS[0]) + suspect = sum(1 for s in signals + if isinstance(s, dict) and s.get('verdict') == VERDICTS[1]) + if totals.get('signals_benign') != benign: + problems.append(f'totals.signals_benign ({totals.get("signals_benign")}) ' + f'does not equal the number of benign signals ({benign})') + if totals.get('signals_susceptible') != suspect: + problems.append(f'totals.signals_susceptible ' + f'({totals.get("signals_susceptible")}) does not equal ' + f'the number of susceptible signals ({suspect})') + return problems + + +def dumps_results(obj) -> str: + """Canonical JSON text of a record -- exactly what ``--export`` writes.""" + return json.dumps(obj, indent=2, ensure_ascii=False) + + +def write_results(obj, path) -> Path: + """Write a validated record to ``path`` as JSON. Local file I/O only. + + Raises ``ValueError`` rather than writing a record that would not survive + :func:`validate_results`. + """ + problems = validate_results(obj) + if problems: + raise ValueError('refusing to write an invalid ' + SCHEMA + ' record: ' + + '; '.join(problems)) + path = Path(path) + if path.parent and not path.parent.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(dumps_results(obj) + '\n', encoding='utf-8', newline='\n') + return path diff --git a/src/pit_release_gate/simulate.py b/src/pit_release_gate/simulate.py index 516193f..8b26f68 100644 --- a/src/pit_release_gate/simulate.py +++ b/src/pit_release_gate/simulate.py @@ -15,13 +15,25 @@ import pandas as pd from scipy.stats import rankdata -from .controller import ReleaseController, ReleaseDecision, _ols_resid +from .controller import MIN_ENTITIES, ReleaseController, ReleaseDecision, _ols_resid from .gate import SusceptibilityGate +from .results import ( + SCHEMA, + SCHEMA_VERSION, + build_results, + screen_config, + summarize_signal, + write_results, +) from .reweight import PropensityReweighter from .store import AsOfDataStore SEED = 20260601 +#: Susceptibility threshold the demo screen runs at (|rho_hat| above it is +#: susceptible). Named so the exported screen record cannot drift from it. +DEMO_RHO_THRESHOLD = 0.10 + #: The four planted-truth signal configurations reported in the demo: #: (key, label, c_a, c_x) DEMO_SIGNALS = [ @@ -113,10 +125,17 @@ def run_demo(n_train=10, n_eval=60, verbose=True) -> dict: for key, name, c_a, c_x in DEMO_SIGNALS: # --- honest trailing estimation on prior completed periods --- - gate = SusceptibilityGate(threshold=0.10) + gate = SusceptibilityGate(threshold=DEMO_RHO_THRESHOLD) train = [make_group(n=120, c_a=c_a, c_x=c_x, rng=rng) for _ in range(n_train)] rho_tr = gate.fit_trailing(train) ctrl = ReleaseController(gate=gate) + # the settings that produced the verdicts, recorded for --export + # (identical for every signal; the kappa sweep below is separate) + results['config'] = screen_config(rho_threshold=gate.threshold, + phi_min=ctrl.phi_min, + kappa=ctrl.suscept_slope, + trailing_k=n_train, + min_entities=MIN_ENTITIES) # --- evaluation on fresh periods, gated with the FROZEN estimate --- agg = {p: {'comp': [], 'flip': [], 'bias': [], 'act': []} for p in policies} rho_realized = [] @@ -133,8 +152,11 @@ def run_demo(n_train=10, n_eval=60, verbose=True) -> dict: sig = {'label': name, 'c_a': c_a, 'c_x': c_x, 'rho_trailing': rho_tr, + 'rho_realized': [float(r) for r in rho_realized], 'rho_realized_mean': float(np.mean(rho_realized)), 'rho_realized_std': float(np.std(rho_realized)), + # constant across periods: the gate runs on the FROZEN estimate + 'phi_req': ctrl.required_completeness(rho_tr), 'susceptible': bool(gate.is_susceptible(rho_tr)), 'policies': {}} for p in policies: @@ -164,7 +186,7 @@ def run_demo(n_train=10, n_eval=60, verbose=True) -> dict: if verbose: print("\n" + "-" * 96) print("Sensitivity: required-completeness slope kappa on Mild-leak (c_a=0.3, c_x=0.7)") - gate = SusceptibilityGate(threshold=0.10) + gate = SusceptibilityGate(threshold=DEMO_RHO_THRESHOLD) train = [make_group(n=120, c_a=0.3, c_x=0.7, rng=rng) for _ in range(n_train)] rho_tr = gate.fit_trailing(train) for kappa in [0.5, 1.0, 2.0]: @@ -212,17 +234,206 @@ def demo(n_train=10, n_eval=60): return run_demo(n_train=n_train, n_eval=n_eval, verbose=True) +def results_from_demo(run: dict, date: str = None) -> dict: + """Reduce a :func:`run_demo` result to a ``pit-screen-results`` record. + + Only summary statistics survive the reduction: per signal, the number of + screened periods, how many of them the susceptibility measure flagged, + the mean and maximum |rho_hat|, the required completeness the controller + assigned, and the verdict. The simulated cross-sections themselves stay + on this machine. + + ``date`` is optional and caller-supplied; nothing here reads a clock, so + the same screen always reduces to the same record. + """ + threshold = run['config']['rho_threshold'] + signals = [ + summarize_signal( + key, + rhos=sig['rho_realized'], + phi_reqs=[sig['phi_req']] * len(sig['rho_realized']), + rho_threshold=threshold, + # the verdict the screen actually acted on: the frozen trailing + # estimate, not the ex-post realized ones summarized above + susceptible=sig['susceptible'], + ) + for key, sig in run['signals'].items() + ] + return build_results(signals, run['config'], date=date) + + +BADGE_MARKDOWN = ( + '[![screened with pit-release-gate]' + '(https://img.shields.io/badge/screened%20with-pit--release--gate-blue)]' + '(https://github.com/MaxWellApexLab/pit-release-gate)' +) + + +def badge_snippet(results) -> str: + """README badge markdown for a completed screen, with a rho_hat summary. + + Pure formatting over an existing ``run_demo`` result -- it reads the + result dict and returns a string. It performs no estimation and changes + no numerical behavior. + + The badge states that the screen was RUN. It is deliberately not a + pass/fail claim: a susceptible verdict is as worth reporting as a + benign one. + """ + try: + from . import __version__ as version + except ImportError: # pragma: no cover + version = '' + + sigs = results['signals'] + keys = [k for k, *_ in DEMO_SIGNALS if k in sigs] + n_susc = sum(bool(sigs[k]['susceptible']) for k in keys) + rhos = ' | '.join(f"{k} {sigs[k]['rho_trailing']:+.3f}" for k in keys) + + rule = '-' * 72 + return '\n'.join([ + rule, + 'Badge snippet (paste into your README):', + '', + BADGE_MARKDOWN, + '', + f'', + '', + 'The badge states that the screen was RUN, not that anything passed.', + 'Point it at your own screen output to make it worth clicking.', + rule, + ]) + + +def read_csv_columns(path): + """Read a CSV into {column: numpy array} using the standard library only. + + Numeric columns become floats; anything else stays a string array, which + is all the screen needs for period keys. + """ + import csv as _csv + with open(path, newline='', encoding='utf-8-sig') as fh: + rows = list(_csv.DictReader(fh)) + if not rows: + raise SystemExit(f'{path}: no data rows') + out = {} + for name in rows[0]: + raw = [r[name] for r in rows] + try: + out[name] = np.array([float(v) if v not in ('', 'NA', 'NaN', 'nan') + else np.nan for v in raw], dtype=float) + except ValueError: + out[name] = np.array(raw, dtype=object) + return out + + +def _version(): + try: + from . import __version__ + return __version__ + except ImportError: # pragma: no cover + return '' + + +def _screen_csv(a): + """Screen a user-supplied panel and print one row per signal.""" + from .frame import screen_dataframe + + data = read_csv_columns(a.csv) + record = screen_dataframe( + data, period=a.period, arrival=a.arrival, value=a.value, + size=a.size, trailing_k=a.trailing_k, rho_threshold=a.threshold) + + print(f'screened {a.csv}: {len(record["signals"])} signal(s), ' + f'trailing_k={a.trailing_k}, threshold={a.threshold}') + print(f' {"signal":<24} {"periods":>8} {"flagged":>8} {"mean rho":>10} ' + f'{"max |rho|":>10} {"phi_req":>8} verdict') + for s in record['signals']: + print(f' {s["name"]:<24} {s["periods_screened"]:>8} {s["periods_flagged"]:>8} ' + f'{s["mean_rho"]:>+10.4f} {s["max_abs_rho"]:>10.4f} ' + f'{s["mean_phi_req"]:>8.3f} {s["verdict"]}') + t = record['totals'] + print(f' totals: {t["signals_benign"]} benign, {t["signals_susceptible"]} susceptible, ' + f'{t["signal_cycles"]} signal-cycles screened') + + if a.badge: + rule = '-' * 72 + print() + print('\n'.join([ + rule, + 'Badge snippet (paste into your README):', + '', + BADGE_MARKDOWN, + '', + f'', + '', + 'The badge states that the screen was RUN, not that anything passed.', + 'Point it at your exported record, not at this repo.', + rule, + ])) + if a.export: + path = write_results(record, a.export) + print(f'\nwrote {SCHEMA} v{SCHEMA_VERSION} to {path} ' + f'(local file only -- no network call was made)') + + def main(argv=None): ap = argparse.ArgumentParser( prog='pit-release-gate', description='Run the self-contained known-ground-truth demo of the ' - 'completeness-aware release controller.') + 'completeness-aware release controller.', + epilog='This tool never reports anything, anywhere. --export writes a ' + 'local file and makes no network call; the package opens no ' + 'socket at all.') ap.add_argument('--train', type=int, default=10, help='number of prior completed periods used to fit rho_hat (default 10)') ap.add_argument('--eval', dest='n_eval', type=int, default=60, help='number of fresh evaluation periods (default 60)') + ap.add_argument('--export', metavar='PATH', + help=f'write the screen result to PATH as a {SCHEMA} ' + f'v{SCHEMA_VERSION} JSON record (fully offline)') + ap.add_argument('--csv', metavar='PATH', + help='screen your own panel instead of running the demo: a CSV ' + 'with one row per (entity, period). Requires --value') + ap.add_argument('--value', metavar='COL', action='append', + help='signal column in --csv; repeat to screen several') + ap.add_argument('--period', metavar='COL', default='period', + help='period column in --csv (default: period)') + ap.add_argument('--arrival', metavar='COL', default='arrival', + help='arrival-time column in --csv (default: arrival)') + ap.add_argument('--size', metavar='COL', default='size', + help='conditioning covariate column in --csv (default: size)') + ap.add_argument('--trailing-k', type=int, default=5, metavar='K', + help='prior completed periods the estimate is fitted on (default 5)') + ap.add_argument('--threshold', type=float, default=0.10, + help='flag a period when |rho_hat| exceeds this (default 0.10)') + ap.add_argument('--badge', action='store_true', + help='after the demo, print a README badge snippet recording ' + 'that the screen was run (does not change the demo output)') a = ap.parse_args(argv) - run_demo(n_train=a.train, n_eval=a.n_eval, verbose=True) + + if a.csv: + if not a.value: + ap.error('--csv requires at least one --value COLUMN') + return _screen_csv(a) + if a.value: + ap.error('--value is only meaningful together with --csv') + + run = run_demo(n_train=a.train, n_eval=a.n_eval, verbose=True) + if a.badge: + print() + print(badge_snippet(run)) + if not a.export: + return + + record = results_from_demo(run) + path = write_results(record, a.export) + print(f'\nwrote {SCHEMA} v{SCHEMA_VERSION} to {path} ' + f'(local file only -- no network call was made)') if __name__ == '__main__': diff --git a/tests/test_cli_and_monitor.py b/tests/test_cli_and_monitor.py index e5ac233..fb10769 100644 --- a/tests/test_cli_and_monitor.py +++ b/tests/test_cli_and_monitor.py @@ -6,7 +6,8 @@ import numpy as np -from pit_release_gate import AsOfDataStore, CompletenessMonitor, make_group +from pit_release_gate import (AsOfDataStore, CompletenessMonitor, badge_snippet, + make_group, run_demo) def test_cli_demo_runs_and_prints_verdicts(): @@ -31,6 +32,43 @@ def test_demo_main_in_process(capsys): assert token in text +def test_badge_snippet_reports_the_screen_result(): + r = run_demo(n_train=2, n_eval=4, verbose=False) + text = badge_snippet(r) + + # the markdown a user actually pastes + assert "img.shields.io/badge/screened%20with-pit--release--gate-blue" in text + assert "github.com/MaxWellApexLab/pit-release-gate" in text + + # the summary comment carries every signal's frozen rho_hat + for key in ("clean", "composition", "mild_leak", "strong_leak"): + assert key in text + # demo plants two benign and two susceptible signals + assert "2 benign, 2 susceptible" in text + + +def test_badge_snippet_makes_no_pass_fail_claim(): + text = badge_snippet(run_demo(n_train=2, n_eval=4, verbose=False)).lower() + assert "screened with" in text + assert "not that anything passed" in text + for forbidden in ("certif", "approv", "endors", "official", "trusted"): + assert forbidden not in text, f"badge output must not claim {forbidden!r}" + + +def test_badge_flag_does_not_change_demo_output(capsys): + from pit_release_gate.simulate import main + + main(["--train", "2", "--eval", "4"]) + plain = capsys.readouterr().out + main(["--train", "2", "--eval", "4", "--badge"]) + badged = capsys.readouterr().out + + # the badge is strictly appended: the demo output is byte-identical + assert badged.startswith(plain) + assert "Badge snippet" in badged[len(plain):] + assert "Badge snippet" not in plain + + def test_completeness_monitor_fraction_and_shift(): rng = np.random.default_rng(7) store = make_group(c_a=0.0, c_x=1.0, rng=rng) diff --git a/tests/test_csv_cli.py b/tests/test_csv_cli.py new file mode 100644 index 0000000..dd285e1 --- /dev/null +++ b/tests/test_csv_cli.py @@ -0,0 +1,84 @@ +"""The `--csv` path: screening a user's own panel from the command line.""" +import csv +import json + +import numpy as np +import pytest + +from pit_release_gate.simulate import main, read_csv_columns +from test_frame import planted_panel + + +def write_panel(path, **kw): + p = planted_panel(**kw) + with open(path, "w", newline="", encoding="utf-8") as fh: + w = csv.writer(fh) + w.writerow(list(p)) + for row in zip(*p.values()): + w.writerow(row) + return path + + +def test_reads_a_csv_into_typed_columns(tmp_path): + f = write_panel(tmp_path / "p.csv", n_periods=7, n=20) + cols = read_csv_columns(f) + assert set(cols) == {"period", "entity", "arrival", "value", "size"} + assert cols["value"].dtype == float + assert len(cols["value"]) == 7 * 20 + + +def test_blank_and_na_cells_become_nan(tmp_path): + f = tmp_path / "gappy.csv" + f.write_text("period,arrival,value,size\n1,0.1,,1.0\n1,0.2,NA,2.0\n", + encoding="utf-8") + cols = read_csv_columns(f) + assert np.isnan(cols["value"]).all() + + +def test_cli_screens_a_planted_leak(tmp_path, capsys): + f = write_panel(tmp_path / "leak.csv", leak=2.0) + main(["--csv", str(f), "--value", "value"]) + out = capsys.readouterr().out + assert "susceptible" in out + assert "signal-cycles screened" in out + + +def test_cli_exports_a_valid_record(tmp_path, capsys): + f = write_panel(tmp_path / "leak.csv", leak=2.0) + out_json = tmp_path / "results.json" + main(["--csv", str(f), "--value", "value", "--export", str(out_json)]) + record = json.loads(out_json.read_text(encoding="utf-8")) + assert record["schema"] == "pit-screen-results" + assert record["signals"][0]["verdict"] == "susceptible" + assert "no network call" in capsys.readouterr().out + + +def test_cli_screens_several_signals_and_honours_settings(tmp_path, capsys): + p = planted_panel(leak=2.0, n=200) + p["quiet"] = p["size"] * 0.3 + np.random.default_rng(5).normal(size=len(p["size"])) + f = tmp_path / "two.csv" + with open(f, "w", newline="", encoding="utf-8") as fh: + w = csv.writer(fh) + w.writerow(list(p)) + for row in zip(*p.values()): + w.writerow(row) + main(["--csv", str(f), "--value", "value", "--value", "quiet", + "--trailing-k", "4", "--threshold", "0.15"]) + out = capsys.readouterr().out + assert "trailing_k=4" in out and "threshold=0.15" in out + assert "1 benign, 1 susceptible" in out + + +def test_cli_rejects_incoherent_flags(tmp_path): + f = write_panel(tmp_path / "p.csv") + with pytest.raises(SystemExit): + main(["--csv", str(f)]) # --csv without --value + with pytest.raises(SystemExit): + main(["--value", "value"]) # --value without --csv + + +def test_demo_path_is_untouched_by_the_csv_flags(capsys): + """The default invocation must still be the known-ground-truth demo.""" + main(["--train", "2", "--eval", "4"]) + out = capsys.readouterr().out + assert "Strong-leak" in out and "gated" in out diff --git a/tests/test_export.py b/tests/test_export.py new file mode 100644 index 0000000..b508561 --- /dev/null +++ b/tests/test_export.py @@ -0,0 +1,287 @@ +"""Tests for the ``pit-screen-results`` record and the ``--export`` path. + +One property matters more than the schema details and is asserted from +several directions here: **this package never talks to the network.** The +urlopen entry point and the socket constructor are monkeypatched to raise, +and every path must still succeed; a structural test then walks every module +in the package and fails if any of them so much as imports a transport. + +There is deliberately no submission path on this branch. The ``--submit`` +half of this work is parked on the ``submit-cli`` branch until a receiving +endpoint exists -- a command pointing at an endpoint that is not there must +not ship. +""" +import getpass +import json +import os +import re +import socket +import sys +import urllib.request +from pathlib import Path + +import pytest + +import pit_release_gate +from pit_release_gate import results as results_mod +from pit_release_gate.results import ( + SCHEMA, + SCHEMA_VERSION, + TOOL, + VERDICTS, + build_results, + screen_config, + summarize_signal, + validate_results, + write_results, +) +from pit_release_gate.simulate import main + + +# --------------------------------------------------------------------------- +# no test in this file may reach the real network +# --------------------------------------------------------------------------- +def _exploding(*a, **k): + raise AssertionError("network call attempted") + + +@pytest.fixture +def no_network(monkeypatch): + """Any attempt to open a socket or a URL blows up the test.""" + monkeypatch.setattr(urllib.request, "urlopen", _exploding) + monkeypatch.setattr(socket, "socket", _exploding) + return None + + +def _tiny_record(**kw): + """A small valid record built through the public API (no demo run).""" + sig = summarize_signal("alpha", rhos=[0.01, -0.02, 0.30], + phi_reqs=[0.35, 0.35, 0.35], rho_threshold=0.10) + cfg = screen_config(rho_threshold=0.10, phi_min=0.35, kappa=1.0, + trailing_k=4, min_entities=6) + return build_results([sig], cfg, **kw) + + +# --------------------------------------------------------------------------- +# 1. the record: summary statistics, totals consistent by construction +# --------------------------------------------------------------------------- +def test_signal_summary_reports_only_summary_statistics(): + sig = summarize_signal("alpha", rhos=[0.01, -0.02, 0.30], + phi_reqs=[0.35, 0.35, 0.35], rho_threshold=0.10) + assert set(sig) == {"name", "periods_screened", "periods_flagged", "mean_rho", + "max_abs_rho", "mean_phi_req", "verdict"} + assert sig["periods_screened"] == 3 + assert sig["periods_flagged"] == 1 # only |0.30| exceeds 0.10 + assert sig["max_abs_rho"] == pytest.approx(0.30) + assert sig["mean_rho"] == pytest.approx((0.01 - 0.02 + 0.30) / 3) + assert sig["mean_phi_req"] == pytest.approx(0.35) + assert sig["verdict"] in VERDICTS + assert sig["periods_flagged"] <= sig["periods_screened"] + + +def test_verdict_follows_the_threshold_and_can_be_overridden(): + benign = summarize_signal("a", [0.01, 0.0], [0.35, 0.35], rho_threshold=0.10) + leaky = summarize_signal("b", [-0.9, -0.8], [1.0, 1.0], rho_threshold=0.10) + assert benign["verdict"] == "benign" + assert leaky["verdict"] == "susceptible" + # the screen's own frozen verdict wins when the caller supplies it + forced = summarize_signal("c", [0.01, 0.0], [0.35, 0.35], + rho_threshold=0.10, susceptible=True) + assert forced["verdict"] == "susceptible" + + +def test_build_results_totals_are_consistent_by_construction(): + sigs = [ + summarize_signal("a", [0.01, 0.0, 0.0], [0.35] * 3, rho_threshold=0.10), + summarize_signal("b", [-0.9, -0.8], [1.0, 1.0], rho_threshold=0.10), + ] + rec = build_results(sigs, screen_config(0.10, 0.35, 1.0, 4, 6)) + assert rec["schema"] == SCHEMA == "pit-screen-results" + assert rec["schema_version"] == SCHEMA_VERSION == "1.0" + assert rec["tool"] == TOOL == "pit-release-gate" + assert rec["tool_version"] == pit_release_gate.__version__ + assert rec["config"] == {"rho_threshold": 0.10, "phi_min": 0.35, "kappa": 1.0, + "trailing_k": 4, "min_entities": 6} + assert rec["totals"] == {"signal_cycles": 5, "signals_benign": 1, + "signals_susceptible": 1} + assert validate_results(rec) == [] + + +def test_record_carries_no_timestamp_unless_the_caller_passes_a_date(): + a = _tiny_record() + b = _tiny_record() + assert a == b # reproducible: nothing time-varying inside + assert json.dumps(a) == json.dumps(b) + assert "date" not in a + assert not [k for k in a if "time" in k.lower() or "date" in k.lower()] + dated = _tiny_record(date="2026-08-16") + assert dated["date"] == "2026-08-16" + + +# --------------------------------------------------------------------------- +# 2. the validator +# --------------------------------------------------------------------------- +def test_validate_accepts_a_built_record(): + assert validate_results(_tiny_record()) == [] + + +@pytest.mark.parametrize("mutate, needle", [ + (lambda r: r.pop("schema_version"), "schema_version"), + (lambda r: r.update(schema_version="9.9"), "9.9"), + (lambda r: r.pop("tool_version"), "tool_version"), + (lambda r: r.pop("config"), "config"), + (lambda r: r["config"].pop("kappa"), "kappa"), + (lambda r: r["signals"][0].pop("mean_rho"), "mean_rho"), + (lambda r: r["signals"][0].update(verdict="probably fine"), "verdict"), + (lambda r: r["signals"][0].update(periods_flagged=99), "periods_flagged"), + (lambda r: r["totals"].update(signal_cycles=999), "signal_cycles"), + (lambda r: r.update(signals="not a list"), "signals"), +]) +def test_validate_reports_each_kind_of_problem(mutate, needle): + rec = _tiny_record() + mutate(rec) + problems = validate_results(rec) + assert problems, "validator missed a broken record" + assert any(needle in p for p in problems), problems + + +def test_validate_rejects_non_mapping(): + assert validate_results([1, 2, 3]) + assert validate_results(None) + + +# --------------------------------------------------------------------------- +# 3. --export is fully offline +# --------------------------------------------------------------------------- +def test_export_writes_a_valid_results_json(tmp_path, capsys): + path = tmp_path / "results.json" + main(["--train", "3", "--eval", "4", "--export", str(path)]) + capsys.readouterr() + + rec = json.loads(path.read_text(encoding="utf-8")) + assert validate_results(rec) == [] + assert rec["schema"] == "pit-screen-results" + assert rec["schema_version"] == "1.0" + assert rec["tool"] == "pit-release-gate" + assert rec["tool_version"] == pit_release_gate.__version__ + assert rec["config"]["trailing_k"] == 3 + + names = [s["name"] for s in rec["signals"]] + assert names == ["clean", "composition", "mild_leak", "strong_leak"] + by_name = {s["name"]: s for s in rec["signals"]} + assert by_name["clean"]["verdict"] == "benign" + assert by_name["mild_leak"]["verdict"] == "susceptible" + assert by_name["strong_leak"]["verdict"] == "susceptible" + # the susceptible signals are graded to a higher required completeness + assert by_name["strong_leak"]["mean_phi_req"] > by_name["clean"]["mean_phi_req"] + + for s in rec["signals"]: + assert s["periods_screened"] == 4 + assert s["periods_flagged"] <= s["periods_screened"] + assert rec["totals"]["signal_cycles"] == 16 + assert rec["totals"]["signals_benign"] + rec["totals"]["signals_susceptible"] == 4 + + +def test_export_performs_no_network_io(tmp_path, capsys, no_network): + """The red line: --export must not touch the network at all.""" + path = tmp_path / "offline.json" + main(["--train", "2", "--eval", "2", "--export", str(path)]) + capsys.readouterr() + assert validate_results(json.loads(path.read_text(encoding="utf-8"))) == [] + + +def test_results_module_has_no_network_machinery(): + # the module that builds and writes the record cannot import a transport + assert not hasattr(results_mod, "urllib") + assert not hasattr(results_mod, "socket") + src = Path(results_mod.__file__).read_text(encoding="utf-8") + assert "urllib" not in src + assert "atexit" not in src + + +def _imported_modules(path: Path) -> set: + src = path.read_text(encoding="utf-8") + return {m.lstrip(".").split(".")[0] + for m in re.findall(r"^\s*(?:from|import)\s+([\w.]+)", src, re.M)} + + +def test_no_module_in_the_package_can_reach_the_network(): + """Structural guard on the no-telemetry red line. + + Not "no module reports by default" -- *no module can*. Nothing in the + package may import a transport, register an exit hook, or spawn a worker + that could report in the background. This is asserted over every module, + so adding a phone-home later fails the suite rather than shipping. + """ + pkg = Path(pit_release_gate.__file__).parent + modules = sorted(pkg.glob("*.py")) + assert len(modules) >= 8 + for py in modules: + imported = _imported_modules(py) + assert "atexit" not in imported, f"{py.name} registers an exit hook" + assert not imported & {"threading", "multiprocessing", "concurrent", + "asyncio", "subprocess"}, f"{py.name} spawns work" + assert not imported & {"urllib", "socket", "http", "ssl", "requests", + "smtplib", "ftplib"}, f"{py.name} imports a transport" + + +def test_the_package_ships_no_submission_path(): + """The --submit half stays parked until a receiving endpoint exists.""" + pkg = Path(pit_release_gate.__file__).parent + assert not (pkg / "submit.py").exists() + assert not hasattr(pit_release_gate, "submit_results") + help_text = Path(pkg / "simulate.py").read_text(encoding="utf-8") + for flag in ("'--submit'", "'--contact'", "'--dry-run'"): + assert flag not in help_text, f"{flag} is still wired into the CLI" + + +def test_the_export_path_never_reads_a_clock(): + """No timestamps generated inside the library: a date must be passed in.""" + pkg = Path(pit_release_gate.__file__).parent + for name in ("results.py", "simulate.py"): + imported = _imported_modules(pkg / name) + assert not imported & {"time", "datetime", "calendar"}, f"{name} reads a clock" + + +def test_export_is_byte_identical_across_runs(tmp_path, capsys): + a, b = tmp_path / "a.json", tmp_path / "b.json" + main(["--train", "2", "--eval", "2", "--export", str(a)]) + main(["--train", "2", "--eval", "2", "--export", str(b)]) + capsys.readouterr() + assert a.read_bytes() == b.read_bytes() + + +def test_exported_payload_carries_no_identifying_details(tmp_path, capsys): + path = tmp_path / "results.json" + main(["--train", "2", "--eval", "2", "--export", str(path)]) + capsys.readouterr() + text = path.read_text(encoding="utf-8") + for secret in (getpass.getuser(), socket.gethostname(), os.getcwd(), + str(Path.home()), sys.executable, str(path)): + if secret and len(secret) > 3: + assert secret not in text, f"payload leaks {secret!r}" + for banned in ("path", "user", "host", "cwd", "platform", "python"): + assert banned not in text.lower() + + +# --------------------------------------------------------------------------- +# 4. the submission flags are not wired in on this branch +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("argv", [ + ["--submit", "https://example.invalid/submit"], + ["--contact", "a@b.org"], + ["--dry-run"], +]) +def test_cli_does_not_accept_submission_flags(argv, capsys, no_network): + """argparse must reject them outright -- they are not shipped.""" + with pytest.raises(SystemExit) as exc: + main(["--train", "2", "--eval", "2", *argv]) + assert exc.value.code != 0 + assert "unrecognized arguments" in capsys.readouterr().err + + +def test_plain_cli_run_writes_no_file_and_opens_no_socket(tmp_path, capsys, no_network): + main(["--train", "2", "--eval", "2"]) + out = capsys.readouterr().out + assert "gated" in out + assert list(tmp_path.iterdir()) == [] diff --git a/tests/test_frame.py b/tests/test_frame.py new file mode 100644 index 0000000..01b9f7f --- /dev/null +++ b/tests/test_frame.py @@ -0,0 +1,142 @@ +"""The tabular entry point: planted-truth behaviour, framework independence, +and the honest-estimation contract.""" +import numpy as np +import pytest + +from pit_release_gate import screen_dataframe, stores_from_frame, validate_results + + +def planted_panel(n_periods=12, n=60, leak=0.0, seed=0): + """A long table whose leakage strength is known by construction. + + ``leak`` couples filing latency to the signal's disturbance: at 0 the + arrival order carries no information about the disturbance, and the + screen should read ~0; raise it and the early cross-section becomes a + selected sample, which is what the screen is built to catch. + """ + rng = np.random.default_rng(seed) + cols = {k: [] for k in ("period", "entity", "arrival", "value", "size")} + for p in range(n_periods): + size = rng.normal(size=n) + u = rng.normal(size=n) + cols["period"] += [p] * n + cols["entity"] += list(range(n)) + cols["arrival"] += list(rng.normal(size=n) - leak * u) + cols["value"] += list(0.5 * size + u) + cols["size"] += list(size) + return {k: np.array(v) for k, v in cols.items()} + + +def test_clean_panel_reads_benign(): + r = screen_dataframe(planted_panel(leak=0.0)) + sig = r["signals"][0] + assert sig["verdict"] == "benign" + assert sig["periods_flagged"] == 0 + assert abs(sig["mean_rho"]) < 0.10 + + +def test_planted_leak_is_flagged_and_gated_to_the_deadline(): + r = screen_dataframe(planted_panel(leak=2.0)) + sig = r["signals"][0] + assert sig["verdict"] == "susceptible" + assert sig["periods_flagged"] == sig["periods_screened"] + # a strongly susceptible signal must be held until the cross-section is complete + assert sig["mean_phi_req"] == pytest.approx(1.0) + + +def test_record_validates_against_the_published_schema(): + r = screen_dataframe(planted_panel(leak=1.0)) + assert validate_results(r) == [] + assert r["config"]["trailing_k"] == 5 + assert r["config"]["rho_threshold"] == pytest.approx(0.10) + + +def test_first_k_periods_are_used_for_fitting_only(): + # 12 periods, k=5 -> exactly 7 screened; the fitted periods are never graded + r = screen_dataframe(planted_panel(n_periods=12), trailing_k=5) + assert r["signals"][0]["periods_screened"] == 7 + r8 = screen_dataframe(planted_panel(n_periods=12), trailing_k=8) + assert r8["signals"][0]["periods_screened"] == 4 + + +def test_several_signals_in_one_pass(): + # 200 entities per period: the sampling noise floor of rho is ~1/sqrt(k*n), + # small enough here that a genuinely unrelated signal stays under threshold + p = planted_panel(n=200, leak=2.0) + p["quiet"] = p["size"] * 0.3 + np.random.default_rng(1).normal(size=len(p["size"])) + r = screen_dataframe(p, value=["value", "quiet"]) + by_name = {s["name"]: s for s in r["signals"]} + assert by_name["value"]["verdict"] == "susceptible" + assert by_name["quiet"]["verdict"] == "benign" + assert r["totals"]["signals_susceptible"] == 1 + assert r["totals"]["signals_benign"] == 1 + + +def test_pandas_polars_and_dict_agree(): + """The screen reads columns, not a dataframe library.""" + base = planted_panel(leak=1.5) + want = screen_dataframe(base) + + pd = pytest.importorskip("pandas") + assert screen_dataframe(pd.DataFrame(base)) == want + + pl = pytest.importorskip("polars") + assert screen_dataframe(pl.DataFrame(base)) == want + + +def test_datetime_arrivals_are_accepted(): + p = planted_panel(leak=2.0) + days = (p["arrival"] - p["arrival"].min()) * 5 + p["filed"] = np.datetime64("2020-01-01") + days.astype("timedelta64[D]") + r = screen_dataframe(p, arrival="filed") + assert r["signals"][0]["verdict"] == "susceptible" + + +def test_custom_column_names(): + p = planted_panel(leak=2.0) + renamed = {"fy": p["period"], "filed_at": p["arrival"], + "accruals": p["value"], "logme": p["size"]} + r = screen_dataframe(renamed, period="fy", arrival="filed_at", + value="accruals", size="logme") + assert r["signals"][0]["name"] == "accruals" + assert r["signals"][0]["verdict"] == "susceptible" + + +def test_short_periods_are_skipped_not_screened_badly(): + p = planted_panel(n_periods=8, n=60) + # starve one period down to 3 entities + keep = ~((p["period"] == 3) & (p["entity"] >= 3)) + p = {k: v[keep] for k, v in p.items()} + stores, kept = stores_from_frame(p, value="value", min_entities=6) + assert 3 not in kept + assert len(stores) == 7 + + +def test_missing_values_are_dropped_per_signal(): + p = planted_panel(leak=2.0) + p["value"] = p["value"].astype(float) + p["value"][:5] = np.nan + r = screen_dataframe(p) # must not raise, must not poison + assert r["signals"][0]["verdict"] == "susceptible" + + +def test_errors_are_actionable(): + p = planted_panel(n_periods=4) + with pytest.raises(KeyError, match="no_such_column"): + screen_dataframe(p, value="no_such_column") + with pytest.raises(ValueError, match="trailing_k"): + screen_dataframe(p, trailing_k=5) # 4 periods, none screenable + with pytest.raises(ValueError, match="at least 1"): + screen_dataframe(p, trailing_k=0) + + +def test_screen_is_deterministic(): + p = planted_panel(leak=1.0) + assert screen_dataframe(p) == screen_dataframe(p) + + +def test_no_network_machinery_in_the_frame_module(): + import pit_release_gate.frame as m + src = open(m.__file__, encoding="utf-8").read() + for forbidden in ("urllib", "requests", "http", "socket", "atexit", "threading"): + assert forbidden not in src, f"frame.py must not reference {forbidden!r}" diff --git a/tools/make_readme_chart.py b/tools/make_readme_chart.py new file mode 100644 index 0000000..6dd08bb --- /dev/null +++ b/tools/make_readme_chart.py @@ -0,0 +1,56 @@ +"""Regenerate docs/assets/demo_bias.png from the shipped fixed-seed demo. + +The chart hardcodes nothing: it runs `run_demo` (the same call the CLI makes) +and plots the systematic size-coefficient bias of naive early release vs the +graded gate for each planted signal type. `tests/test_reproduces_paper.py` +pins the underlying numbers. +""" +import pathlib + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +from pit_release_gate import run_demo + +OUT = pathlib.Path(__file__).resolve().parents[1] / "docs" / "assets" +OUT.mkdir(parents=True, exist_ok=True) + + +def main(): + res = run_demo(verbose=False)["signals"] + order = ["clean", "composition", "mild_leak", "strong_leak"] + labels = ["Clean", "Composition\n(obs. selection)", "Mild leak", "Strong leak"] + naive = [res[k]["policies"]["naive"]["bias_mean"] for k in order] + gated = [res[k]["policies"]["gated"]["bias_mean"] for k in order] + comp = [res[k]["policies"]["gated"]["comp_mean"] for k in order] + + fig, ax = plt.subplots(figsize=(8.6, 4.2), dpi=150) + x = range(len(order)) + w = 0.38 + ax.bar([i - w / 2 for i in x], naive, w, label="naive early release", + color="#c0392b", alpha=0.85) + ax.bar([i + w / 2 for i in x], gated, w, label="graded gate", + color="#2471a3", alpha=0.9) + ax.axhline(0, color="black", lw=0.8) + for i, (g, c) in enumerate(zip(gated, comp)): + ax.annotate(f"releases at {c:.0%}", (i + w / 2, g), + textcoords="offset points", xytext=(0, -14), + ha="center", fontsize=8, color="#2471a3") + ax.set_xticks(list(x)) + ax.set_xticklabels(labels, fontsize=10) + ax.set_ylabel("systematic bias of released signal\n(size-coefficient, signed)") + ax.set_title("Known-ground-truth demo: the gate withholds only the signals that need it", + fontsize=11) + ax.legend(frameon=False, fontsize=9) + ax.spines[["top", "right"]].set_visible(False) + fig.tight_layout() + fig.savefig(OUT / "demo_bias.png", facecolor="white") + print("wrote", OUT / "demo_bias.png") + for k, n, g, c in zip(order, naive, gated, comp): + print(f" {k:<12} naive {n:+.3f} gated {g:+.3f} at {c:.0%}") + + +if __name__ == "__main__": + main()