From b216d8348ef30fd869107cb81453cf1cfc57d76c Mon Sep 17 00:00:00 2001 From: ShravyaHegade Date: Wed, 26 Aug 2026 09:03:36 +0530 Subject: [PATCH 1/2] Add pandas DataFrame worked example --- README.md | 1 + docs/from-dataframe.md | 182 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 docs/from-dataframe.md diff --git a/README.md b/README.md index 2964ff6..f22be55 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Completeness-aware release control for staggered-arrival cross-sectional data. **[Audit registry](https://github.com/MaxWellApexLab/pit-audit-registry) · [Pledge](https://github.com/MaxWellApexLab/pit-hygiene) · [Papers](#papers) · +[Worked example: pandas DataFrame](docs/from-dataframe.md) · [Result schema](docs/results-schema.md)** Rebuilt from as-filed SEC EDGAR filings, on observed filing dates, **7 of 14 diff --git a/docs/from-dataframe.md b/docs/from-dataframe.md new file mode 100644 index 0000000..dabe39a --- /dev/null +++ b/docs/from-dataframe.md @@ -0,0 +1,182 @@ +\# Worked example: build an `AsOfDataStore` from a pandas DataFrame + + + +If your input already looks like a long table, this is the normal path: + + + +\- one row per `(entity, period)`; + +\- a filing-arrival timestamp or date; + +\- a signal value you want to gate; + +\- a size column the gate conditions on. + + + +The idea is simple: + + + +1\. take one period; + +2\. build a design matrix from the size column; + +3\. normalize the arrival times to `\[0, 1]` inside that period; + +4\. package those pieces into `AsOfDataStore`; + +5\. fit the gate on earlier completed periods; + +6\. call `ReleaseController.decide(...)` on the live period. + + + +This is the same pattern production pipelines use: honest estimation first, then a gated decision on the new period. + + + +```python + +import numpy as np + +import pandas as pd + + + +from pit\_release\_gate import AsOfDataStore, ReleaseController, SusceptibilityGate + + + + + +def build\_store(frame: pd.DataFrame) -> AsOfDataStore: + + """Turn one period of a long table into one as-of store.""" + + size = (frame\["size"] - frame\["size"].mean()) / frame\["size"].std(ddof=0) + + X = np.column\_stack(\[np.ones(len(frame)), size.to\_numpy()]) + + y = frame\["signal\_value"].to\_numpy() + + + + # The estimand is the complete-cross-section residual from the design matrix. + + beta, \*\_ = np.linalg.lstsq(X, y, rcond=None) + + truth\_resid = y - X @ beta + + + + # Arrival is a time inside the period: 0 = earliest filer, 1 = deadline. + + arrival\_ts = pd.to\_datetime(frame\["arrival\_date"]).astype("int64").to\_numpy() / 1e9 + + lo = arrival\_ts.min() + + hi = arrival\_ts.max() + + if hi == lo: + + arrival = np.zeros\_like(arrival\_ts, dtype=float) + + else: + + arrival = (arrival\_ts - lo) / (hi - lo) + + + + return AsOfDataStore( + + X=X, + + y=y, + + arrival=arrival, + + size=size.to\_numpy(), + + truth\_resid=truth\_resid, + + ) + + + + + +rows = \[] + +for period in \[2023, 2024, 2025]: + + for entity in \["A", "B", "C", "D", "E", "F"]: + + size = 10 + (ord(entity) % 10) + 0.2 \* period + + signal\_value = 5 + 0.9 \* size + (0.4 if period in (2023, 2024) else 0.0) + + arrival\_date = pd.Timestamp(f"{period}-01-01") + pd.Timedelta( + + days=(ord(entity) % 6) \* 5 + (period - 2023) \* 12 + + ) + + rows.append( + + { + + "entity": entity, + + "period": period, + + "arrival\_date": arrival\_date, + + "signal\_value": signal\_value, + + "size": size, + + } + + ) + + + +panel = pd.DataFrame(rows) + + + +\# Honest estimate: fit on earlier completed periods only. + +completed = \[build\_store(panel\[panel\["period"] == p]) for p in \[2023, 2024]] + +gate = SusceptibilityGate(threshold=0.10) + +rho = gate.fit\_trailing(completed) + + + +\# Live period: the gate uses the frozen rho estimate from the completed periods. + +live = build\_store(panel\[panel\["period"] == 2025]) + +controller = ReleaseController(gate=gate) + +decision = controller.decide(live, t=1.0, policy="gated") + + + +assert decision.action == "RELEASE" + +assert decision.t == 1.0 + +assert decision.completeness == 1.0 + +assert decision.values is not None + + + +print(f"rho\_hat={rho:.3f} -> {decision.action} at completeness {decision.completeness:.0%}") + From c4b097cb6225fd5f9a1cca70cc824fafef43aa2f Mon Sep 17 00:00:00 2001 From: Shravya Ravindra Hegade Date: Wed, 26 Aug 2026 09:16:01 +0530 Subject: [PATCH 2/2] Refactor documentation for AsOfDataStore example Refactor the documentation for building an AsOfDataStore from a pandas DataFrame. Improved clarity and formatting of the example code and explanations. --- docs/from-dataframe.md | 226 +++++++++++++---------------------------- 1 file changed, 72 insertions(+), 154 deletions(-) diff --git a/docs/from-dataframe.md b/docs/from-dataframe.md index dabe39a..22bf474 100644 --- a/docs/from-dataframe.md +++ b/docs/from-dataframe.md @@ -1,182 +1,100 @@ -\# Worked example: build an `AsOfDataStore` from a pandas DataFrame - - +# Worked example: build an `AsOfDataStore` from a pandas DataFrame If your input already looks like a long table, this is the normal path: - - -\- one row per `(entity, period)`; - -\- a filing-arrival timestamp or date; - -\- a signal value you want to gate; - -\- a size column the gate conditions on. - - +- one row per `(entity, period)`; +- a filing-arrival timestamp or date; +- a signal value you want to gate; +- a size column the gate conditions on. The idea is simple: - - -1\. take one period; - -2\. build a design matrix from the size column; - -3\. normalize the arrival times to `\[0, 1]` inside that period; - -4\. package those pieces into `AsOfDataStore`; - -5\. fit the gate on earlier completed periods; - -6\. call `ReleaseController.decide(...)` on the live period. - - +1. take one period; +2. build a design matrix from the size column; +3. normalize the arrival times to `[0, 1]` inside that period; +4. package those pieces into `AsOfDataStore`; +5. fit the gate on earlier completed periods; +6. call `ReleaseController.decide(...)` on the live period. This is the same pattern production pipelines use: honest estimation first, then a gated decision on the new period. - - ```python - import numpy as np - import pandas as pd - - -from pit\_release\_gate import AsOfDataStore, ReleaseController, SusceptibilityGate - - - - - -def build\_store(frame: pd.DataFrame) -> AsOfDataStore: - - """Turn one period of a long table into one as-of store.""" - - size = (frame\["size"] - frame\["size"].mean()) / frame\["size"].std(ddof=0) - - X = np.column\_stack(\[np.ones(len(frame)), size.to\_numpy()]) - - y = frame\["signal\_value"].to\_numpy() - - - - # The estimand is the complete-cross-section residual from the design matrix. - - beta, \*\_ = np.linalg.lstsq(X, y, rcond=None) - - truth\_resid = y - X @ beta - - - - # Arrival is a time inside the period: 0 = earliest filer, 1 = deadline. - - arrival\_ts = pd.to\_datetime(frame\["arrival\_date"]).astype("int64").to\_numpy() / 1e9 - - lo = arrival\_ts.min() - - hi = arrival\_ts.max() - - if hi == lo: - - arrival = np.zeros\_like(arrival\_ts, dtype=float) - - else: - - arrival = (arrival\_ts - lo) / (hi - lo) - - - - return AsOfDataStore( - - X=X, - - y=y, - - arrival=arrival, - - size=size.to\_numpy(), - - truth\_resid=truth\_resid, - - ) - - - - - -rows = \[] - -for period in \[2023, 2024, 2025]: - - for entity in \["A", "B", "C", "D", "E", "F"]: - - size = 10 + (ord(entity) % 10) + 0.2 \* period - - signal\_value = 5 + 0.9 \* size + (0.4 if period in (2023, 2024) else 0.0) - - arrival\_date = pd.Timestamp(f"{period}-01-01") + pd.Timedelta( - - days=(ord(entity) % 6) \* 5 + (period - 2023) \* 12 - - ) - - rows.append( - - { - - "entity": entity, - - "period": period, - - "arrival\_date": arrival\_date, - - "signal\_value": signal\_value, - - "size": size, - - } - - ) - - +from pit_release_gate import AsOfDataStore, ReleaseController, SusceptibilityGate + + +def build_store(frame: pd.DataFrame) -> AsOfDataStore: + """Turn one period of a long table into one as-of store.""" + size = (frame["size"] - frame["size"].mean()) / frame["size"].std(ddof=0) + X = np.column_stack([np.ones(len(frame)), size.to_numpy()]) + y = frame["signal_value"].to_numpy() + + # The estimand is the complete-cross-section residual from the design matrix. + beta, *_ = np.linalg.lstsq(X, y, rcond=None) + truth_resid = y - X @ beta + + # Arrival is a time inside the period: 0 = earliest filer, 1 = deadline. + arrival_ts = pd.to_datetime(frame["arrival_date"]).astype("int64").to_numpy() / 1e9 + lo = arrival_ts.min() + hi = arrival_ts.max() + if hi == lo: + arrival = np.zeros_like(arrival_ts, dtype=float) + else: + arrival = (arrival_ts - lo) / (hi - lo) + + return AsOfDataStore( + X=X, + y=y, + arrival=arrival, + size=size.to_numpy(), + truth_resid=truth_resid, + ) + + +rows = [] +for period in [2023, 2024, 2025]: + for entity in ["A", "B", "C", "D", "E", "F"]: + size = 10 + (ord(entity) % 10) + 0.2 * period + signal_value = 5 + 0.9 * size + (0.4 if period in (2023, 2024) else 0.0) + arrival_date = pd.Timestamp(f"{period}-01-01") + pd.Timedelta( + days=(ord(entity) % 6) * 5 + (period - 2023) * 12 + ) + rows.append( + { + "entity": entity, + "period": period, + "arrival_date": arrival_date, + "signal_value": signal_value, + "size": size, + } + ) panel = pd.DataFrame(rows) - - -\# Honest estimate: fit on earlier completed periods only. - -completed = \[build\_store(panel\[panel\["period"] == p]) for p in \[2023, 2024]] - +# Honest estimate: fit on earlier completed periods only. +completed = [build_store(panel[panel["period"] == p]) for p in [2023, 2024]] gate = SusceptibilityGate(threshold=0.10) +rho = gate.fit_trailing(completed) -rho = gate.fit\_trailing(completed) - - - -\# Live period: the gate uses the frozen rho estimate from the completed periods. - -live = build\_store(panel\[panel\["period"] == 2025]) - +# Live period: the gate uses the frozen rho estimate from the completed periods. +live = build_store(panel[panel["period"] == 2025]) controller = ReleaseController(gate=gate) - decision = controller.decide(live, t=1.0, policy="gated") - - assert decision.action == "RELEASE" - assert decision.t == 1.0 - assert decision.completeness == 1.0 - assert decision.values is not None +print(f"rho_hat={rho:.3f} -> {decision.action} at completeness {decision.completeness:.0%}") +``` +What happened here? +- `panel` was a long table, not a custom object. +- `build_store()` converted that long table into one `AsOfDataStore` for one period. +- `gate.fit_trailing(completed)` learned the susceptibility estimate from earlier periods only. +- `controller.decide(live, t=1.0, policy="gated")` asked, "Should we release now?" At the deadline, the answer is `RELEASE` because the full cross-section has arrived. -print(f"rho\_hat={rho:.3f} -> {decision.action} at completeness {decision.completeness:.0%}") - +The important point is that the gate never uses the same period it is trying to judge. That is the honest-estimation rule inside `pit-release-gate`.