|
| 1 | +"""CDC / event-time quality gate with ``fd.cdc_profile``. |
| 2 | +
|
| 3 | +Change-data-capture streams fail in ways that are *not* missing values: records |
| 4 | +arrive stale, late, out of order, with duplicate keys, bad operation codes, or as |
| 5 | +a replayed batch. ``cdc_profile`` classifies those freshness/ordering defects |
| 6 | +separately from nulls and reports trust penalties you can gate on. Run: |
| 7 | +
|
| 8 | + python examples/cdc_event_time_gate.py |
| 9 | +""" |
| 10 | + |
| 11 | +import pandas as pd |
| 12 | + |
| 13 | +import freshdata as fd |
| 14 | + |
| 15 | + |
| 16 | +def main() -> None: |
| 17 | + # An order-change feed captured from a CDC connector, in arrival order. |
| 18 | + changes = pd.DataFrame( |
| 19 | + { |
| 20 | + "order_id": [101, 101, 102, 103, 103, 101, 104], |
| 21 | + "event_ts": [ |
| 22 | + "2024-03-01 09:00:00", |
| 23 | + "2024-03-01 09:05:00", |
| 24 | + "2024-03-01 09:06:00", |
| 25 | + "2024-03-01 09:05:30", # 102/103 interleave is fine (different keys) |
| 26 | + None, # missing event time |
| 27 | + "2024-03-01 08:30:00", # 101 arrives 35m late, past the watermark |
| 28 | + "2024-03-01 09:05:30", |
| 29 | + ], |
| 30 | + "op": ["c", "u", "c", "c", "u", "u", "Q"], # 'Q' is not a valid op code |
| 31 | + } |
| 32 | + ) |
| 33 | + |
| 34 | + report = fd.cdc_profile( |
| 35 | + changes, |
| 36 | + event_time="event_ts", |
| 37 | + key="order_id", |
| 38 | + lateness="10m", |
| 39 | + operation_col="op", |
| 40 | + now=pd.Timestamp("2024-03-01 09:10:00"), |
| 41 | + stale_after="30m", |
| 42 | + ) |
| 43 | + |
| 44 | + print(report.summary()) |
| 45 | + print("\ntrust penalties (0=clean, 1=worst):", report.trust_penalties) |
| 46 | + print(f"freshness: {report.freshness_seconds:.0f}s behind reference") |
| 47 | + print(f"\ngate passed: {report.passed}\n") |
| 48 | + print("defect table:") |
| 49 | + print(report.to_frame()[["kind", "level", "n_rows", "rationale"]].to_string(index=False)) |
| 50 | + |
| 51 | + |
| 52 | +if __name__ == "__main__": |
| 53 | + main() |
0 commit comments