diff --git a/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.md b/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.md index 754bb78a..77f489a8 100644 --- a/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.md +++ b/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.md @@ -1,5 +1,18 @@ # `rsi_meanrev`'s edge is selectivity, not alpha — and the search for it found a simulator defect +> **⚠️ AMENDED 2026-08-13 — read alongside +> [`2026-08-13-restated-under-a-production-faithful-engine.md`](2026-08-13-restated-under-a-production-faithful-engine.md).** +> +> Every number below was produced by an engine carrying two defects, since fixed (#256, #258). +> **All conclusions here survive**, and the central one strengthens: the level shift across the +> trade floor widens from 1.1631 → 0.8938 to **1.1251 → 0.8396**, and gross-positive cells at the +> floor nearly halve (11/76 → 6/82). Still 0 net-positive at maker. +> +> §5's monotonicity anomaly — the 34× UNI-USD collapse that led to both fixes — is **structurally +> gone**: 3 non-monotonic assets → 0. +> +> These numbers are **annotated, not restated**, per the convention #247 set. + **Date:** 2026-08-12 **Issue:** #253 — closes the single open question left by #252 **Change:** documentation only. No code, no config, no rule status, no version bump. The simulator diff --git a/docs/experiments/2026-08-12-shipped-defaults-intersection.md b/docs/experiments/2026-08-12-shipped-defaults-intersection.md index 57dec50a..fd325119 100644 --- a/docs/experiments/2026-08-12-shipped-defaults-intersection.md +++ b/docs/experiments/2026-08-12-shipped-defaults-intersection.md @@ -1,5 +1,21 @@ # Three rules, 24 assets, zero free parameters — the viable intersection is empty, and one of the three fails for the opposite reason we recorded +> **⚠️ AMENDED 2026-08-13 — read alongside +> [`2026-08-13-restated-under-a-production-faithful-engine.md`](2026-08-13-restated-under-a-production-faithful-engine.md).** +> +> Every number below was produced by an engine carrying two defects, since fixed: a pending setup +> that never expired and silently switched the detector off (#256), and entries that filled at a +> price production never waits for (#258). They pushed in opposite directions — one suppressed +> opportunity, the other flattered execution. +> +> Re-measured on the corrected engine, **every conclusion here survives except one**: ZEC-`turtle` +> no longer clears the maker line (1.034 → 0.968), so §6's three-probe narrative describes a +> survivor the faithful engine does not produce. The finding becomes simpler — the viable quadrant +> is empty at every reachable fee, 0 of 90, with nothing needing three probes to eliminate. +> +> These numbers are **annotated, not restated**, per the convention #247 set: they were real +> outputs of the code as it stood. + **Date:** 2026-08-12 **Issue:** #251 **Change:** documentation only. No code, no config, no rule status, no parameter, no version bump. diff --git a/docs/experiments/2026-08-13-restated-intersection.py b/docs/experiments/2026-08-13-restated-intersection.py new file mode 100644 index 00000000..b07cf2a7 --- /dev/null +++ b/docs/experiments/2026-08-13-restated-intersection.py @@ -0,0 +1,187 @@ +import json +import time +from concurrent.futures import ProcessPoolExecutor, as_completed +from decimal import Decimal + +DB = "/Users/elmehdiaitbrahim/keel/keel.db" +OUT_DIR = "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/28ff9a61-09d1-498b-b325-1631c0662734/scratchpad" +JSONL_PATH = f"{OUT_DIR}/intersection_257.jsonl" +JSON_PATH = f"{OUT_DIR}/intersection_257.json" + +UNIVERSE = [ + "BTC-USD", "ETH-USD", "ADA-USD", "LINK-USD", "LTC-USD", "SOL-USD", + "XLM-USD", "PAXG-USDT", "BCH-USD", "AAVE-USD", "DOGE-USD", "DOT-USD", + "UNI-USD", "ZEC-USD", "ALGO-USD", "FET-USD", "CRV-USD", "ICP-USD", + "AVAX-USD", "NEAR-USD", "XRP-USD", "PAXG-USD", "WLD-USD", "TON-USD", +] + +ARM_B_EXCLUDE = {"ZEC-USD", "FET-USD", "SOL-USD", "DOGE-USD", "ETH-USD", "BTC-USD"} +ARM_B_UNIVERSE = [a for a in UNIVERSE if a not in ARM_B_EXCLUDE] + +FEES = ["0", "0.006", "0.012"] +SLIPPAGE = Decimal("0.0005") + +RULES = ["turtle", "rsi", "pullback"] + + +def build_jobs(): + jobs = [] + for rule in RULES: + for asset in UNIVERSE: + jobs.append(("A", rule, asset)) + for asset in ARM_B_UNIVERSE: + jobs.append(("B", "turtle", asset)) + return jobs + + +def make_rule(arm, rule, asset): + from keel.strategy.rules.turtle_breakout import TurtleBreakout + from keel.strategy.rules.rsi_meanrev import RsiMeanReversion + from keel.strategy.rules.pullback_continuation import PullbackContinuation + + if arm == "A": + if rule == "turtle": + return TurtleBreakout(product_id=asset) + elif rule == "rsi": + return RsiMeanReversion(product_id=asset) + elif rule == "pullback": + return PullbackContinuation(product_id=asset) + else: + raise ValueError(f"unknown rule {rule}") + elif arm == "B": + if rule != "turtle": + raise ValueError("arm B is turtle only") + return TurtleBreakout( + product_id=asset, + entry_lookback=336, + exit_lookback=80, + atr_stop_mult=Decimal("2"), + target_rr=Decimal("6"), + adx_threshold=25.0, + ) + else: + raise ValueError(f"unknown arm {arm}") + + +def run_job(job): + arm, rule, asset = job + from keel.data.db import connect + from keel.data.repository import Repository + from keel.strategy import backtest as bt + from keel_core.types import Granularity + + rows = [] + try: + repo = Repository(connect(DB)) + candles = repo.get_candles(asset, Granularity.ONE_HOUR) + except Exception as e: + for fee in FEES: + rows.append({ + "arm": arm, "rule": rule, "product": asset, "fee": fee, + "error": f"{type(e).__name__}: {e}", + }) + return rows + + for fee in FEES: + try: + rule_obj = make_rule(arm, rule, asset) + result = bt.backtest( + rule_obj, candles, fee_pct=Decimal(fee), slippage_pct=SLIPPAGE + ) + rows.append({ + "arm": arm, + "rule": rule, + "product": asset, + "fee": fee, + "n_trades": int(result.n_trades), + "win_rate": float(result.win_rate), + "profit_factor": float(result.profit_factor), + "expectancy": float(result.expectancy), + "max_drawdown": float(result.max_drawdown), + }) + except Exception as e: + rows.append({ + "arm": arm, "rule": rule, "product": asset, "fee": fee, + "error": f"{type(e).__name__}: {e}", + }) + return rows + + +def done_combos(): + """Combos already in the JSONL. The first run died at 25/90 with the pool still + holding results; the file is append-only and each row names its own combo, so the + completed set is recoverable exactly. Resume rather than redo.""" + import os + + if not os.path.exists(JSONL_PATH): + return set() + done = set() + for line in open(JSONL_PATH): + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue # torn final line from a kill mid-write + done.add((r["arm"], r["rule"], r["product"])) + return done + + +def main(): + jobs = build_jobs() + total_declared = len(jobs) + done = done_combos() + jobs = [j for j in jobs if j not in done] + + # Run order is a scheduling choice ONLY -- every declared job still runs, and a backtest is + # independent of the order it is dispatched in, so this cannot touch the result. + # `rsi` at the shipped defaults is by far the slowest cell: `backtest()` calls `rule.detect()` + # only while flat, so the rule that almost never fires pays full level-detection cost on + # nearly all 44k bars. Leaving it first starves the two arms that actually answer the + # open question (Arm B's out-of-sample transfer, and pullback's first-ever run) for an hour. + prio = {("B", "turtle"): 0, ("A", "pullback"): 1, ("A", "rsi"): 2} + jobs.sort(key=lambda j: prio.get((j[0], j[1]), 3)) + total = len(jobs) + print(f"Declared jobs: {total_declared}; already done: {len(done)}; running: {total}", flush=True) + + start = time.time() + completed = 0 + all_rows = [] + + with open(JSONL_PATH, "a") as jf: + with ProcessPoolExecutor(max_workers=8) as ex: + futures = {ex.submit(run_job, job): job for job in jobs} + for fut in as_completed(futures): + job = futures[fut] + try: + rows = fut.result() + except Exception as e: + arm, rule, asset = job + rows = [{ + "arm": arm, "rule": rule, "product": asset, "fee": fee, + "error": f"{type(e).__name__}: {e}", + } for fee in FEES] + + for row in rows: + jf.write(json.dumps(row) + "\n") + jf.flush() + all_rows.append(row) + + completed += 1 + if completed % 10 == 0 or completed == total: + elapsed = time.time() - start + print(f"Progress: {completed}/{total} jobs, elapsed={elapsed:.1f}s", flush=True) + + # Serialise the WHOLE jsonl, not just this process's rows -- on a resume `all_rows` + # holds only the newly-run subset and dumping it would silently drop the earlier run. + final = [json.loads(x) for x in open(JSONL_PATH) if x.strip()] + with open(JSON_PATH, "w") as f: + json.dump(final, f, indent=2) + + elapsed = time.time() - start + print(f"Done. {len(all_rows)} new rows, {len(final)} total. elapsed={elapsed:.1f}s", flush=True) + + +if __name__ == "__main__": + main() diff --git a/docs/experiments/2026-08-13-restated-rsi-scale.py b/docs/experiments/2026-08-13-restated-rsi-scale.py new file mode 100644 index 00000000..a3118d5d --- /dev/null +++ b/docs/experiments/2026-08-13-restated-rsi-scale.py @@ -0,0 +1,253 @@ +"""Does `rsi_meanrev`'s gross edge survive being made to fire more often? + +PRE-REGISTERED IN THIS FILE, BEFORE THE RUN. This docstring is the declaration -- unlike +`2026-08-12-shipped-defaults-intersection.py`, whose declaration lived in a dispatch brief and had +to be reconstructed afterwards. That was recorded as a defect; this is the correction. + +## The question, and why it is not an optimisation + +`2026-08-12-shipped-defaults-intersection.md` measured all three shipped rules at their +constructor defaults and found `rsi_meanrev` has the BEST gross-edge distribution of the three +(median gross PF 1.1631, 58% of assets gross-positive) while reaching the promotion floor on ZERO +of 24 assets (median n=38 against min_trades=100). It does not lose. It is not observable. + +Two explanations, with opposite consequences, and no data yet separates them: + + (a) the edge is REAL and the defaults are simply over-constrained -- in which case relaxing + them reaches n>=100 with the edge intact, and this is the only promotable rule in the + codebase; + (b) the edge is an ARTIFACT OF SELECTIVITY -- the rule looks good precisely because it only + fires on the rare, easy setups, and buying trades means accepting worse ones. + +## The design, and why it cannot cherry-pick + +This is a MONOTONICITY TEST, not a search. The reported statistic is the SLOPE of gross profit +factor against n, computed per asset across the oversold levels, then averaged across the 24 +assets. **The best cell is never reported as a result.** With 120 cells a maximum is guaranteed; +a slope is not, and a slope cannot be manufactured by trying more cells. + +PRE-REGISTERED PREDICTION, recorded so the result can contradict it: + + slope < 0 -> hypothesis (b). The rule is unpromotable BY CONSTRUCTION: no parameter choice + escapes buying volume with quality. Report and retire the line of enquiry. + slope >= 0 -> hypothesis (a). The edge is scalable and the defaults are the problem. This is + the single outcome in the whole study that points at a promotable rule, and it + would then need a fresh out-of-sample confirmation before any promotion. + +Either outcome is publishable. Neither requires a winner, which is the point. + +## Axes + +VARIABLE, one only: + oversold in {20, 25, 30, 35, 40} + +`oversold` is the entire frequency mechanism and this is measured, not assumed: the 108-cell +diagnostic in `2026-08-12-fee-curve-and-rsi-meanrev-diag.py` found oversold 25->30 multiplied +trade count x2.18 and 30->35 by x3.93, against x1.186 for `support_proximity_pct` and x1.185 for +`level_min_touches`. The other two are noise on this axis. + +FIXED, deliberately: + overbought = 80 (shipped default) + support_proximity_pct = 0.005 (shipped default) + everything else = shipped defaults + +`overbought` is held not because it is a weak lever but because it is the WRONG KIND of lever: it +governs the exit side, so moving it changes trade OUTCOMES and not merely trade COUNTS. If both +axes move, a fall in gross PF cannot be attributed to firing more rather than exiting differently, +and the slope -- the entire point of the run -- becomes uninterpretable. + +`support_proximity_pct` is held because it is a STRUCTURAL filter (distance to a level) while +`oversold` is a MOMENTUM filter. Sweeping both confounds "does firing more degrade edge" with +"does relaxing which filter degrade edge". + +## Anchor + +oversold=20 is the shipped default and was already measured across all 24 assets by the +intersection run. Those rows are REUSED as the curve's left-hand anchor rather than recomputed, so +this grid runs 4 new levels x 24 assets = 96 combinations. The anchor rows are identical in every +other parameter (all defaults) and in cost treatment (same three fees, same explicit 0.0005 +slippage pin), which is what makes them poolable with the new ones. + +## Conditional second arm, DECLARED NOW so it cannot become a post-hoc rescue + + TRIGGER: if FEWER THAN 8 of 24 assets reach n >= 100 at oversold = 40. + THEN: add support_proximity_pct in {0.005, 0.02, 0.05} as a second axis. + REPORT: as a SEPARATE curve. Never pooled with, averaged into, or compared cell-to-cell + against the primary arm -- it varies a different filter and answers a different + question. + +Declaring the trigger and the reporting rule before any data exists is what stops the widening +from being invented later to rescue a disappointing primary arm. It writes to its own output file +for the same reason. + +## Costs + +Fees 0 / 0.006 (maker) / 0.012 (taker); `slippage_pct` pinned EXPLICITLY at 0.0005. The pin makes +every figure independent of library defaults and keeps the "zero fee" column honestly labelled -- +it is zero FEE, not zero COST. + +## Compute + +`rsi_meanrev` at LOW oversold is the slowest cell in the codebase: `backtest()` calls +`rule.detect()` only while flat, so a rule that almost never fires pays full support-level +detection on nearly all 44k bars. The grid therefore gets CHEAPER as oversold rises. Expect the +oversold=25 block to dominate wall-clock. +""" + +from __future__ import annotations + +import itertools +import json +import os +import time +from concurrent.futures import ProcessPoolExecutor +from decimal import Decimal + +DB = "/Users/elmehdiaitbrahim/keel/keel.db" +SCRATCH = ( + "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/" + "28ff9a61-09d1-498b-b325-1631c0662734/scratchpad" +) +ANCHOR = f"{SCRATCH}/intersection_257.jsonl" # supplies the oversold=20 rows +OUT_PRIMARY = f"{SCRATCH}/rsi_scale_257.jsonl" +OUT_CONDITIONAL = f"{SCRATCH}/rsi_scale_257_proximity.jsonl" # separate file, never merged + +UNIVERSE = [ + "BTC-USD", "ETH-USD", "ADA-USD", "LINK-USD", "LTC-USD", "SOL-USD", + "XLM-USD", "PAXG-USDT", "BCH-USD", "AAVE-USD", "DOGE-USD", "DOT-USD", + "UNI-USD", "ZEC-USD", "ALGO-USD", "FET-USD", "CRV-USD", "ICP-USD", + "AVAX-USD", "NEAR-USD", "XRP-USD", "PAXG-USD", "WLD-USD", "TON-USD", +] + +FEES = ["0", "0.006", "0.012"] +SLIPPAGE = Decimal("0.0005") + +ANCHOR_OVERSOLD = 20.0 +NEW_LEVELS = [25.0, 30.0, 35.0, 40.0] +PROXIMITY_LEVELS = ["0.005", "0.02", "0.05"] + +TRIGGER_MIN_ASSETS = 8 +TRIGGER_AT_OVERSOLD = 40.0 + + +def _run(job: tuple[str, float, str]) -> list[dict]: + product, oversold, proximity = job + from keel.data.db import connect + from keel.data.repository import Repository + from keel.strategy import backtest as bt + from keel.strategy.rules.rsi_meanrev import RsiMeanReversion + from keel_core.types import Granularity + + out: list[dict] = [] + try: + candles = Repository(connect(DB)).get_candles(product, Granularity.ONE_HOUR) + except Exception as exc: + return [ + {"product": product, "oversold": oversold, "proximity": proximity, "fee": f, + "error": f"{type(exc).__name__}: {exc}"} + for f in FEES + ] + + for fee in FEES: + try: + rule = RsiMeanReversion( + product_id=product, + oversold=oversold, + support_proximity_pct=Decimal(proximity), + ) + r = bt.backtest(rule, candles, fee_pct=Decimal(fee), slippage_pct=SLIPPAGE) + out.append({ + "product": product, "oversold": oversold, "proximity": proximity, "fee": fee, + "n_trades": int(r.n_trades), + "win_rate": float(r.win_rate), + "profit_factor": float(r.profit_factor), + "expectancy": float(r.expectancy), + }) + except Exception as exc: + out.append({"product": product, "oversold": oversold, "proximity": proximity, + "fee": fee, "error": f"{type(exc).__name__}: {exc}"}) + return out + + +def _done(path: str) -> set: + if not os.path.exists(path): + return set() + seen = set() + for line in open(path): + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue # torn final line + seen.add((r["product"], r["oversold"], r["proximity"])) + return seen + + +def _execute(jobs: list, path: str, label: str) -> None: + jobs = [j for j in jobs if (j[0], j[1], j[2]) not in _done(path)] + print(f"[{label}] running {len(jobs)} combinations", flush=True) + if not jobs: + return + t0 = time.perf_counter() + with ProcessPoolExecutor(max_workers=8) as ex, open(path, "a") as fh: + for i, rows in enumerate(ex.map(_run, jobs, chunksize=1), 1): + for row in rows: + fh.write(json.dumps(row) + "\n") + fh.flush() + if i % 4 == 0 or i == len(jobs): + print(f"[{label}] {i}/{len(jobs)} {time.perf_counter() - t0:.0f}s", flush=True) + + +def anchor_rows() -> list[dict]: + """The oversold=20 rows from the intersection run, relabelled into this grid's schema.""" + rows = [] + for line in open(ANCHOR): + line = line.strip() + if not line: + continue + r = json.loads(line) + if r.get("rule") != "rsi" or r.get("arm") != "A" or "error" in r: + continue + rows.append({**r, "oversold": ANCHOR_OVERSOLD, "proximity": "0.005", "anchor": True}) + return rows + + +def main() -> None: + print(f"PRIMARY ARM: oversold {NEW_LEVELS} x {len(UNIVERSE)} assets " + f"(+ {ANCHOR_OVERSOLD} reused as anchor)", flush=True) + jobs = [(a, o, "0.005") for o in NEW_LEVELS for a in UNIVERSE] + _execute(jobs, OUT_PRIMARY, "primary") + + # Evaluate the PRE-DECLARED trigger. No judgement is applied here -- the condition and the + # threshold were both fixed in the docstring above before any of this ran. + rows = [json.loads(x) for x in open(OUT_PRIMARY) if x.strip()] + at_max = { + r["product"] + for r in rows + if "error" not in r and r["oversold"] == TRIGGER_AT_OVERSOLD + and r["fee"] == "0" and r["n_trades"] >= 100 + } + print(f"\nTRIGGER CHECK: {len(at_max)}/{len(UNIVERSE)} assets reach n>=100 at " + f"oversold={TRIGGER_AT_OVERSOLD} (threshold: fewer than {TRIGGER_MIN_ASSETS} fires it)", + flush=True) + + if len(at_max) < TRIGGER_MIN_ASSETS: + print("TRIGGERED -> running the pre-declared conditional proximity arm, " + "to its own file, reported as a separate curve.", flush=True) + cond = [ + (a, o, p) + for o, p in itertools.product(NEW_LEVELS, PROXIMITY_LEVELS) + if p != "0.005" # 0.005 already covered by the primary arm + for a in UNIVERSE + ] + _execute(cond, OUT_CONDITIONAL, "conditional") + else: + print("NOT triggered -- primary arm reached the frequency floor on its own.", flush=True) + + print("\ndone", flush=True) + + +if __name__ == "__main__": + main() diff --git a/docs/experiments/2026-08-13-restated-under-a-production-faithful-engine.md b/docs/experiments/2026-08-13-restated-under-a-production-faithful-engine.md new file mode 100644 index 00000000..4a614ada --- /dev/null +++ b/docs/experiments/2026-08-13-restated-under-a-production-faithful-engine.md @@ -0,0 +1,223 @@ +# Two engine defects, both invisible to 2,712 passing tests — and what the conclusions become without them + +**Date:** 2026-08-13 +**Amends:** `2026-08-12-shipped-defaults-intersection.md` (#252) and +`2026-08-12-rsi-meanrev-scale-vs-selectivity.md` (#255) +**Engine changes it restates them under:** #256 (`fix/pending-setup-never-expires`) and #258 +(`fix/next-bar-open-fills`) +**Ledger:** two rows, session `restated-production-faithful-engine-2026-08-13`. + +**This document annotates; it does not rewrite.** #252 and #255 keep their numbers exactly as +printed, per the convention #247 set: *"Past numbers in `docs/experiments/` were real outputs of +the code as it stood; they are annotated by this change, not restated."* Each remains a true record +of what its engine produced. What follows is what those same experiments produce on an engine that +matches production. + +**Verdict: every conclusion in #252 and #255 survives except one, and losing it makes the finding +simpler and harder. Under production-faithful execution the viable quadrant is empty at every +reachable fee — 0 of 90 in #252's matrix and 0 of 82 in #255's — with no survivor requiring three +probes to eliminate.** + +--- + +## 1. What was wrong with the engine + +Two defects, found within hours of each other, pushing in **opposite directions**. + +**#256 — a pending setup never expired.** `backtest()` held an unfilled `Setup` indefinitely; the +only escape without a fill was the stop being touched first. A setup whose entry and stop were both +never revisited pinned `pending` forever, so the flat branch never ran again and `rule.detect()` +was never called again. **The engine switched its own detector off.** Measured: `rsi_meanrev` on +UNI-USD at `oversold=35` stopped detecting in November 2021 and sat dead for ~40,000 bars — 9 +trades against 309 at the *stricter* `oversold=30`. + +**#258 — entries filled at a price production never waits for.** The simulator held the setup until +a later bar's range *touched* `entry`, then filled *at that level*. Live places market orders +(`order_type="market"`, `limit_price=None`, the setup's price kept only as `expected_fill`), so it +never rests an order and never waits. The old model granted two things production does not have: +free optionality on the entry price (unfavourable entries were silently declined, because a setup +only became a trade if the market offered the chosen level) and unbounded patience. + +**The directions matter, and this is the single most useful sentence in this document:** + +> #256 suppressed **opportunity**. #258 flattered **execution**. Correcting both did not produce +> symmetrical noise — it moved everything the same way. Across the 90-combination matrix, trade +> counts rose in **87 of 90** and gross profit factors fell in **69 of 90**. + +Every prior conclusion in `docs/experiments/` was therefore measured on an engine that was +simultaneously too pessimistic about how often a rule fires and too optimistic about what it pays +to get in. + +## 2. The one conclusion that changed + +#252 recorded ZEC-`turtle` as the single cell clearing `n≥100 ∧ gross>1 ∧ maker>1`, subsequently +eliminated by a temporal probe showing three consecutive losing years. Under the faithful engine it +never clears at all: + +``` +ZEC-USD turtle touch-fill (#256): n=247 gross 1.542 maker 1.034 + market-fill (#258): n=268 gross 1.442 maker 0.968 +``` + +That collapses #252 §6's architecture. Its argument — three probes, each catching what the others +miss, with ZEC as the through-line surviving the fee curve and the tail test before dying to the +temporal one — describes a survivor the faithful engine does not produce. + +**The replacement is shorter and worse for the strategy library.** Nothing needs three gates to +die, because nothing passes the first. The temporal and tail probes are not thereby worthless: +they remain the reason we know ZEC's apparent edge was regime-bound (92.7% of lifetime PnL in +2025–26) and tail-carried (six of seven gross-positive assets fell below break-even on deleting +three trades). They are repositioned from *"the mechanism that killed the last survivor"* to +*"forensics explaining an artifact a defective engine manufactured."* + +#255's conclusions all survive unchanged. + +## 3. #252 restated + +| rule | median n | n≥100 | median gross | C1+C2 | C1+C2+C3 maker | taker | +|---|---|---|---|---|---|---| +| `turtle_breakout` | 241 → **262** | 21 → **21** | 0.9892 → **0.9262** | 7 → **5** | ZEC → **NONE** | 0 → 0 | +| `pullback_continuation` | 60 → **124** | 4 → **14** | 0.9292 → **0.7736** | 1 → **1** | 0 → 0 | 0 → 0 | +| `rsi_meanrev` | 38 → **42** | 0 → **0** | 1.1631 → **1.1251** | 0 → 0 | 0 → 0 | 0 → 0 | +| Arm B (OOS) | 84 → **92** | 1 → **2** | 1.2311 → **1.2015** | 0 → **1** | 0 → 0 | 0 → 0 | + +**Every cell clearing `n≥100 ∧ gross>1`, and what it does once costs are charged:** + +``` +turtle ZEC-USD n=268 gross 1.442 maker 0.968 taker 0.685 +turtle XRP-USD n=157 gross 1.223 maker 0.688 taker 0.438 +turtle FET-USD n=269 gross 1.206 maker 0.825 taker 0.599 +turtle PAXG-USDT n=238 gross 1.145 maker 0.194 taker 0.051 +turtle CRV-USD n=260 gross 1.017 maker 0.648 taker 0.445 +pullback ZEC-USD n=170 gross 1.044 maker 0.344 taker 0.144 +Arm B PAXG-USDT n=101 gross 1.987 maker 0.484 taker 0.154 +``` + +Seven cells of ninety are gross-positive at the floor. **All seven die at the maker rate**, before +the taker rate we actually pay is even reached. + +The only cells anywhere above 1.0 at taker are WLD-USD (n=58, taker 1.061) and TON-USD (n=31, +taker 0.774 — below), both far under the trade floor and both on histories starting in 2025. + +### 3.1 `pullback_continuation`, and a distinction worth protecting + +Its numbers moved most: median n **58 → 124**, assets clearing the floor **1 → 14**, median gross +**0.9219 → 0.7736**. The mechanism is legible — `entry = signal_candle.high + buffer_ticks` is a +confirmation condition, and a market fill removes it, so the rule now takes the trades it was +designed to decline. The doubling *is* the count of those trades. + +**It does not follow that the offset entry was generating alpha.** Both sides of that comparison +lose money gross: 0.9219 with the filter, 0.7736 without. The filter separated **bad from worse**, +not good from toxic. There is no profitable subset of `pullback_continuation` that the entry +condition was protecting, and this dataset must not be cited as evidence that offset entries add +edge. + +What it *is* evidence for is #260: production silently overrides a rule's stated entry logic. The +landmine is not this rule — it is the next one that expresses a condition through its entry price. + +### 3.2 Arm B, restated on a single engine + +#252's headline transfer number compared figures produced by one engine. Recomputed with both +sides on #258: + +``` +in-sample mean net PF@1.2%, on the 6 selection assets : 0.5770 +out-of-sample mean, on the 18 disjoint assets : 0.5427 +gap : 0.0343 +``` + +#252 reported 0.6335 vs 0.6346 — a three-decimal agreement that was partly luck. The honest figure +is a gap of **0.034**, still a clean transfer, and the conclusion is unchanged and now +engine-consistent: **the sweep winner is not overfit; it is stably unprofitable**, which is the +harder result of the two. + +## 4. #255 restated + +| | old engine | faithful engine | +|---|---|---| +| `oversold=20` (the anchor) | 1.1631 at median n=38 | **1.1251** at median n=42 | +| every cell with `n≥100` | 0.8938 across 76 cells | **0.8396** across 82 cells | +| gross>1 at the floor | 11/76 | **6/82** | +| net>1 at maker | 0/76 | **0/82** | +| trigger (assets reaching `n≥100` at `oversold=40`) | 21/24 | **22/24** — did not fire | +| slope per +100 trades | −0.0386, 15/24 negative | **−0.0328, 15/24 negative** | + +**The level shift widens.** 1.1251 → 0.8396 against the previous 1.1631 → 0.8938, and the count of +gross-positive cells at the floor nearly halves. The conclusion is unchanged and stronger: +`rsi_meanrev`'s apparent edge is a property of firing rarely, and it does not survive being made +measurable. The rule is unpromotable by construction. + +### 4.1 The monotonicity anomaly is structurally gone + +#255 §5 recorded that `oversold` did not reliably increase firing rate — three of 24 assets went +backwards, UNI-USD by 34×. That was the symptom that led to #256 and then #258. Re-measured: + +``` +OLD engine : 3 assets non-monotonic AAVE 1.1x, BTC 2.3x, UNI 34.3x +#258 engine: 0 assets non-monotonic +``` + +Trade count is now monotonic in `oversold` across all 24 assets. This is an independent check on +the fixes rather than a restatement: the anomaly that exposed the defects is absent, not merely +smaller. + +## 5. The operational takeaway: the engine needs observable invariants, not more tests + +**Neither defect was found by looking for defects, and neither was findable by the means we had.** + +- The full suite — **2,712 tests** — passed throughout, before and after both fixes. +- Every experiment in `docs/experiments/` ran on the defective engine and produced plausible, + internally consistent output. +- #256 surfaced only from a 34× non-monotonicity, which was visible only because a grid happened + to sweep one parameter across a wide enough range for the anomaly to be obvious. +- #258 surfaced only from asking why #256's fix *reduced* trade counts on some assets — a question + nobody had a reason to ask. + +The reason unit tests could not catch either is structural, and generalises: + +> **A frozen backtest and a highly selective strategy produce identical-looking output.** So do a +> patient limit fill and a lucky one. Nothing about a summary ledger — n, win rate, profit factor — +> distinguishes "the rule declined to fire" from "the engine stopped asking", or "we got a good +> entry" from "we skipped every bad one". + +Adding tests does not fix this, because a test asserts a behaviour someone already imagined. The +fix is **invariants the engine reports about itself**, so that a defect announces itself in ordinary +output rather than waiting to be inferred. Three, in the order they would have paid off: + +**1. Dead-tail warning.** Report when a backtest's last trade closes far before its candle corpus +ends. #256 would have announced itself immediately — `rsi_meanrev` on UNI-USD had a 4.7-year dead +tail sitting in plain sight of every run. + +**2. Intent-divergence logging (#260).** Have the live executor log when `intent.entry` differs +materially from the price at routing time. The order row already records both numbers side by side +and nothing compares them; #258's entire finding was one comparison away from being visible. + +**3. Cost anchored to output (#247, already shipped).** `rules backtest` now prints +`fee_pct=1.2000% (taker, from config fees.taker_pct)` beside every result. This is the model for +the other two, and its own commit put it best: *prior numbers were unfalsifiable by their readers*. +A profit factor without its fee rate cannot be checked; neither can one without its slippage +assumption (#259) or its fill model. + +The common property is that each makes a class of defect **visible in the normal output of a normal +run**, to a reader who is not hunting for it. That is the cheap half of correctness, and this +project has been paying for its absence. + +## 6. Status of the two research branches + +**Both are closed, and the closure is now engine-consistent.** Every signal rule the codebase ships +has been measured at its shipped defaults across 24 assets, and `rsi_meanrev` additionally along +its own frequency axis, on an engine matching production in both detection and fills. There is no +asset-rule-parameter combination that is simultaneously measurable (`n≥100`), gross-positive, and +net-positive at any fee reachable from this venue. + +**Open, in priority order, none scheduled here:** + +1. **#260** — the executor discards conditional entry prices. Not urgent for any current rule; a + landmine for any future price-conditional one. The cheap mitigation is invariant 2 above. +2. **#259** — one global 5bp slippage constant applied from BTC to TON. Matters when a thin asset + becomes a candidate, which is exactly when someone will want to trust the number. +3. **The PBO/CSCV gate is deployed and unfed.** #247 wired `g4_pbo_gate` into `can_promote` where + `pbo=None` blocks. Nothing supplies it a trial matrix. Worth noting that Arm B (§3.2) is + informal evidence overfitting has *not* been our binding constraint — a config selected on six + assets reproduced within 0.034 on eighteen it never saw. The gate is a guard for a future + candidate, not an instrument for finding one. diff --git a/docs/experiments/trials-ledger.jsonl b/docs/experiments/trials-ledger.jsonl index e349b3f9..abcc97fe 100644 --- a/docs/experiments/trials-ledger.jsonl +++ b/docs/experiments/trials-ledger.jsonl @@ -81,3 +81,5 @@ {"decision":"diagnostic_only","kind":"ablation","params":{"arm":"A -- shipped constructor defaults, ZERO free parameters","assets":24,"construction":"each rule built with product_id= and nothing else","criteria_declared_before_run":"C1 n>=100; C2 C1 and gross PF>1; C3 C2 and PF@0.6%>1","dca_excluded":"scheduled accumulation, not a signal edge","document":"docs/experiments/2026-08-12-shipped-defaults-intersection.md","fees":["0","0.006 maker","0.012 taker"],"granularity":"ONE_HOUR","predeclaration_location":"the dispatch brief, NOT the script docstring -- weaker than the other harnesses in this directory, and recorded as such in the script and the write-up","rules":["turtle_breakout","rsi_meanrev","pullback_continuation"],"slippage_pct":"0.0005 pinned explicitly, not inherited","trials":"72 combinations x 3 fee levels = 216 backtests"},"per_bar_pnl":[],"per_trade_pnl":[],"prev_hash":"706422899e8fb759a4732e39aa63e79c3ebcc3371622e4f0ad9193e08ab3740e","provenance":"a_priori","row_hash":"74cfa02fa95ee5d5cafe5ee6dda36ea69c3755317722233d6d2745b390243651","rule":"turtle_breakout+rsi_meanrev+pullback_continuation","series_missing":true,"session":"shipped-defaults-intersection-2026-08-12","summary":{"pullback_gross_pos":"1","pullback_maker_pos":"0","pullback_median_gross":"0.9292","pullback_n100":"4","rsi_median_gross":"1.1631","rsi_n100":"0","taker_survivors_all_rules":"0","trade_count":25,"turtle_gross_pos":"7","turtle_maker_pos":"1","turtle_median_gross":"0.9892","turtle_n100":"21"},"timestamp":1786000000,"trial_id":"shipped-defaults-intersection-arm-a-2026-08-12"} {"decision":"diagnostic_only","kind":"ablation","params":{"adx_threshold":25.0,"arm":"B -- out-of-sample transfer of the 864-trial sweep winner","atr_stop_mult":"2","consumes_no_new_selection":"the config was fixed before this arm ran; this arm spends no further multiple-testing budget, it only tests generalisation","document":"docs/experiments/2026-08-12-shipped-defaults-intersection.md","entry_lookback":336,"evaluated_on":"the 18 assets NOT in that six -- disjoint from the selection","exit_lookback":80,"selected_on":["ZEC-USD","FET-USD","SOL-USD","DOGE-USD","ETH-USD","BTC-USD"],"target_rr":"6","trials":"18 combinations x 3 fee levels = 54 backtests"},"per_bar_pnl":[],"per_trade_pnl":[],"prev_hash":"74cfa02fa95ee5d5cafe5ee6dda36ea69c3755317722233d6d2745b390243651","provenance":"fitted","row_hash":"19ecedc7148cb12c8a33d59f5e9cbbba5145beb1ddabd7cbffa1e63dd755ba6c","rule":"turtle_breakout","series_missing":true,"session":"shipped-defaults-intersection-2026-08-12","summary":{"in_sample_mean_net_pf":"0.6335","oos_assets_over_floor":"1","oos_mean_gross_pf":"1.5317","oos_mean_net_pf":"0.6346","oos_median_n":"84","oos_median_net_pf":"0.5138","trade_count":18,"verdict_not_overfit":"1"},"timestamp":1786000001,"trial_id":"shipped-defaults-intersection-arm-b-2026-08-12"} {"decision":"rejected","kind":"ablation","params":{"anchor":"oversold=20 reused from the #252 intersection run (identical in every other parameter and cost treatment)","assets":24,"caveat_declared_statistic":"the slope was UNDERPOWERED for its own question -- the relationship is a threshold effect at the floor, not linear, and two barely-trading assets supply most of its variance. The partition on the pre-declared n>=100 floor is the decisive reading and is reported alongside it.","conditional_arm":"declared in advance: widen support_proximity_pct if FEWER THAN 8 of 24 assets reach n>=100 at oversold=40. DID NOT FIRE (21/24 reached it).","design":"MONOTONICITY TEST, not a search -- declared statistic is the SLOPE of gross PF against n, per asset then averaged. The best cell is never reported as a result.","document":"docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.md","fees":["0","0.006 maker","0.012 taker"],"granularity":"ONE_HOUR","held_fixed":{"overbought":80.0,"support_proximity_pct":"0.005","why_overbought_held":"it governs the EXIT side, so moving it changes trade outcomes and not merely trade counts, which would make the slope uninterpretable"},"instrument_defect_found":"backtest() never expires a pending setup, silently freezing a strategy for the rest of a series -- filed as #254. #252's headline results were checked and are clean.","predeclaration_location":"the script docstring, written before the run -- the correction to the defect #252 recorded against itself","slippage_pct":"0.0005 pinned explicitly","trials":"4 new levels x 24 assets = 96 combinations x 3 fees = 288 backtests","variable_axis":{"oversold":[20.0,25.0,30.0,35.0,40.0]}},"per_bar_pnl":[],"per_trade_pnl":[],"prev_hash":"19ecedc7148cb12c8a33d59f5e9cbbba5145beb1ddabd7cbffa1e63dd755ba6c","provenance":"a_priori","row_hash":"a16af414291b6034ccaa58ffa1577e56aa42eab1c72edb9364ac77988acd6b83","rule":"rsi_meanrev","series_missing":true,"session":"rsi-meanrev-scale-vs-selectivity-2026-08-12","summary":{"assets_negative_slope":"15","assets_reaching_floor_at_oversold_40":"21","cells_gross_positive_over_floor":"11","cells_net_positive_over_floor_any_fee":"0","cells_over_floor":"76","mean_slope_per_100_trades":"-0.0386","median_gross_pf_at_defaults":"1.1631","median_gross_pf_at_n_over_100":"0.8938","median_slope_per_100_trades":"-0.0197","trade_count":96},"timestamp":1786010000,"trial_id":"rsi-meanrev-scale-vs-selectivity-2026-08-12"} +{"decision":"diagnostic_only","kind":"ablation","params":{"bias_directions":"OPPOSITE -- #256 suppressed opportunity, #258 flattered execution. Correcting both moved everything one way: n rose in 87/90, gross PF fell in 69/90.","conclusion_changed":"ONE -- ZEC-turtle no longer clears the maker line (1.034 -> 0.968), so the three-probe narrative of the original describes a survivor the faithful engine does not produce","document":"docs/experiments/2026-08-13-restated-under-a-production-faithful-engine.md","engine_changes":"#256 pending setup never expired (detector froze); #258 entries filled at a price production never waits for (live places market orders)","identical_design":"same 90 combinations, same 3 fee levels, same explicit 0.0005 slippage pin; only the engine differs","restates":"docs/experiments/2026-08-12-shipped-defaults-intersection.md"},"per_bar_pnl":[],"per_trade_pnl":[],"prev_hash":"a16af414291b6034ccaa58ffa1577e56aa42eab1c72edb9364ac77988acd6b83","provenance":"a_priori","row_hash":"605ecff048e50fa9cb0d1b649e1eab9e254f1463dd17e91b622d81fd7ef1f31e","rule":"turtle_breakout+rsi_meanrev+pullback_continuation","series_missing":true,"session":"restated-production-faithful-engine-2026-08-13","summary":{"arm_b_in_sample_net_pf":"0.5770","arm_b_oos_net_pf":"0.5427","arm_b_transfer_gap":"0.0343","cells_gross_positive_over_floor":"7","cells_net_positive_at_maker":"0","cells_net_positive_at_taker":"0","pullback_median_gross":"0.7736","rsi_median_gross":"1.1251","trade_count":90,"turtle_median_gross":"0.9262"},"timestamp":1786100000,"trial_id":"intersection-restated-faithful-engine-2026-08-13"} +{"decision":"rejected","kind":"ablation","params":{"conditional_arm":"did not fire -- 22/24 assets reached n>=100 at oversold=40","design_unchanged":"oversold {20,25,30,35,40} x 24 assets, overbought and support_proximity_pct held; anchor re-derived on the same engine","document":"docs/experiments/2026-08-13-restated-under-a-production-faithful-engine.md","engine_changes":"#256 and #258, as above","monotonicity_check":"the anomaly that exposed both defects is structurally gone: 3 non-monotonic assets (UNI 34.3x, BTC 2.3x, AAVE 1.1x) -> 0","restates":"docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.md"},"per_bar_pnl":[],"per_trade_pnl":[],"prev_hash":"605ecff048e50fa9cb0d1b649e1eab9e254f1463dd17e91b622d81fd7ef1f31e","provenance":"a_priori","row_hash":"a782b48b637e5e9f83b8c0a25af352edc38dc9d21115f2dbdd4974745751b8b5","rule":"rsi_meanrev","series_missing":true,"session":"restated-production-faithful-engine-2026-08-13","summary":{"assets_reaching_floor_at_oversold_40":"22","cells_gross_positive_over_floor":"6","cells_net_positive_at_maker":"0","cells_over_floor":"82","mean_slope_per_100_trades":"-0.0328","median_gross_at_anchor":"1.1251","median_gross_over_floor":"0.8396","non_monotonic_assets":"0","trade_count":96},"timestamp":1786100001,"trial_id":"rsi-scale-restated-faithful-engine-2026-08-13"}