From 12f5264258926b6f72b0ca5deb104b25088e34b4 Mon Sep 17 00:00:00 2001
From: Elmehdi Aitbrahim <eaitbrahim@gmail.com>
Date: Thu, 13 Aug 2026 02:21:51 -0400
Subject: [PATCH] fix(experiments): resolve the sweep harnesses' paths, so a
 recorded run can be re-run

Four harnesses hardcoded two absolute paths: the deployment DB under one laptop's
home, and an output directory inside a per-session temp dir that is collected when
that session ends. A recorded experiment that cannot be re-run is the one thing a
recorded experiment must not be, and the output path had already gone stale.

Both now resolve, and both are overridable:

    DB      = $KEEL_EXPERIMENT_DB  or ~/keel/keel.db
    OUT_DIR = $KEEL_EXPERIMENT_OUT or <script dir>/_out

The default DB resolves to the same physical file it named before, so nothing
changes on the machine these were written on -- it just stops being the only
machine where they work.

`_out/` is gitignored but deliberately NOT temporary: these sweeps are resumable,
reading their own append-only JSONL back to skip completed cells, so that file has
to outlive the run. A per-session scratch dir is precisely what broke that, and it
is also the shared handoff point -- each `*-rsi-scale` run reads the JSONL its
matching `*-intersection` run writes as the ANCHOR supplying its oversold=20 rows.
Both chains were checked by name, and the two studies were checked not to collide
(`intersection.jsonl` vs `intersection_257.jsonl`).

Only two of the four were visible to ruff. The other two spelled the dead path as
a multi-line `SCRATCH` constant, which is under the line limit and so was flagged
by nothing; they were found by grepping for the literal rather than by trusting the
linter's list.

Also clears the 13 remaining ruff findings in `docs/` -- 6 import blocks (autofix)
and 5 long lines rewrapped without changing any output, the other 2 being the path
literals above. `ruff check .` now passes repo-wide; previously only the CI-scoped
`ruff check keel tests packages` did.

Verified: candles load through the resolved DB path (44,407 bars for BTC-USD, the
~44k those docstrings cite); defaults and both env overrides assert-checked,
including that the output dir is created; `_out/` confirmed ignored; mypy clean;
pytest 2717 passed, 1 skipped.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .gitignore                                    |  6 +++++
 .../2026-08-11-round-number-scale.py          | 11 +++++---
 ...-08-12-rsi-meanrev-scale-vs-selectivity.py | 19 +++++++++-----
 ...026-08-12-shipped-defaults-intersection.py | 26 ++++++++++++++-----
 .../2026-08-13-restated-intersection.py       | 26 ++++++++++++++-----
 .../2026-08-13-restated-rsi-scale.py          | 19 +++++++++-----
 6 files changed, 76 insertions(+), 31 deletions(-)

diff --git a/.gitignore b/.gitignore
index 6a3a67a8..97c0407e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,6 +49,12 @@ config.local.yaml
 coverage.xml
 htmlcov/
 
+# Sweep output from the scripts in docs/experiments/. The resumable JSONL is what lets a killed
+# multi-hour sweep pick up where it stopped, so it must persist between runs -- but it is bulk
+# derived data regenerated from keel.db, and the .md beside each script is the committed finding.
+# Previously these went to a per-session temp dir, which is why those runs stopped reproducing.
+/docs/experiments/_out/
+
 # Generated by code-quality.yml from uv.lock so Snyk has a manifest it supports on its GA plan
 # (see the comment in that workflow). uv.lock is the source of truth; this is never committed,
 # because a stale copy beside a fresh lockfile would scan the wrong versions.
diff --git a/docs/experiments/2026-08-11-round-number-scale.py b/docs/experiments/2026-08-11-round-number-scale.py
index d8d86c7a..5199d2c4 100644
--- a/docs/experiments/2026-08-11-round-number-scale.py
+++ b/docs/experiments/2026-08-11-round-number-scale.py
@@ -208,7 +208,8 @@ def print_presence(
     rates = [_rate(s.vectors[FACTOR]) for s in after.values()]
     old_rates = [_rate(s.vectors[FACTOR]) for s in before.values()]
     print(
-        f"\n  cross-asset spread (max/min):  before {_fmt(max(old_rates) / min(old_rates), '0.01')}x"
+        f"\n  cross-asset spread (max/min):  before "
+        f"{_fmt(max(old_rates) / min(old_rates), '0.01')}x"
         f"   after {_fmt(max(rates) / min(rates), '0.01')}x"
     )
     print(
@@ -291,14 +292,18 @@ def print_thresholds(
         now = [entry_technique(t, CTS_LOW, CTS_HIGH) for t in after[product_id].totals]
         moved = sum(1 for a, b in zip(was, now) if a != b)
         n = len(now)
-        print(f"{product_id:10} {'before':>7} " + " ".join(f"{was.count(t):14d}" for t in techniques))
+        print(
+            f"{product_id:10} {'before':>7} "
+            + " ".join(f"{was.count(t):14d}" for t in techniques)
+        )
         print(
             f"{'':10} {'after':>7} " + " ".join(f"{now.count(t):14d}" for t in techniques)
             + f" {moved:7d}"
         )
         print(
             f"{'':10} {'':>7} " + " ".join(
-                f"{(Decimal(now.count(t) - was.count(t)) / Decimal(n)):>+13.4f} " for t in techniques
+                f"{(Decimal(now.count(t) - was.count(t)) / Decimal(n)):>+13.4f} "
+                for t in techniques
             )
             + f" {Decimal(moved) / Decimal(n):>6.4f}"
         )
diff --git a/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.py b/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.py
index b62a2723..1e6f9307 100644
--- a/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.py
+++ b/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.py
@@ -102,12 +102,16 @@
 import time
 from concurrent.futures import ProcessPoolExecutor
 from decimal import Decimal
-
-DB = "/Users/elmehdiaitbrahim/keel/keel.db"
-SCRATCH = (
-    "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/"
-    "28ff9a61-09d1-498b-b325-1631c0662734/scratchpad"
-)
+from pathlib import Path
+
+# Resolved rather than hardcoded: these were absolute literals into one laptop's home and into a
+# per-session temp directory that is collected when the session ends -- so this recorded run could
+# not be re-run, which is the one thing a recorded run is for. `_out/` sits beside this script
+# (gitignored), which is also where the intersection run writes the ANCHOR this reads.
+# Override either with KEEL_EXPERIMENT_DB / KEEL_EXPERIMENT_OUT.
+DB = os.environ.get("KEEL_EXPERIMENT_DB") or str(Path.home() / "keel" / "keel.db")
+SCRATCH = os.environ.get("KEEL_EXPERIMENT_OUT") or str(Path(__file__).resolve().parent / "_out")
+Path(SCRATCH).mkdir(parents=True, exist_ok=True)
 ANCHOR = f"{SCRATCH}/intersection.jsonl"  # supplies the oversold=20 rows
 OUT_PRIMARY = f"{SCRATCH}/rsi_scale.jsonl"
 OUT_CONDITIONAL = f"{SCRATCH}/rsi_scale_proximity.jsonl"  # separate file, never merged
@@ -132,11 +136,12 @@
 
 def _run(job: tuple[str, float, str]) -> list[dict]:
     product, oversold, proximity = job
+    from keel_core.types import Granularity
+
     from keel.data.db import connect
     from keel.data.repository import Repository
     from keel.strategy import backtest as bt
     from keel.strategy.rules.rsi_meanrev import RsiMeanReversion
-    from keel_core.types import Granularity
 
     out: list[dict] = []
     try:
diff --git a/docs/experiments/2026-08-12-shipped-defaults-intersection.py b/docs/experiments/2026-08-12-shipped-defaults-intersection.py
index f40ee2e1..b7028f27 100644
--- a/docs/experiments/2026-08-12-shipped-defaults-intersection.py
+++ b/docs/experiments/2026-08-12-shipped-defaults-intersection.py
@@ -53,12 +53,20 @@
 """
 
 import json
+import os
 import time
 from concurrent.futures import ProcessPoolExecutor, as_completed
 from decimal import Decimal
-
-DB = "/Users/elmehdiaitbrahim/keel/keel.db"
-OUT_DIR = "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/28ff9a61-09d1-498b-b325-1631c0662734/scratchpad"
+from pathlib import Path
+
+# Resolved rather than hardcoded: these were absolute literals into one laptop's home and into a
+# per-session temp directory that is collected when the session ends -- so this recorded run could
+# not be re-run, which is the one thing a recorded run is for. `_out/` sits beside this script
+# (gitignored) so the resumable JSONL survives and the rsi-scale scripts find it as their ANCHOR.
+# Override either with KEEL_EXPERIMENT_DB / KEEL_EXPERIMENT_OUT.
+DB = os.environ.get("KEEL_EXPERIMENT_DB") or str(Path.home() / "keel" / "keel.db")
+OUT_DIR = os.environ.get("KEEL_EXPERIMENT_OUT") or str(Path(__file__).resolve().parent / "_out")
+Path(OUT_DIR).mkdir(parents=True, exist_ok=True)
 JSONL_PATH = f"{OUT_DIR}/intersection.jsonl"
 JSON_PATH = f"{OUT_DIR}/intersection.json"
 
@@ -89,9 +97,9 @@ def build_jobs():
 
 
 def make_rule(arm, rule, asset):
-    from keel.strategy.rules.turtle_breakout import TurtleBreakout
-    from keel.strategy.rules.rsi_meanrev import RsiMeanReversion
     from keel.strategy.rules.pullback_continuation import PullbackContinuation
+    from keel.strategy.rules.rsi_meanrev import RsiMeanReversion
+    from keel.strategy.rules.turtle_breakout import TurtleBreakout
 
     if arm == "A":
         if rule == "turtle":
@@ -119,10 +127,11 @@ def make_rule(arm, rule, asset):
 
 def run_job(job):
     arm, rule, asset = job
+    from keel_core.types import Granularity
+
     from keel.data.db import connect
     from keel.data.repository import Repository
     from keel.strategy import backtest as bt
-    from keel_core.types import Granularity
 
     rows = []
     try:
@@ -197,7 +206,10 @@ def main():
     prio = {("B", "turtle"): 0, ("A", "pullback"): 1, ("A", "rsi"): 2}
     jobs.sort(key=lambda j: prio.get((j[0], j[1]), 3))
     total = len(jobs)
-    print(f"Declared jobs: {total_declared}; already done: {len(done)}; running: {total}", flush=True)
+    print(
+        f"Declared jobs: {total_declared}; already done: {len(done)}; running: {total}",
+        flush=True,
+    )
 
     start = time.time()
     completed = 0
diff --git a/docs/experiments/2026-08-13-restated-intersection.py b/docs/experiments/2026-08-13-restated-intersection.py
index b07cf2a7..13931ee3 100644
--- a/docs/experiments/2026-08-13-restated-intersection.py
+++ b/docs/experiments/2026-08-13-restated-intersection.py
@@ -1,10 +1,18 @@
 import json
+import os
 import time
 from concurrent.futures import ProcessPoolExecutor, as_completed
 from decimal import Decimal
-
-DB = "/Users/elmehdiaitbrahim/keel/keel.db"
-OUT_DIR = "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/28ff9a61-09d1-498b-b325-1631c0662734/scratchpad"
+from pathlib import Path
+
+# Resolved rather than hardcoded: these were absolute literals into one laptop's home and into a
+# per-session temp directory that is collected when the session ends -- so this recorded run could
+# not be re-run, which is the one thing a recorded run is for. `_out/` sits beside this script
+# (gitignored) so the resumable JSONL survives and the rsi-scale scripts find it as their ANCHOR.
+# Override either with KEEL_EXPERIMENT_DB / KEEL_EXPERIMENT_OUT.
+DB = os.environ.get("KEEL_EXPERIMENT_DB") or str(Path.home() / "keel" / "keel.db")
+OUT_DIR = os.environ.get("KEEL_EXPERIMENT_OUT") or str(Path(__file__).resolve().parent / "_out")
+Path(OUT_DIR).mkdir(parents=True, exist_ok=True)
 JSONL_PATH = f"{OUT_DIR}/intersection_257.jsonl"
 JSON_PATH = f"{OUT_DIR}/intersection_257.json"
 
@@ -35,9 +43,9 @@ def build_jobs():
 
 
 def make_rule(arm, rule, asset):
-    from keel.strategy.rules.turtle_breakout import TurtleBreakout
-    from keel.strategy.rules.rsi_meanrev import RsiMeanReversion
     from keel.strategy.rules.pullback_continuation import PullbackContinuation
+    from keel.strategy.rules.rsi_meanrev import RsiMeanReversion
+    from keel.strategy.rules.turtle_breakout import TurtleBreakout
 
     if arm == "A":
         if rule == "turtle":
@@ -65,10 +73,11 @@ def make_rule(arm, rule, asset):
 
 def run_job(job):
     arm, rule, asset = job
+    from keel_core.types import Granularity
+
     from keel.data.db import connect
     from keel.data.repository import Repository
     from keel.strategy import backtest as bt
-    from keel_core.types import Granularity
 
     rows = []
     try:
@@ -143,7 +152,10 @@ def main():
     prio = {("B", "turtle"): 0, ("A", "pullback"): 1, ("A", "rsi"): 2}
     jobs.sort(key=lambda j: prio.get((j[0], j[1]), 3))
     total = len(jobs)
-    print(f"Declared jobs: {total_declared}; already done: {len(done)}; running: {total}", flush=True)
+    print(
+        f"Declared jobs: {total_declared}; already done: {len(done)}; running: {total}",
+        flush=True,
+    )
 
     start = time.time()
     completed = 0
diff --git a/docs/experiments/2026-08-13-restated-rsi-scale.py b/docs/experiments/2026-08-13-restated-rsi-scale.py
index a3118d5d..2d998f61 100644
--- a/docs/experiments/2026-08-13-restated-rsi-scale.py
+++ b/docs/experiments/2026-08-13-restated-rsi-scale.py
@@ -102,12 +102,16 @@
 import time
 from concurrent.futures import ProcessPoolExecutor
 from decimal import Decimal
-
-DB = "/Users/elmehdiaitbrahim/keel/keel.db"
-SCRATCH = (
-    "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/"
-    "28ff9a61-09d1-498b-b325-1631c0662734/scratchpad"
-)
+from pathlib import Path
+
+# Resolved rather than hardcoded: these were absolute literals into one laptop's home and into a
+# per-session temp directory that is collected when the session ends -- so this recorded run could
+# not be re-run, which is the one thing a recorded run is for. `_out/` sits beside this script
+# (gitignored), which is also where the intersection run writes the ANCHOR this reads.
+# Override either with KEEL_EXPERIMENT_DB / KEEL_EXPERIMENT_OUT.
+DB = os.environ.get("KEEL_EXPERIMENT_DB") or str(Path.home() / "keel" / "keel.db")
+SCRATCH = os.environ.get("KEEL_EXPERIMENT_OUT") or str(Path(__file__).resolve().parent / "_out")
+Path(SCRATCH).mkdir(parents=True, exist_ok=True)
 ANCHOR = f"{SCRATCH}/intersection_257.jsonl"  # supplies the oversold=20 rows
 OUT_PRIMARY = f"{SCRATCH}/rsi_scale_257.jsonl"
 OUT_CONDITIONAL = f"{SCRATCH}/rsi_scale_257_proximity.jsonl"  # separate file, never merged
@@ -132,11 +136,12 @@
 
 def _run(job: tuple[str, float, str]) -> list[dict]:
     product, oversold, proximity = job
+    from keel_core.types import Granularity
+
     from keel.data.db import connect
     from keel.data.repository import Repository
     from keel.strategy import backtest as bt
     from keel.strategy.rules.rsi_meanrev import RsiMeanReversion
-    from keel_core.types import Granularity
 
     out: list[dict] = []
     try: