diff --git a/.gitignore b/.gitignore index 6a3a67a8..97c0407e 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,12 @@ config.local.yaml coverage.xml htmlcov/ +# Sweep output from the scripts in docs/experiments/. The resumable JSONL is what lets a killed +# multi-hour sweep pick up where it stopped, so it must persist between runs -- but it is bulk +# derived data regenerated from keel.db, and the .md beside each script is the committed finding. +# Previously these went to a per-session temp dir, which is why those runs stopped reproducing. +/docs/experiments/_out/ + # Generated by code-quality.yml from uv.lock so Snyk has a manifest it supports on its GA plan # (see the comment in that workflow). uv.lock is the source of truth; this is never committed, # because a stale copy beside a fresh lockfile would scan the wrong versions. diff --git a/docs/experiments/2026-08-11-round-number-scale.py b/docs/experiments/2026-08-11-round-number-scale.py index d8d86c7a..5199d2c4 100644 --- a/docs/experiments/2026-08-11-round-number-scale.py +++ b/docs/experiments/2026-08-11-round-number-scale.py @@ -208,7 +208,8 @@ def print_presence( rates = [_rate(s.vectors[FACTOR]) for s in after.values()] old_rates = [_rate(s.vectors[FACTOR]) for s in before.values()] print( - f"\n cross-asset spread (max/min): before {_fmt(max(old_rates) / min(old_rates), '0.01')}x" + f"\n cross-asset spread (max/min): before " + f"{_fmt(max(old_rates) / min(old_rates), '0.01')}x" f" after {_fmt(max(rates) / min(rates), '0.01')}x" ) print( @@ -291,14 +292,18 @@ def print_thresholds( now = [entry_technique(t, CTS_LOW, CTS_HIGH) for t in after[product_id].totals] moved = sum(1 for a, b in zip(was, now) if a != b) n = len(now) - print(f"{product_id:10} {'before':>7} " + " ".join(f"{was.count(t):14d}" for t in techniques)) + print( + f"{product_id:10} {'before':>7} " + + " ".join(f"{was.count(t):14d}" for t in techniques) + ) print( f"{'':10} {'after':>7} " + " ".join(f"{now.count(t):14d}" for t in techniques) + f" {moved:7d}" ) print( f"{'':10} {'':>7} " + " ".join( - f"{(Decimal(now.count(t) - was.count(t)) / Decimal(n)):>+13.4f} " for t in techniques + f"{(Decimal(now.count(t) - was.count(t)) / Decimal(n)):>+13.4f} " + for t in techniques ) + f" {Decimal(moved) / Decimal(n):>6.4f}" ) diff --git a/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.py b/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.py index b62a2723..1e6f9307 100644 --- a/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.py +++ b/docs/experiments/2026-08-12-rsi-meanrev-scale-vs-selectivity.py @@ -102,12 +102,16 @@ import time from concurrent.futures import ProcessPoolExecutor from decimal import Decimal - -DB = "/Users/elmehdiaitbrahim/keel/keel.db" -SCRATCH = ( - "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/" - "28ff9a61-09d1-498b-b325-1631c0662734/scratchpad" -) +from pathlib import Path + +# Resolved rather than hardcoded: these were absolute literals into one laptop's home and into a +# per-session temp directory that is collected when the session ends -- so this recorded run could +# not be re-run, which is the one thing a recorded run is for. `_out/` sits beside this script +# (gitignored), which is also where the intersection run writes the ANCHOR this reads. +# Override either with KEEL_EXPERIMENT_DB / KEEL_EXPERIMENT_OUT. +DB = os.environ.get("KEEL_EXPERIMENT_DB") or str(Path.home() / "keel" / "keel.db") +SCRATCH = os.environ.get("KEEL_EXPERIMENT_OUT") or str(Path(__file__).resolve().parent / "_out") +Path(SCRATCH).mkdir(parents=True, exist_ok=True) ANCHOR = f"{SCRATCH}/intersection.jsonl" # supplies the oversold=20 rows OUT_PRIMARY = f"{SCRATCH}/rsi_scale.jsonl" OUT_CONDITIONAL = f"{SCRATCH}/rsi_scale_proximity.jsonl" # separate file, never merged @@ -132,11 +136,12 @@ def _run(job: tuple[str, float, str]) -> list[dict]: product, oversold, proximity = job + from keel_core.types import Granularity + from keel.data.db import connect from keel.data.repository import Repository from keel.strategy import backtest as bt from keel.strategy.rules.rsi_meanrev import RsiMeanReversion - from keel_core.types import Granularity out: list[dict] = [] try: diff --git a/docs/experiments/2026-08-12-shipped-defaults-intersection.py b/docs/experiments/2026-08-12-shipped-defaults-intersection.py index f40ee2e1..b7028f27 100644 --- a/docs/experiments/2026-08-12-shipped-defaults-intersection.py +++ b/docs/experiments/2026-08-12-shipped-defaults-intersection.py @@ -53,12 +53,20 @@ """ import json +import os import time from concurrent.futures import ProcessPoolExecutor, as_completed from decimal import Decimal - -DB = "/Users/elmehdiaitbrahim/keel/keel.db" -OUT_DIR = "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/28ff9a61-09d1-498b-b325-1631c0662734/scratchpad" +from pathlib import Path + +# Resolved rather than hardcoded: these were absolute literals into one laptop's home and into a +# per-session temp directory that is collected when the session ends -- so this recorded run could +# not be re-run, which is the one thing a recorded run is for. `_out/` sits beside this script +# (gitignored) so the resumable JSONL survives and the rsi-scale scripts find it as their ANCHOR. +# Override either with KEEL_EXPERIMENT_DB / KEEL_EXPERIMENT_OUT. +DB = os.environ.get("KEEL_EXPERIMENT_DB") or str(Path.home() / "keel" / "keel.db") +OUT_DIR = os.environ.get("KEEL_EXPERIMENT_OUT") or str(Path(__file__).resolve().parent / "_out") +Path(OUT_DIR).mkdir(parents=True, exist_ok=True) JSONL_PATH = f"{OUT_DIR}/intersection.jsonl" JSON_PATH = f"{OUT_DIR}/intersection.json" @@ -89,9 +97,9 @@ def build_jobs(): def make_rule(arm, rule, asset): - from keel.strategy.rules.turtle_breakout import TurtleBreakout - from keel.strategy.rules.rsi_meanrev import RsiMeanReversion from keel.strategy.rules.pullback_continuation import PullbackContinuation + from keel.strategy.rules.rsi_meanrev import RsiMeanReversion + from keel.strategy.rules.turtle_breakout import TurtleBreakout if arm == "A": if rule == "turtle": @@ -119,10 +127,11 @@ def make_rule(arm, rule, asset): def run_job(job): arm, rule, asset = job + from keel_core.types import Granularity + from keel.data.db import connect from keel.data.repository import Repository from keel.strategy import backtest as bt - from keel_core.types import Granularity rows = [] try: @@ -197,7 +206,10 @@ def main(): prio = {("B", "turtle"): 0, ("A", "pullback"): 1, ("A", "rsi"): 2} jobs.sort(key=lambda j: prio.get((j[0], j[1]), 3)) total = len(jobs) - print(f"Declared jobs: {total_declared}; already done: {len(done)}; running: {total}", flush=True) + print( + f"Declared jobs: {total_declared}; already done: {len(done)}; running: {total}", + flush=True, + ) start = time.time() completed = 0 diff --git a/docs/experiments/2026-08-13-restated-intersection.py b/docs/experiments/2026-08-13-restated-intersection.py index b07cf2a7..13931ee3 100644 --- a/docs/experiments/2026-08-13-restated-intersection.py +++ b/docs/experiments/2026-08-13-restated-intersection.py @@ -1,10 +1,18 @@ import json +import os import time from concurrent.futures import ProcessPoolExecutor, as_completed from decimal import Decimal - -DB = "/Users/elmehdiaitbrahim/keel/keel.db" -OUT_DIR = "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/28ff9a61-09d1-498b-b325-1631c0662734/scratchpad" +from pathlib import Path + +# Resolved rather than hardcoded: these were absolute literals into one laptop's home and into a +# per-session temp directory that is collected when the session ends -- so this recorded run could +# not be re-run, which is the one thing a recorded run is for. `_out/` sits beside this script +# (gitignored) so the resumable JSONL survives and the rsi-scale scripts find it as their ANCHOR. +# Override either with KEEL_EXPERIMENT_DB / KEEL_EXPERIMENT_OUT. +DB = os.environ.get("KEEL_EXPERIMENT_DB") or str(Path.home() / "keel" / "keel.db") +OUT_DIR = os.environ.get("KEEL_EXPERIMENT_OUT") or str(Path(__file__).resolve().parent / "_out") +Path(OUT_DIR).mkdir(parents=True, exist_ok=True) JSONL_PATH = f"{OUT_DIR}/intersection_257.jsonl" JSON_PATH = f"{OUT_DIR}/intersection_257.json" @@ -35,9 +43,9 @@ def build_jobs(): def make_rule(arm, rule, asset): - from keel.strategy.rules.turtle_breakout import TurtleBreakout - from keel.strategy.rules.rsi_meanrev import RsiMeanReversion from keel.strategy.rules.pullback_continuation import PullbackContinuation + from keel.strategy.rules.rsi_meanrev import RsiMeanReversion + from keel.strategy.rules.turtle_breakout import TurtleBreakout if arm == "A": if rule == "turtle": @@ -65,10 +73,11 @@ def make_rule(arm, rule, asset): def run_job(job): arm, rule, asset = job + from keel_core.types import Granularity + from keel.data.db import connect from keel.data.repository import Repository from keel.strategy import backtest as bt - from keel_core.types import Granularity rows = [] try: @@ -143,7 +152,10 @@ def main(): prio = {("B", "turtle"): 0, ("A", "pullback"): 1, ("A", "rsi"): 2} jobs.sort(key=lambda j: prio.get((j[0], j[1]), 3)) total = len(jobs) - print(f"Declared jobs: {total_declared}; already done: {len(done)}; running: {total}", flush=True) + print( + f"Declared jobs: {total_declared}; already done: {len(done)}; running: {total}", + flush=True, + ) start = time.time() completed = 0 diff --git a/docs/experiments/2026-08-13-restated-rsi-scale.py b/docs/experiments/2026-08-13-restated-rsi-scale.py index a3118d5d..2d998f61 100644 --- a/docs/experiments/2026-08-13-restated-rsi-scale.py +++ b/docs/experiments/2026-08-13-restated-rsi-scale.py @@ -102,12 +102,16 @@ import time from concurrent.futures import ProcessPoolExecutor from decimal import Decimal - -DB = "/Users/elmehdiaitbrahim/keel/keel.db" -SCRATCH = ( - "/private/tmp/claude-501/-Users-elmehdiaitbrahim-Development-work-CodeGate-keel/" - "28ff9a61-09d1-498b-b325-1631c0662734/scratchpad" -) +from pathlib import Path + +# Resolved rather than hardcoded: these were absolute literals into one laptop's home and into a +# per-session temp directory that is collected when the session ends -- so this recorded run could +# not be re-run, which is the one thing a recorded run is for. `_out/` sits beside this script +# (gitignored), which is also where the intersection run writes the ANCHOR this reads. +# Override either with KEEL_EXPERIMENT_DB / KEEL_EXPERIMENT_OUT. +DB = os.environ.get("KEEL_EXPERIMENT_DB") or str(Path.home() / "keel" / "keel.db") +SCRATCH = os.environ.get("KEEL_EXPERIMENT_OUT") or str(Path(__file__).resolve().parent / "_out") +Path(SCRATCH).mkdir(parents=True, exist_ok=True) ANCHOR = f"{SCRATCH}/intersection_257.jsonl" # supplies the oversold=20 rows OUT_PRIMARY = f"{SCRATCH}/rsi_scale_257.jsonl" OUT_CONDITIONAL = f"{SCRATCH}/rsi_scale_257_proximity.jsonl" # separate file, never merged @@ -132,11 +136,12 @@ def _run(job: tuple[str, float, str]) -> list[dict]: product, oversold, proximity = job + from keel_core.types import Granularity + from keel.data.db import connect from keel.data.repository import Repository from keel.strategy import backtest as bt from keel.strategy.rules.rsi_meanrev import RsiMeanReversion - from keel_core.types import Granularity out: list[dict] = [] try: