diff --git a/.gitignore b/.gitignore index c080ad0d..9a0ff75f 100644 --- a/.gitignore +++ b/.gitignore @@ -41,15 +41,10 @@ keel/_build_info.py # operator's own working config (never the shipped template) config.local.yaml -# operational paper-forward config (funded rehearsal; never the shipped template) -config.paperforward.yaml -paperforward-run.sh -com.keel.paperforward.plist -# supervised-live sandbox (tiny-cap confirm-mode turtle detector; operational, never shipped) -config.live-sandbox.yaml -keel-live-run.sh -com.keel.live.plist - -# operator wrappers (config+db pinned) -- deployment artifacts, copied to ~/keel -keel-live -keel-paper + +# The paper-forward and supervised-live deployment files (configs, run scripts, launchd plists, +# operator wrappers) were ignored here until 2026-08-03, on the reasoning that they are +# operational and never shipped. Both are still true -- they are excluded from the wheel and from +# the release assets by the packaging config, not by this file -- but ignoring them meant the only +# copy of the running deployment's configuration lived on one laptop, with no history and no +# offsite copy. They are now tracked. They contain no credentials; those stay in `.env` (line 2). diff --git a/com.keel.live.plist b/com.keel.live.plist new file mode 100644 index 00000000..37d3ec56 --- /dev/null +++ b/com.keel.live.plist @@ -0,0 +1,57 @@ + + + + + Label + com.keel.live + + + ProgramArguments + + /bin/bash + /Users/elmehdiaitbrahim/keel/keel-live-run.sh + + + StartCalendarInterval + + Hour9Minute5 + Hour10Minute5 + Hour11Minute5 + Hour12Minute5 + Hour13Minute5 + Hour14Minute5 + Hour15Minute5 + Hour16Minute5 + Hour17Minute5 + Hour18Minute5 + Hour19Minute5 + Hour20Minute5 + + + + RunAtLoad + + + WorkingDirectory + /Users/elmehdiaitbrahim/keel + + StandardOutPath + /Users/elmehdiaitbrahim/keel/logs/keel-live.detector.out.log + StandardErrorPath + /Users/elmehdiaitbrahim/keel/logs/keel-live.detector.err.log + + diff --git a/com.keel.paperforward.plist b/com.keel.paperforward.plist new file mode 100644 index 00000000..1af7eb41 --- /dev/null +++ b/com.keel.paperforward.plist @@ -0,0 +1,54 @@ + + + + + Label + com.keel.paperforward + + + ProgramArguments + + /bin/bash + /Users/elmehdiaitbrahim/keel/paperforward-run.sh + + + StartCalendarInterval + + Hour9Minute0 + Hour10Minute0 + Hour11Minute0 + Hour12Minute0 + Hour13Minute0 + Hour14Minute0 + Hour15Minute0 + Hour16Minute0 + Hour17Minute0 + Hour18Minute0 + Hour19Minute0 + Hour20Minute0 + + + + RunAtLoad + + + WorkingDirectory + /Users/elmehdiaitbrahim/keel + + StandardOutPath + /Users/elmehdiaitbrahim/keel/logs/paperforward.out.log + StandardErrorPath + /Users/elmehdiaitbrahim/keel/logs/paperforward.err.log + + diff --git a/config.live-sandbox.yaml b/config.live-sandbox.yaml new file mode 100644 index 00000000..b843ad31 --- /dev/null +++ b/config.live-sandbox.yaml @@ -0,0 +1,111 @@ +# keel runtime configuration -- SUPERVISED-LIVE SANDBOX (confirm mode, tiny caps). +# +# Purpose: a small, human-approved live test of the 5-trend Turtle running IN PARALLEL with the +# paper-forward (which stays on config.paperforward.yaml + keel.db, untouched). Every order is +# PREVIEWED and requires a typed `y`. Caps are deliberately tiny so a bad fill can lose only a +# few dollars. +# +# Authored in the dev repo (gitignored). DEPLOYED to ~/keel at arm-time alongside its +# OWN database keel-live.db. There it runs as: +# .venv/bin/keel --config config.live-sandbox.yaml --db keel-live.db agent (interactive) + +# The 5-trend Turtle set (same as the paper-forward + the walk-forward/PBO analysis). +# SOL/LTC/LINK are deliberately excluded -- they're the unvalidated skip set. +allowlist: + - BTC + - ETH + - PAXG + - ADA + - XLM + +target_weights: + BTC: 0.30 + ETH: 0.20 + PAXG: 0.20 + ADA: 0.15 + XLM: 0.15 + +risk_pct: 0.01 # 1% of the (proxy) equity risked per trade + +caps: + # NOTE: these are POSITION/SIZE caps, NOT a monthly spend cap. The MONTHLY spend limit you + # asked for (start $200/mo, scale to your Coinbase Basic $500/mo) is Rail 14, stored in the DB + # (keel-live.db) via `keel subscription set --free-volume-usd 200`, NOT here. To scale it later: + # keel --config config.live-sandbox.yaml --db keel-live.db subscription set --free-volume-usd 500 + # + # max_exposure_usd = the TOTAL held-at-once ceiling AND the live sizing equity proxy, so it also + # bounds order size: at risk_pct 0.01 and a ~2xATR stop, a proposed order lands around $60-80. + # Think of it as "this sandbox is a $200 account." Raise it alongside the monthly cap as you scale. + max_exposure_usd: 200 # total open exposure across all positions AT ONCE (NOT per day, NOT per month) + max_per_order_usd: 100 # per-single-order ceiling + max_per_day_usd: 200 # per-rolling-24h ceiling + max_per_asset_pct: 0.50 # no single asset may exceed 50% of the $200 exposure ($100) + +market_data: + granularities: + - ONE_DAY + - ONE_HOUR + - FIFTEEN_MINUTE + history_days: 365 + +auto_trade: + mode: confirm # LIVE, but every order asks for a typed `y` first (fails closed with no human) + enabled: false + interval_sec: 900 + +promotion: + min_trades: 100 + min_expectancy: 0.0 + min_rr: 1.5 + min_win_rate: 0.55 + +money_mgmt: + profit_trigger_pct: 0.10 + acceleration_pct: 0.05 + max_total_dd_pct: 0.20 + max_weekly_dd_pct: 0.08 + max_consecutive_losses: 0 + streak_cooloff_days: 0 + +dca: + budget_usd: 50 + cadence_days: 7 + +# The settlement currency this deployment TRADES IN -- USD, matching the cached -USD history and +# the paper-forward. (each product spends USD, so the account needs a USD balance to fund a buy; +# rail 13 vetoes if there's no USD available -- see the note when you arm.) +quote_currency: USD + +subscription: + assumed_free_volume_usd: 500 + unsubscribed_allowance_usd: 0 + pacing: opportunistic + +tiers: + - name: Basic + free_volume_usd: 500 + subscription_usd_month: 4.99 + - name: Preferred + free_volume_usd: 10000 + subscription_usd_month: 29.99 + - name: Premium + free_volume_usd: null + subscription_usd_month: 299.99 + +fees: + taker_pct: 0.012 + maker_pct: 0.006 + +# verbose: true since 2026-07-29, matching config.paperforward.yaml -- the detector's `signals=N` +# line says nothing about why a setup was or wasn't taken, and on the live side the rail vetoes +# are exactly what you want on record. Separate file from the paper deployment, so the two +# never interleave. +logging: + verbose: true + file: logs/keel-live.log + max_file_mb: 25 + file_count: 5 + +research: + pbo_max: 0.05 + slope_floor: -0.5 diff --git a/config.paperforward.yaml b/config.paperforward.yaml new file mode 100644 index 00000000..efe599d4 --- /dev/null +++ b/config.paperforward.yaml @@ -0,0 +1,139 @@ +# keel runtime configuration. +# +# These are Phase 1 placeholders (spec §21 open item) — no live orders are placed in Phase 1. +# `allowlist` and `caps` are required and validated by keel.config.load_config; missing or +# invalid values raise ConfigError naming the offending key rather than silently defaulting. + +allowlist: + - BTC + - ETH + - PAXG + - SOL + - XLM + - LTC + - ADA + - LINK + +target_weights: + BTC: 0.30 + ETH: 0.20 + PAXG: 0.20 + SOL: 0.06 + XLM: 0.06 + LTC: 0.06 + ADA: 0.06 + LINK: 0.06 + +risk_pct: 0.01 + +caps: + # max_per_order_usd / max_per_day_usd are OPTIONAL internal RISK limits, not real Coinbase + # limits -- Coinbase One's only subscription constraint is monthly fee-free trading VOLUME + # (see `subscription:` below). They default to a non-binding $1B when omitted (Issue #85); + # set explicit values here only if you want an extra per-order/per-day risk ceiling tighter + # than the exposure/concentration caps below. Left at their non-binding default here so + # risk-sized rule orders ($400-24k typical) aren't silently rejected. + max_exposure_usd: 5000 + max_per_asset_pct: 0.50 + +market_data: + granularities: + - ONE_DAY + - ONE_HOUR + - FIFTEEN_MINUTE + history_days: 365 + +auto_trade: + mode: paper + # NOTE: currently UNUSED by any code path -- it is NOT a kill-switch and setting it + # true or false changes nothing. Use `keel kill` to halt trading. + enabled: false + interval_sec: 900 + +promotion: + min_trades: 100 + min_expectancy: 0.0 + min_rr: 1.5 + min_win_rate: 0.55 + +money_mgmt: + profit_trigger_pct: 0.10 + acceleration_pct: 0.05 + max_total_dd_pct: 0.20 + max_weekly_dd_pct: 0.08 + # Rail 16 (consecutive-loss breaker) — DISABLED by default (0 = off). + # Set from a backtest sweep, and set it ABOVE the strategy's tested max losing streak: + # turtle_breakout's max streak is 5, so a threshold of 3 would fire on normal variance. + max_consecutive_losses: 0 + streak_cooloff_days: 0 + +dca: + budget_usd: 50 + cadence_days: 7 + +paper: + starting_equity_usd: 10000 # FUNDED paper-forward: seed the synthetic account at $10k + monthly_contribution_usd: 500 # $500/mo contribution during the paper-forward + +# The settlement currency this deployment TRADES IN. It must match the quote leg of the +# products you actually trade: everything here is `-USD` (see `_default_sim_products` / +# `_history_product`), so this is USD. It is NOT used to decide which balance funds a given +# order -- that comes from the product itself (`quote_currency_of`), because BTC-USD spends USD +# whatever this says. Used to screen candidates and to exclude the settlement balance from +# `keel assets holdings`. +quote_currency: USD + +subscription: + # The SIMULATOR's assumed fee-free monthly volume. The LIVE rail-14 cap is not set here -- + # it comes from the attested record: `keel subscription attest --venue coinbase --tier `. + assumed_free_volume_usd: 500 + # What rail 14 permits on a venue that is unattested, suspect, lapsed, or overdue. + # 0 means such a venue cannot buy at all until it is attested. + unsubscribed_allowance_usd: 0 + pacing: opportunistic # opportunistic (monthly cap only) | even_daily (also paces per business day) + +# Coinbase One subscription tiers (Issue #86) -- fee-free monthly TRADING VOLUME (buys + sells) +# allowance per tier, used by `keel simulate`'s tier/fee analysis matrix to compare staying +# within a tier's free volume (throttled, 0 trading fees, but you still pay the subscription) +# against trading freely and paying the taker fee on volume EXCEEDING it. +# free_volume_usd: null means unlimited (Premium -- always fee-free, no cap to exceed). +tiers: + - name: Basic + free_volume_usd: 500 + subscription_usd_month: 4.99 + - name: Preferred + free_volume_usd: 10000 + subscription_usd_month: 29.99 + - name: Premium + free_volume_usd: null + subscription_usd_month: 299.99 + +# Coinbase Advanced trading fees applied to volume beyond a tier's free allowance, for a +# <$1k-30d-volume account (Coinbase's published fee schedule). taker_pct is the sim's default -- +# it fills market-style at next-bar open; maker_pct is exposed for a caller that wants to model +# limit-order fills instead. +fees: + taker_pct: 0.012 + maker_pct: 0.006 + +# Engine-activity logging. verbose=false (default) means only errors/exceptions are ever logged +# (the "keel" logger stays at ERROR level); set verbose: true (or `keel -v`) to also log major +# operations/decisions (INFO level) -- cycle starts, signals, guard vetoes, order outcomes, etc. +# file_count is the TOTAL number of files kept (the active log + rotated backups), each capped +# at max_file_mb. +# verbose: true since 2026-07-29. With it off, a cycle reporting `signals=0` said nothing about +# WHY -- engine.no_signal and engine.setup_rejected are INFO events, so answering "why no order +# today?" meant replaying the rules against the database by hand. One cycle a day over 5 products +# is a trivial amount of INFO, and rotation is bounded at max_file_mb x file_count anyway. +logging: + verbose: true + file: logs/keel.log + max_file_mb: 25 + file_count: 5 + +# G4 overfitting gate (KB §78). NEVER tune these to obtain a desired verdict -- doing so is +# the exact Strathern misuse the gate exists to prevent (§78.7). slope_floor is calibrated +# from §78.8's worked cases: real strategy -0.35, pure random walk -0.61, overfit -0.75. +research: + pbo_max: 0.05 + slope_floor: -0.5 diff --git a/deploy/live-rules.json b/deploy/live-rules.json new file mode 100644 index 00000000..5d3e3dd0 --- /dev/null +++ b/deploy/live-rules.json @@ -0,0 +1,111 @@ +{ + "_comment": "Generated by scripts/rule_manifest.py -- the deployment's strategy library. Regenerate with `python scripts/rule_manifest.py export --db ` and commit the diff whenever the live rule set changes.", + "rules": [ + { + "kind": "turtle_breakout", + "status": "live", + "params": { + "entry_lookback": 40, + "exit_lookback": 20, + "adx_period": 14, + "adx_threshold": 25.0, + "atr_period": 20, + "atr_stop_mult": "2", + "use_macd_confirm": false, + "s1_filter": false, + "min_volume_filter": false, + "volume_ma_period": 20, + "volume_mult": 1.2, + "target_rr": "6", + "product_id": "BTC-USD" + } + }, + { + "kind": "turtle_breakout", + "status": "live", + "params": { + "entry_lookback": 40, + "exit_lookback": 20, + "adx_period": 14, + "adx_threshold": 25.0, + "atr_period": 20, + "atr_stop_mult": "2", + "use_macd_confirm": false, + "s1_filter": false, + "min_volume_filter": false, + "volume_ma_period": 20, + "volume_mult": 1.2, + "target_rr": "6", + "product_id": "ETH-USD" + } + }, + { + "kind": "turtle_breakout", + "status": "live", + "params": { + "entry_lookback": 40, + "exit_lookback": 20, + "adx_period": 14, + "adx_threshold": 25.0, + "atr_period": 20, + "atr_stop_mult": "2", + "use_macd_confirm": false, + "s1_filter": false, + "min_volume_filter": false, + "volume_ma_period": 20, + "volume_mult": 1.2, + "target_rr": "6", + "product_id": "PAXG-USD" + } + }, + { + "kind": "turtle_breakout", + "status": "live", + "params": { + "entry_lookback": 40, + "exit_lookback": 20, + "adx_period": 14, + "adx_threshold": 25.0, + "atr_period": 20, + "atr_stop_mult": "2", + "use_macd_confirm": false, + "s1_filter": false, + "min_volume_filter": false, + "volume_ma_period": 20, + "volume_mult": 1.2, + "target_rr": "6", + "product_id": "ADA-USD" + } + }, + { + "kind": "turtle_breakout", + "status": "live", + "params": { + "entry_lookback": 40, + "exit_lookback": 20, + "adx_period": 14, + "adx_threshold": 25.0, + "atr_period": 20, + "atr_stop_mult": "2", + "use_macd_confirm": false, + "s1_filter": false, + "min_volume_filter": false, + "volume_ma_period": 20, + "volume_mult": 1.2, + "target_rr": "6", + "product_id": "XLM-USD" + } + }, + { + "kind": "dca", + "status": "candidate", + "params": { + "product_id": "BTC-USD", + "cadence_days": 7, + "budget_usd": "25", + "dip_bonus_pct": "0", + "lookback_days": 90 + } + } + ] +} diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 86c6c689..17a45c27 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -62,6 +62,35 @@ keel migrate # EXISTING database: apply outstanding schema migrations. Never s database. It never re-seeds, because that would resurrect rules deliberately deleted or refuted. `--db ` targets a database directly. +### The rule set does not survive a fresh deployment on its own + +An **in-place upgrade** (new wheels + `keel migrate`) keeps the strategy library: the schema steps +are additive (`CREATE TABLE IF NOT EXISTS rules`) and `migrate` never seeds. + +A **fresh deployment** does not. `keel init` seeds every rule at `candidate` from each rule kind's +**constructor defaults**, so any parameter an operator tuned by hand silently reverts — a DCA rule +deliberately set to `budget_usd: 25` comes back as the built-in `50`, unpromoted, on a box that +otherwise looks correctly provisioned. Nothing errors. + +`deploy/live-rules.json` is the committed record of the live deployment's rule set, so that state +is a diff in a PR rather than a fact stored on one laptop: + +``` +python scripts/rule_manifest.py export --db keel-live.db # snapshot; commit the diff +python scripts/rule_manifest.py apply --db keel-live.db # dry run: what a rebuild would do +python scripts/rule_manifest.py apply --db keel-live.db --apply --allow-live +``` + +Regenerate and commit whenever the live rule set changes deliberately. `apply` refuses to rewrite +an existing rule's params (it reports drift and exits non-zero) and refuses to create `live` rules +without `--allow-live` — a manifest must never be able to resize or arm a rule by file edit, which +is what the promotion ladder exists to prevent. Promotion stays `keel rules promote`'s job. + +The deployment's **configs** (`config.live-sandbox.yaml`, `config.paperforward.yaml`, the run +scripts, the launchd plists and the `keel-live`/`keel-paper` wrappers) are tracked as of +2026-08-03 for the same reason. They are still excluded from the wheel and the release assets — +that exclusion comes from the packaging config, which ships only `keel/`, not from `.gitignore`. + The **Migrate database** workflow (Actions → Migrate database → Run workflow) is manual-only. Give it a `db_path` to migrate that database; leave it empty and it verifies the migration chain instead (a fresh DB and a downgraded DB both reach `SCHEMA_VERSION`). CI has no database to reach diff --git a/docs/go-live-runbook.md b/docs/go-live-runbook.md index 911560b8..e1dbb3b1 100644 --- a/docs/go-live-runbook.md +++ b/docs/go-live-runbook.md @@ -66,6 +66,16 @@ keel rules promote # candidate -> paper -> live Promoting to `live` is what the promotion floor exists to gate. If you are deliberately short-circuiting it for this test, know that you are — and promote a single rule, not the library. +A **DCA** rule cannot clear the gate at all, structurally: it has no stop and no target, so +`backtest()` opens a position that never closes, every trade stays `open`, and the aggregates see +`n_trades=0` against `min_trades: 100`. It also needs `--granularity ONE_DAY`, because `Dca` never +sets `self.granularity` the way `TurtleBreakout` does. So a DCA go-live is +`keel rules promote --force`, twice — the bypass that flag documents, not a judgement call. + +Once promoted, record the change: `python scripts/rule_manifest.py export --db ` and commit +`deploy/live-rules.json`. A fresh deployment re-seeds from constructor defaults and will otherwise +bring back a differently-sized rule (see `docs/RELEASING.md`). + ## 4. Run one cycle, in confirm mode, and watch ```bash diff --git a/keel-live b/keel-live new file mode 100755 index 00000000..d25b37d6 --- /dev/null +++ b/keel-live @@ -0,0 +1,25 @@ +#!/bin/bash +# keel-live -- run ANY keel command against the SUPERVISED-LIVE sandbox, with the config and +# database pinned together so they can never be mismatched. +# +# The footgun this exists to remove: `--db` defaults to keel.db, so running +# keel --config config.live-sandbox.yaml agent +# without `--db keel-live.db` would drive the 5 live-status rules against the PAPER ledger. +# Config and db must always travel as a pair; here they do. +# +# THIS IS THE REAL-MONEY PATH. Placing still requires a typed `y` per order (mode: confirm), +# and it fails closed with no TTY -- this wrapper does not weaken any rail, it only fixes the +# argument pairing. +# +# ./keel-live status # read-only snapshot +# ./keel-live agent # interactive cycle; approve a detected breakout SAME DAY +# ./keel-live rules list +# +# Authored in the dev repo (gitignored). DEPLOY (copy) to ~/keel. +set -euo pipefail + +# Resolve the deployment dir from this script's own location, so it works from any cwd. +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$DIR" + +exec ./.venv/bin/keel --config config.live-sandbox.yaml --db keel-live.db "$@" diff --git a/keel-live-run.sh b/keel-live-run.sh new file mode 100755 index 00000000..5dc09108 --- /dev/null +++ b/keel-live-run.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# SUPERVISED-LIVE daily DETECTOR for the 5-trend Turtle sandbox (keel v0.2, ~/keel). +# +# Runs ONE confirm-mode agent cycle HEADLESS. Because confirm mode FAILS CLOSED with no TTY +# (keel's _interactive_confirm declines when stdin is not a terminal), this NEVER places an order +# and moves no money -- it only DETECTS whether a Turtle breakout fired today. If one did, it +# posts a macOS notification telling you to run the agent INTERACTIVELY to approve it. +# +# IMPORTANT: approve SAME DAY -- a daily breakout can fade, so a signal seen today may not still +# be there tomorrow. This is a detector + reminder, not the thing that trades. +# +# EXACTLY ONCE PER CALENDAR DAY, and it CATCHES UP -- same day-stamp scheme as +# paperforward-run.sh, for the same reason: launchd re-runs a missed StartCalendarInterval on +# wake from SLEEP but NOT when the trigger passed while the machine was powered OFF, so a +# shutdown over 09:05 silently skipped the detector for the day (observed 2026-07-28). The plist +# now fires hourly and on load; the stamp keeps that to one cycle. Detecting twice would be +# harmless in itself, but it would notify twice for the same breakout. +# +# The stamp is written only after keel EXITS CLEAN, so a failed cycle is retried on the next +# trigger instead of being recorded as done -- and a detector that failed must not look like a +# quiet "no signals today". +# +# Authored in the dev repo (gitignored). DEPLOY (copy) to ~/keel and schedule via +# com.keel.live.plist. Runs from the deployment's own venv + config + db. +set -uo pipefail # NOT -e: a nonzero exit from keel/grep must not skip the notify path + +DIR="/Users/elmehdiaitbrahim/keel" +KEEL="$DIR/.venv/bin/keel" +CONFIG="config.live-sandbox.yaml" +DB="keel-live.db" +OUTLOG="$DIR/logs/keel-live.out.log" +PENDLOG="$DIR/logs/keel-live.pending.log" +STAMP="$DIR/logs/.keel-live-last-run" +SCHED_HOUR=9 + +cd "$DIR" || exit 1 + +TODAY="$(date '+%Y-%m-%d')" +# 10# forces base 10: `date +%H` yields 08/09, which arithmetic would otherwise read as octal. +HOUR="$((10#$(date '+%H')))" +STAMPED="$(cat "$STAMP" 2>/dev/null || true)" + +if [ "$STAMPED" = "$TODAY" ]; then + printf '%s [keel-live] detector already ran today -- skipping\n' "$(date '+%Y-%m-%d %H:%M')" + exit 0 +fi + +if [ "$HOUR" -lt "$SCHED_HOUR" ]; then + printf '%s [keel-live] before %02d:00 -- leaving today to the scheduled run\n' \ + "$(date '+%Y-%m-%d %H:%M')" "$SCHED_HOUR" + exit 0 +fi + +# One headless confirm cycle. Places NOTHING (no TTY -> the confirm gate declines). Captures the +# LoopResult line, which reads e.g.: [ts] mode=confirm polled=.. products=[..] signals=N entered=0 .. +OUT="$("$KEEL" --config "$CONFIG" --db "$DB" agent 2>&1)" +STATUS=$? +printf '%s\n' "$OUT" >> "$OUTLOG" + +# Parse `signals=N` from the LoopResult (default 0 if the line is absent, e.g. a kill-switch skip). +SIGNALS="$(printf '%s\n' "$OUT" | grep -oE 'signals=[0-9]+' | tail -1 | cut -d= -f2)" +SIGNALS="${SIGNALS:-0}" + +if [ "${SIGNALS}" -gt 0 ]; then + MSG="${SIGNALS} Turtle signal(s) PENDING -- run the agent interactively to approve (same day)." + # macOS notification (LaunchAgents run in your GUI session, so this shows up + plays a sound). + /usr/bin/osascript -e "display notification \"${MSG}\" with title \"keel-live\" subtitle \"supervised live\" sound name \"Glass\"" 2>/dev/null || true + printf '%s [keel-live] %s\n' "$(date '+%Y-%m-%d %H:%M')" "${MSG}" >> "$PENDLOG" +fi + +# Only a clean cycle counts as "today is done"; anything else leaves the day open for a retry. +if [ "$STATUS" -eq 0 ]; then + printf '%s\n' "$TODAY" > "$STAMP" +else + printf '%s [keel-live] cycle exited %d -- not stamping, will retry\n' \ + "$(date '+%Y-%m-%d %H:%M')" "$STATUS" >> "$OUTLOG" +fi +exit "$STATUS" diff --git a/keel-paper b/keel-paper new file mode 100755 index 00000000..dd772c8c --- /dev/null +++ b/keel-paper @@ -0,0 +1,20 @@ +#!/bin/bash +# keel-paper -- run ANY keel command against the PAPER-FORWARD deployment, with the config and +# database pinned together. Symmetric with `keel-live`; having both means you never type a raw +# `keel` invocation and never have to remember which db goes with which config. +# +# Paper mode places NOTHING real -- it loads only `paper`-status rules and routes them to the +# PaperTrader. The live READ-ONLY Coinbase API is still used for candles. +# +# ./keel-paper status +# ./keel-paper tui # the full-screen operator dashboard +# ./keel-paper agent # one paper cycle (same as paperforward-run.sh) +# +# Authored in the dev repo (gitignored). DEPLOY (copy) to ~/keel. +set -euo pipefail + +# Resolve the deployment dir from this script's own location, so it works from any cwd. +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$DIR" + +exec ./.venv/bin/keel --config config.paperforward.yaml "$@" diff --git a/paperforward-run.sh b/paperforward-run.sh new file mode 100755 index 00000000..9c0cfc73 --- /dev/null +++ b/paperforward-run.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Daily paper-forward runner -- ~/keel deployment (keel release install). +# One agent cycle in PAPER mode against the live READ-ONLY Coinbase API; it places NOTHING real. +# Self-contained: calls the deployment's own venv binary, so no `uv`/asdf PATH is needed. +# +# Lives OUTSIDE ~/Documents on purpose: launchd-spawned processes are not granted TCC access to +# ~/Documents, so a copy under there fails to exec with "Operation not permitted" and the daily +# cycle silently never runs. Home root is not TCC-protected. +# +# EXACTLY ONCE PER CALENDAR DAY, and it CATCHES UP. launchd re-runs a missed +# StartCalendarInterval when the machine wakes from SLEEP, but NOT when the trigger time passed +# while the machine was powered OFF -- a shutdown over 09:00 silently skipped the day (observed +# 2026-07-28: booted 09:24, `launchctl print` showed runs = 0, and the paper-forward lost the +# day). So the plist now also fires hourly through the day and on load, and the day-stamp below +# is what keeps that from running more than one cycle: the daily Turtle wants exactly one. +# +# Two guards, in order: +# 1. already stamped for today -> skip (the hourly triggers are RETRIES, not extra cycles) +# 2. before SCHED_HOUR -> skip (an early boot must not consume the day; leave it to +# the scheduled run so the cadence stays put) +# The stamp is written only AFTER a successful cycle, so a failed run (no network on wake, say) +# is retried by the next trigger rather than being recorded as done. +# +# Authored in the dev repo (gitignored). DEPLOY (copy) to ~/keel and schedule via +# com.keel.paperforward.plist. +set -euo pipefail + +DIR="/Users/elmehdiaitbrahim/keel" +cd "$DIR" + +STAMP="$DIR/logs/.paperforward-last-run" +SCHED_HOUR=9 + +TODAY="$(date '+%Y-%m-%d')" +# 10# forces base 10: `date +%H` yields 08/09, which arithmetic would otherwise read as octal. +HOUR="$((10#$(date '+%H')))" +STAMPED="$(cat "$STAMP" 2>/dev/null || true)" + +if [ "$STAMPED" = "$TODAY" ]; then + printf '%s [paperforward] cycle already ran today -- skipping\n' "$(date '+%Y-%m-%d %H:%M')" + exit 0 +fi + +if [ "$HOUR" -lt "$SCHED_HOUR" ]; then + printf '%s [paperforward] before %02d:00 -- leaving today to the scheduled run\n' \ + "$(date '+%Y-%m-%d %H:%M')" "$SCHED_HOUR" + exit 0 +fi + +# One cycle per day; the LaunchAgent supplies the cadence and the retries. Paper mode + funded +# config. `set -e` means a failure here stops short of the stamp, so the next trigger retries. +./.venv/bin/keel --config config.paperforward.yaml agent + +printf '%s\n' "$TODAY" > "$STAMP" diff --git a/scripts/rule_manifest.py b/scripts/rule_manifest.py new file mode 100644 index 00000000..95cf5f43 --- /dev/null +++ b/scripts/rule_manifest.py @@ -0,0 +1,186 @@ +"""Export the `rules` table to a reviewable JSON manifest, and rebuild it from one. + +**Why this exists.** A deployment's strategy library lives in the DATABASE, not in `config.yaml`. +`keel migrate` deliberately never re-seeds (see `docs/RELEASING.md`), so an in-place upgrade keeps +the rule set -- but a FRESH deployment gets `keel init`, which seeds every rule at `candidate` +using each rule kind's CONSTRUCTOR DEFAULTS. Any parameter an operator tuned by hand is silently +replaced by the default: a DCA rule deliberately set to `budget_usd: 25` comes back as the +built-in `50`, at `candidate`, on a box that otherwise looks correctly provisioned. Nothing +errors. This module makes that state an artifact you can diff in a PR instead of a fact that +lives only on one laptop. + +**`export`** writes the manifest. It is the source of truth's snapshot, not the source of truth: +re-run it and commit the diff whenever the live rule set changes deliberately. + +**`apply`** rebuilds a database from the manifest. Two safety properties, both deliberate: + +- It **never rewrites an existing rule's params.** Matching is by `(kind, product_id)`; a rule + that exists with different params is reported as DRIFT and the run fails. Silently rewriting + params from a JSON file would let an edited manifest change a LIVE rule's position size, which + is exactly the kind of un-reviewed money change the promotion ladder exists to prevent. + Reconcile drift deliberately, by hand. +- It **refuses to create `live` rules without `--allow-live`.** Seeding straight to `live` bypasses + the promotion gate (same bypass as `keel rules seed --status live` and `rules promote --force`), + so it takes an explicit flag and prints a loud warning, mirroring how those commands behave. + +Dry-run is the default. Nothing is written without `--apply`. + +Release tooling: deliberately NOT shipped in the wheel. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from keel.data.db import connect +from keel.data.repository import Repository + +#: Statuses that can trade real money, and so require `--allow-live` to create from a manifest. +_ARMED_STATUSES = frozenset({"live"}) + + +def _key(kind: str, params: dict[str, Any]) -> tuple[str, str]: + """Identity of a rule for manifest purposes: its kind and product. + + Matches `keel rules seed`'s own idempotency key, so `apply` and `seed` agree on what "already + exists" means rather than each having a private notion of duplicate. + """ + return (kind, str(params.get("product_id", ""))) + + +def export(db_path: str, out_path: Path) -> int: + """Write every rule in `db_path` to `out_path` as JSON. Returns the number exported.""" + repo = Repository(connect(db_path)) + rules = [ + {"kind": row["kind"], "status": row["status"], "params": row["params"]} + for row in repo.get_rules() + ] + manifest = { + "_comment": ( + "Generated by scripts/rule_manifest.py -- the deployment's strategy library. " + "Regenerate with `python scripts/rule_manifest.py export --db ` and commit the " + "diff whenever the live rule set changes." + ), + "rules": rules, + } + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(manifest, indent=2) + "\n") + return len(rules) + + +def apply( + db_path: str, + manifest_path: Path, + *, + write: bool = False, + allow_live: bool = False, +) -> int: + """Reconcile `db_path`'s rules against `manifest_path`. + + Returns a process exit code: 0 when the database matches (or was made to match) the manifest, + 1 when drift or a refused `live` insert means it does not. + """ + manifest = json.loads(manifest_path.read_text()) + wanted = manifest["rules"] if isinstance(manifest, dict) else manifest + + repo = Repository(connect(db_path)) + existing = {_key(row["kind"], row["params"]): row for row in repo.get_rules()} + + to_insert: list[dict[str, Any]] = [] + drift: list[str] = [] + refused: list[str] = [] + + for entry in wanted: + kind, status, params = entry["kind"], entry["status"], entry["params"] + key = _key(kind, params) + current = existing.get(key) + + if current is None: + if status in _ARMED_STATUSES and not allow_live: + refused.append(f"{kind}:{key[1]} wants status={status}") + else: + to_insert.append(entry) + continue + + if current["params"] != params: + differing = sorted( + k + for k in set(current["params"]) | set(params) + if current["params"].get(k) != params.get(k) + ) + drift.append( + f"{kind}:{key[1]} rule id={current['id']} params differ on {differing} " + f"(db={ {k: current['params'].get(k) for k in differing} }, " + f"manifest={ {k: params.get(k) for k in differing} })" + ) + elif current["status"] != status: + drift.append( + f"{kind}:{key[1]} rule id={current['id']} status db={current['status']!r} " + f"manifest={status!r} -- advance it with `keel rules promote`, not this script" + ) + + for entry in to_insert: + key = _key(entry["kind"], entry["params"]) + verb = "would insert" if not write else "inserted" + print(f"{verb}: {entry['kind']}:{key[1]} status={entry['status']}") + if write: + if entry["status"] in _ARMED_STATUSES: + print( + f" WARNING: created at status={entry['status']}, BYPASSING the promotion " + f"gate. This rule can place real orders as soon as the rails pass.", + file=sys.stderr, + ) + repo.insert_rule(entry["kind"], entry["params"], status=entry["status"]) + + for line in refused: + print(f"REFUSED: {line} -- pass --allow-live to create armed rules", file=sys.stderr) + for line in drift: + print(f"DRIFT: {line}", file=sys.stderr) + + if not to_insert and not drift and not refused: + print(f"in sync: {len(wanted)} rule(s) match {manifest_path}") + elif not write and to_insert: + print("\ndry run -- nothing written. Re-run with --apply to write.") + + return 1 if (drift or refused) else 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + + p_export = sub.add_parser("export", help="write the DB's rules to a JSON manifest") + p_export.add_argument("--db", required=True, help="database to read") + p_export.add_argument( + "--out", type=Path, default=Path("deploy/live-rules.json"), help="manifest path to write" + ) + + p_apply = sub.add_parser("apply", help="rebuild a DB's rules from a JSON manifest") + p_apply.add_argument("--db", required=True, help="database to write") + p_apply.add_argument( + "--manifest", type=Path, default=Path("deploy/live-rules.json"), help="manifest to read" + ) + p_apply.add_argument( + "--apply", action="store_true", dest="write", help="actually write (default: dry run)" + ) + p_apply.add_argument( + "--allow-live", + action="store_true", + help="permit creating rules at status=live, bypassing the promotion gate", + ) + + args = parser.parse_args(argv) + + if args.command == "export": + count = export(args.db, args.out) + print(f"exported {count} rule(s) to {args.out}") + return 0 + return apply(args.db, args.manifest, write=args.write, allow_live=args.allow_live) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_rule_manifest.py b/tests/test_rule_manifest.py new file mode 100644 index 00000000..4e3214e8 --- /dev/null +++ b/tests/test_rule_manifest.py @@ -0,0 +1,144 @@ +"""`scripts/rule_manifest.py` -- the strategy library as a reviewable artifact. + +The rules table is deployment state that no config file records, and `keel init` re-seeds it from +CONSTRUCTOR DEFAULTS -- so a hand-tuned rule silently reverts on a fresh box. These tests pin the +two safety properties that make rebuilding from a manifest safe to automate: an existing rule's +params are never rewritten from a file, and `live` rules are never created without an explicit +opt-in. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from keel.data.db import connect, migrate # noqa: E402 +from keel.data.repository import Repository # noqa: E402 +from scripts.rule_manifest import apply, export # noqa: E402 + +DCA = { + "kind": "dca", + "status": "candidate", + "params": { + "product_id": "BTC-USD", + "cadence_days": 7, + "budget_usd": "25", + "dip_bonus_pct": "0", + "lookback_days": 90, + }, +} + + +def _db(tmp_path: Path, name: str = "t.db") -> str: + path = str(tmp_path / name) + migrate(connect(path)) + return path + + +def _manifest(tmp_path: Path, rules: list[dict]) -> Path: + path = tmp_path / "m.json" + path.write_text(json.dumps({"rules": rules})) + return path + + +def _rules(db: str) -> list[dict]: + return Repository(connect(db)).get_rules() + + +def test_export_then_apply_round_trips(tmp_path: Path) -> None: + source = _db(tmp_path, "source.db") + Repository(connect(source)).insert_rule(DCA["kind"], DCA["params"], status="candidate") + + out = tmp_path / "deploy" / "live-rules.json" + assert export(source, out) == 1 + + target = _db(tmp_path, "target.db") + assert apply(target, out, write=True) == 0 + + rebuilt = _rules(target) + assert len(rebuilt) == 1 + assert rebuilt[0]["kind"] == "dca" + assert rebuilt[0]["params"] == DCA["params"] + assert rebuilt[0]["status"] == "candidate" + + +def test_dry_run_writes_nothing(tmp_path: Path) -> None: + db = _db(tmp_path) + manifest = _manifest(tmp_path, [DCA]) + + assert apply(db, manifest) == 0 + assert _rules(db) == [] + + +def test_apply_is_idempotent(tmp_path: Path) -> None: + db = _db(tmp_path) + manifest = _manifest(tmp_path, [DCA]) + + assert apply(db, manifest, write=True) == 0 + assert apply(db, manifest, write=True) == 0 + assert len(_rules(db)) == 1 + + +def test_live_rules_refused_without_allow_live(tmp_path: Path) -> None: + """Seeding straight to `live` bypasses the promotion gate, so it takes an explicit flag.""" + db = _db(tmp_path) + manifest = _manifest(tmp_path, [{**DCA, "status": "live"}]) + + assert apply(db, manifest, write=True) == 1 + assert _rules(db) == [] + + assert apply(db, manifest, write=True, allow_live=True) == 0 + assert _rules(db)[0]["status"] == "live" + + +def test_param_drift_is_reported_and_never_rewritten(tmp_path: Path) -> None: + """A manifest must not be able to resize a rule that already exists -- especially a live one.""" + db = _db(tmp_path) + Repository(connect(db)).insert_rule(DCA["kind"], DCA["params"], status="live") + + fatter = {**DCA["params"], "budget_usd": "500"} + manifest = _manifest(tmp_path, [{**DCA, "status": "live", "params": fatter}]) + + assert apply(db, manifest, write=True, allow_live=True) == 1 + assert _rules(db)[0]["params"]["budget_usd"] == "25" + + +def test_status_drift_is_reported_not_promoted(tmp_path: Path) -> None: + """Promotion is `keel rules promote`'s job; a file edit must not arm a candidate.""" + db = _db(tmp_path) + Repository(connect(db)).insert_rule(DCA["kind"], DCA["params"], status="candidate") + + manifest = _manifest(tmp_path, [{**DCA, "status": "live"}]) + + assert apply(db, manifest, write=True, allow_live=True) == 1 + assert _rules(db)[0]["status"] == "candidate" + + +def test_rules_match_on_kind_and_product(tmp_path: Path) -> None: + """Same kind, different product is a different rule -- matching `keel rules seed`'s key.""" + db = _db(tmp_path) + eth = {**DCA, "params": {**DCA["params"], "product_id": "ETH-USD"}} + manifest = _manifest(tmp_path, [DCA, eth]) + + assert apply(db, manifest, write=True) == 0 + assert {r["params"]["product_id"] for r in _rules(db)} == {"BTC-USD", "ETH-USD"} + + +def test_committed_manifest_is_valid(tmp_path: Path) -> None: + """The checked-in manifest must actually rebuild -- a stale or malformed one is worthless.""" + committed = REPO_ROOT / "deploy" / "live-rules.json" + assert committed.exists(), "deploy/live-rules.json is missing" + + db = _db(tmp_path) + assert apply(db, committed, write=True, allow_live=True) == 0 + + rebuilt = _rules(db) + assert len(rebuilt) == len(json.loads(committed.read_text())["rules"]) + dca = [r for r in rebuilt if r["kind"] == "dca"] + assert len(dca) == 1, "the live DCA rule is missing from the manifest" + assert dca[0]["params"]["budget_usd"] == "25", "DCA budget must not revert to the default 50"