From f12eed5675e8056c50ac79ac6a143d0c1193d5e7 Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim Date: Mon, 20 Jul 2026 19:02:10 -0400 Subject: [PATCH 1/9] feat(cli): ship a config template + keel init / init-config / seed --status live Makes a fresh installed release usable without copying files out of the repo, and gives the supervised live test a first-class seeding path. 1. CONFIG TEMPLATE IN THE WHEEL. keel/templates/config.yaml is packaged via pyproject `artifacts`, and `keel init-config` writes it into the working directory (refuses to clobber without --force). Verified the template is actually INSIDE the built wheel and that init-config runs from a clean installed artifact -- packaging that silently drops a data file is exactly the trap here. A test guards that the packaged copy stays in sync with the repo config.yaml, and that the template parses as a valid config. 2. `keel init` -- convenience: init-config then `rules seed` (candidates). Scaffolds a working dir in one command. 3. `rules seed --status {candidate|paper|live}` (default candidate). Live bypasses the promotion gate and prints a loud warning -- it is the supervised live-order test's seeding path, replacing the runbook's hand-rolled insert_rule() poke. Live-seeded rules are still confirm-gated and rail-guarded; the warning says to remove them after. Verified end to end: init-config from an installed wheel writes a valid config; `rules seed --kinds dca --products BTC-USD --status live` creates one live DCA rule; rules list shows status=live. Co-Authored-By: Claude Opus 4.8 (1M context) --- keel/cli.py | 73 +++++++++++++++++++++- keel/templates/config.yaml | 118 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 3 + tests/test_init_and_seed.py | 111 +++++++++++++++++++++++++++++++++ 4 files changed, 302 insertions(+), 3 deletions(-) create mode 100644 keel/templates/config.yaml create mode 100644 tests/test_init_and_seed.py diff --git a/keel/cli.py b/keel/cli.py index cf2b5bb8..6f53d8e2 100644 --- a/keel/cli.py +++ b/keel/cli.py @@ -273,6 +273,53 @@ def cli( ctx.obj["verbose"] = verbose +# -- init (scaffold a working directory) ---------------------------------------------------- + + +def _template_config_text() -> str: + """The default config.yaml shipped inside the wheel (see pyproject `artifacts`).""" + from importlib.resources import files + + return (files("keel.templates") / "config.yaml").read_text(encoding="utf-8") + + +@cli.command("init-config") +@click.option( + "--config", "config_path", default=DEFAULT_CONFIG_PATH, show_default=True, + help="Where to write the config file.", +) +@click.option("--force", is_flag=True, default=False, help="Overwrite an existing config.") +def init_config(config_path: str, force: bool) -> None: + """Write a default `config.yaml` into the current directory, ready to edit. + + The installed wheel ships this template so a fresh working directory has a config to start + from -- edit `allowlist`, `caps`, and `auto_trade.mode` before running anything live. + """ + path = Path(config_path) + if path.exists() and not force: + raise click.ClickException(f"{path} already exists; pass --force to overwrite") + path.write_text(_template_config_text(), encoding="utf-8") + click.echo(f"wrote {path}. Edit allowlist/caps/auto_trade.mode before going live.") + + +@cli.command("init") +@click.option( + "--config", "config_path", default=DEFAULT_CONFIG_PATH, show_default=True, + help="Config file to write.", +) +@click.option("--force", is_flag=True, default=False, help="Overwrite an existing config.") +@click.pass_context +def init_cmd(ctx: click.Context, config_path: str, force: bool) -> None: + """Scaffold a working directory: write `config.yaml`, then seed the rules table (candidates). + + A convenience for a fresh install -- equivalent to `keel init-config` followed by + `keel rules seed`. Seeds `candidate` rules only; promoting to paper/live is a separate, + deliberate step. + """ + ctx.invoke(init_config, config_path=config_path, force=force) + ctx.invoke(rules_seed, products=None, kinds=None, force=False, status="candidate") + + # -- db import ------------------------------------------------------------------------------ @@ -1321,6 +1368,14 @@ def _json_plain(value: Any) -> Any: default=None, help="Comma-separated rule kinds (default: every kind in agent.RULE_REGISTRY).", ) +@click.option( + "--status", + type=click.Choice(["candidate", "paper", "live"]), + default="candidate", + show_default=True, + help="Status to seed at. `live` bypasses the promotion gate -- for the supervised " + "live-order test only (see the go-live runbook).", +) @click.option( "--force", is_flag=True, @@ -1329,7 +1384,13 @@ def _json_plain(value: Any) -> Any: ) @click.pass_context @with_disclaimer -def rules_seed(ctx: click.Context, products: str | None, kinds: str | None, force: bool) -> None: +def rules_seed( + ctx: click.Context, + products: str | None, + kinds: str | None, + force: bool, + status: str = "candidate", +) -> None: """Seed the `rules` table with one `candidate` rule per (kind, product) pair (Issue #81). The `rules` table starts out empty and nothing else populates it -- with zero rows, @@ -1387,10 +1448,16 @@ def rules_seed(ctx: click.Context, products: str | None, kinds: str | None, forc rule = rule_cls(product_id=product) params = _json_plain(rule.describe()["params"]) params["product_id"] = product - repo.insert_rule(kind, params, status="candidate", now_ts=now_ts) + repo.insert_rule(kind, params, status=status, now_ts=now_ts) seeded.append(label) - click.echo(f"seeded={len(seeded)} skipped={len(skipped)}") + click.echo(f"seeded={len(seeded)} skipped={len(skipped)} status={status}") + if status == "live": + click.echo( + "⚠️ seeded at LIVE status, bypassing the promotion gate. This is for the " + "supervised live-order test only -- the agent will act on these (still confirm-" + "gated and rail-guarded). Do not leave live-seeded rules in place afterwards." + ) for label in seeded: click.echo(f" seeded: {label}") for label in skipped: diff --git a/keel/templates/config.yaml b/keel/templates/config.yaml new file mode 100644 index 00000000..3d589682 --- /dev/null +++ b/keel/templates/config.yaml @@ -0,0 +1,118 @@ +# keel runtime configuration. +# +# These are Phase 1 placeholders (spec §21 open item) — no live orders are placed in Phase 1. +# `allowlist` and `caps` are required and validated by keel.config.load_config; missing or +# invalid values raise ConfigError naming the offending key rather than silently defaulting. + +allowlist: + - BTC + - ETH + - PAXG + +target_weights: + BTC: 0.40 + ETH: 0.30 + PAXG: 0.30 + +risk_pct: 0.01 + +caps: + # max_per_order_usd / max_per_day_usd are OPTIONAL internal RISK limits, not real Coinbase + # limits -- Coinbase One's only subscription constraint is monthly fee-free trading VOLUME + # (see `subscription:` below). They default to a non-binding $1B when omitted (Issue #85); + # set explicit values here only if you want an extra per-order/per-day risk ceiling tighter + # than the exposure/concentration caps below. Left at their non-binding default here so + # risk-sized rule orders ($400-24k typical) aren't silently rejected. + max_exposure_usd: 5000 + max_per_asset_pct: 0.50 + +market_data: + granularities: + - ONE_DAY + - ONE_HOUR + - FIFTEEN_MINUTE + history_days: 365 + +auto_trade: + mode: paper + enabled: false + interval_sec: 900 + # how long a `keel arm-bypass` token stays valid (seconds) before bypass mode requires + # re-arming (Issue #60) -- `keel agent --bypass` still needs a fresh armed token even with + # the CLI passphrase gate satisfied. + bypass_arm_ttl_sec: 3600 + +promotion: + min_trades: 100 + min_expectancy: 0.0 + min_rr: 1.5 + min_win_rate: 0.55 + +money_mgmt: + profit_trigger_pct: 0.10 + acceleration_pct: 0.05 + max_total_dd_pct: 0.20 + max_weekly_dd_pct: 0.08 + # Rail 16 (consecutive-loss breaker) — DISABLED by default (0 = off). + # Set from a backtest sweep, and set it ABOVE the strategy's tested max losing streak: + # turtle_breakout's max streak is 5, so a threshold of 3 would fire on normal variance. + max_consecutive_losses: 0 + streak_cooloff_days: 0 + +dca: + budget_usd: 50 + cadence_days: 7 + +# quote-currency the executor draws BUY notional from (rail 13, USDC-funding) -- never bank/ACH. +quote_currency: USDC + +subscription: + # The SIMULATOR's assumed fee-free monthly volume. The LIVE rail-14 cap is not set here -- + # it comes from the attested record: `keel subscription attest --venue coinbase --tier `. + assumed_free_volume_usd: 500 + # What rail 14 permits on a venue that is unattested, suspect, lapsed, or overdue. + # 0 means such a venue cannot buy at all until it is attested. + unsubscribed_allowance_usd: 0 + pacing: opportunistic # opportunistic (monthly cap only) | even_daily (also paces per business day) + +# Coinbase One subscription tiers (Issue #86) -- fee-free monthly TRADING VOLUME (buys + sells) +# allowance per tier, used by `keel simulate`'s tier/fee analysis matrix to compare staying +# within a tier's free volume (throttled, 0 trading fees, but you still pay the subscription) +# against trading freely and paying the taker fee on volume EXCEEDING it. +# free_volume_usd: null means unlimited (Premium -- always fee-free, no cap to exceed). +tiers: + - name: Basic + free_volume_usd: 500 + subscription_usd_month: 4.99 + - name: Preferred + free_volume_usd: 10000 + subscription_usd_month: 29.99 + - name: Premium + free_volume_usd: null + subscription_usd_month: 299.99 + +# Coinbase Advanced trading fees applied to volume beyond a tier's free allowance, for a +# <$1k-30d-volume account (Coinbase's published fee schedule). taker_pct is the sim's default -- +# it fills market-style at next-bar open; maker_pct is exposed for a caller that wants to model +# limit-order fills instead. +fees: + taker_pct: 0.012 + maker_pct: 0.006 + +# Engine-activity logging. verbose=false (default) means only errors/exceptions are ever logged +# (the "keel" logger stays at ERROR level); set verbose: true (or `keel -v`) to also log major +# operations/decisions (INFO level) -- cycle starts, signals, guard vetoes, order outcomes, etc. +# file_count is the TOTAL number of files kept (the active log + rotated backups), each capped +# at max_file_mb. +logging: + verbose: false + file: logs/keel.log + max_file_mb: 25 + file_count: 5 + +# G4 overfitting gate (KB §78). NEVER tune these to obtain a desired verdict -- doing so is +# the exact Strathern misuse the gate exists to prevent (§78.7). slope_floor is calibrated +# from §78.8's worked cases: real strategy -0.35, pure random walk -0.61, overfit -0.75. +research: + pbo_max: 0.05 + slope_floor: -0.5 diff --git a/pyproject.toml b/pyproject.toml index 9b113ce0..d873c90d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,9 @@ module-root = "" # but the import package and the CLI command stay `keel`. Without this the backend would infer # `keel_trader/` from the distribution name. module-name = "keel" +# Ship the default config template inside the wheel so `keel init-config` can write it out on a +# fresh install (the wheel otherwise contains only .py files). +artifacts = ["keel/templates/*.yaml"] [tool.uv.workspace] members = ["packages/*"] diff --git a/tests/test_init_and_seed.py b/tests/test_init_and_seed.py new file mode 100644 index 00000000..7516b6dd --- /dev/null +++ b/tests/test_init_and_seed.py @@ -0,0 +1,111 @@ +"""`keel init-config` / `init` / `rules seed --status` (prod-install scaffolding).""" + +from __future__ import annotations + +from pathlib import Path + +from click.testing import CliRunner + +from keel.cli import _template_config_text, cli +from keel.config import load_config +from keel.data.db import connect, migrate +from keel.data.repository import Repository + + +def _repo(tmp_path): + conn = connect(str(tmp_path / "t.db")) + migrate(conn) + return Repository(conn) + + +# -- the packaged template ----------------------------------------------------- + + +def test_the_shipped_template_is_a_VALID_config(tmp_path): + """A default config that does not parse would break every fresh install.""" + p = tmp_path / "c.yaml" + p.write_text(_template_config_text()) + load_config(str(p)) # raises ConfigError if invalid + + +def test_the_template_stays_in_sync_with_the_repo_config(): + """The wheel ships a COPY of config.yaml; this fails if they drift apart. + + Without it, edits to the repo config.yaml silently never reach the packaged template. + """ + repo_config = Path(__file__).resolve().parent.parent / "config.yaml" + assert _template_config_text() == repo_config.read_text(encoding="utf-8"), ( + "keel/templates/config.yaml has drifted from the repo config.yaml -- re-copy it" + ) + + +# -- init-config --------------------------------------------------------------- + + +def test_init_config_writes_a_config(tmp_path): + out = tmp_path / "config.yaml" + result = CliRunner().invoke(cli, ["init-config", "--config", str(out)]) + assert result.exit_code == 0, result.output + assert out.exists() + load_config(str(out)) + + +def test_init_config_refuses_to_clobber_without_force(tmp_path): + out = tmp_path / "config.yaml" + out.write_text("mine") + result = CliRunner().invoke(cli, ["init-config", "--config", str(out)]) + assert result.exit_code != 0 + assert "already exists" in result.output + assert out.read_text() == "mine" + + forced = CliRunner().invoke(cli, ["init-config", "--config", str(out), "--force"]) + assert forced.exit_code == 0 + assert out.read_text() != "mine" + + +# -- init (config + seed) ------------------------------------------------------ + + +def test_init_writes_config_and_seeds_candidates(tmp_path): + db = tmp_path / "t.db" + cfg = tmp_path / "config.yaml" + result = CliRunner().invoke(cli, ["--db", str(db), "init", "--config", str(cfg)]) + assert result.exit_code == 0, result.output + assert cfg.exists() + rules = Repository(connect(str(db))).get_rules() + assert rules, "init should have seeded rules" + assert all(r["status"] == "candidate" for r in rules), "init must seed candidates only" + + +# -- rules seed --status ------------------------------------------------------- + + +def test_seed_defaults_to_candidate(tmp_path): + repo = _repo(tmp_path) + CliRunner().invoke( + cli, ["--db", str(tmp_path / "t.db"), "rules", "seed", + "--kinds", "dca", "--products", "BTC-USD"] + ) + rules = repo.get_rules() + assert rules and all(r["status"] == "candidate" for r in rules) + + +def test_seed_status_live_bypasses_the_gate_and_warns(tmp_path): + repo = _repo(tmp_path) + result = CliRunner().invoke( + cli, ["--db", str(tmp_path / "t.db"), "rules", "seed", + "--kinds", "dca", "--products", "BTC-USD", "--status", "live"] + ) + assert result.exit_code == 0, result.output + assert "LIVE status" in result.output + assert "supervised live-order test only" in result.output + live = repo.get_rules("live") + assert len(live) == 1 + assert live[0]["kind"] == "dca" + + +def test_seed_rejects_an_unknown_status(tmp_path): + result = CliRunner().invoke( + cli, ["--db", str(tmp_path / "t.db"), "rules", "seed", "--status", "bogus"] + ) + assert result.exit_code != 0 From 6968c386dcbe3bba34b9a4d65d69853a7726921b Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim Date: Tue, 21 Jul 2026 18:10:22 -0400 Subject: [PATCH 2/9] docs(spec): release packaging & bootstrap design (items 4/5/6) PR-body release notes, config.yaml as a confirm-mode live release asset, and a seed/migrate lifecycle with a manually-dispatchable migration workflow. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...7-21-release-packaging-bootstrap-design.md | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-21-release-packaging-bootstrap-design.md diff --git a/docs/superpowers/specs/2026-07-21-release-packaging-bootstrap-design.md b/docs/superpowers/specs/2026-07-21-release-packaging-bootstrap-design.md new file mode 100644 index 00000000..3759f171 --- /dev/null +++ b/docs/superpowers/specs/2026-07-21-release-packaging-bootstrap-design.md @@ -0,0 +1,228 @@ +# Release packaging & bootstrap — design + +**Date:** 2026-07-21 +**Status:** Approved design (pending user review) +**Workstream:** B of three (see the decomposition below). Items 4, 5, 6 of the 2026-07-21 requirements. + +## Context + +`keel` is a live-money spot-crypto trading agent. Its release path is deliberately manual and +human-gated (`.github/workflows/release.yml`, `docs/RELEASING.md`): nothing that can move money +ships on a merge. A release runs tests+ruff, stamps a build-identity hash, builds wheels for the +workspace (`keel` + `keel-core` + `keel-broker-*`), verifies the artifact self-identifies as a +clean `[release]`, tags `v`, and publishes a GitHub Release with the wheels attached and +auto-generated notes. + +Three requirements refine that path. This spec covers them as one workstream because they all +answer "what does a release ship, and how does a fresh deployment come up correctly?" + +The 2026-07-21 requirement list decomposed into three workstreams; this is **B**: + +| | Workstream | Items | +|---|---|---| +| A | Security simplification (drop vault + passphrase; autonomous = profile flag) | 1, 2 | +| **B** | **Release packaging & bootstrap (this spec)** | **4, 5, 6** | +| C | Asset sourcing & vetting (fetch Coinbase holdings → screen gate) | 3 | + +A and C are **out of scope here** and get their own spec → plan → implementation cycles. + +## Goals + +1. **Item 4 — Self-contained release notes.** Release notes carry each merged PR's *body content*, + not just its title and a link, so a reader never has to click through to know what changed. +2. **Item 5 — A production config as a release asset.** The GitHub Release ships a `config.yaml` + shaped for real use (real allowlist, sensible caps) in `mode: confirm` — ready for live, but + never live-on-download. The in-wheel dev template stays `mode: paper`. +3. **Item 6 — Seed + migration lifecycle.** A fresh deployment seeds the strategy (rules) library; + an existing database is migrated by a dedicated, idempotent `keel migrate` command; a manually + dispatchable GitHub workflow exercises/dispatches migration, structured for the server future + without pretending to migrate a database that does not yet exist. + +## Non-goals + +- No coupling of migration to a hosted/server database from release CI — there is no server DB + target yet (that arrives with workstream A's "hosted future"). We build the *plumbing and a + manual workflow*, not a live CI-against-prod step. +- No change to the money-safety model: seeded rules stay `candidate` (cannot trade until a + deliberate `rules promote`); the released config stays `mode: confirm`. +- No change to versioning, the build-identity stamp, or the wheel-by-path install guarantees. + +--- + +## Design + +### 3.1 Release notes from PR bodies (item 4) + +**Problem.** `release.yml`'s "Compose release notes" step calls the GitHub +`releases/generate-notes` API, which returns a flat/categorised list of `* by @author in +#N` — titles and links only. We want each PR's cleaned body inlined, grouped by the same labels. + +**Approach — a tested pure function + thin fetch glue.** The categorisation and body-cleaning is +logic worth testing; the GitHub fetching is I/O. Split them: + +- **`scripts/release_notes.py`** — a repo-level (not shipped in the wheel) module with a pure + function: + + ```python + def compose_release_notes(prs: list[PullRequest], *, categories: list[Category]) -> str: ... + ``` + + where `PullRequest` is a small dataclass `{number, title, body, labels}` and `Category` mirrors + `.github/release.yml` (`title`, `labels`, with `"*"` catch-all). It: + 1. Drops any PR labelled `norelease`. + 2. Assigns each PR to the **first** category whose labels intersect the PR's labels; unlabelled / + unmatched PRs fall to the `"*"` catch-all ("Other changes"), exactly like today's grouping. + 3. Renders, per category with ≥1 PR: an `##` section heading, then per PR `### <title> (#N)` + followed by the **cleaned body**. + 4. **Body cleaning** (`clean_pr_body`): strip the `🤖 Generated with [Claude Code]…` footer and + everything after it; strip `<!-- … -->` HTML comments; strip trailing `Co-Authored-By:` lines; + collapse 3+ blank lines to one; trim. An empty cleaned body renders as `_(no description)_`. + + A `__main__` guard reads a JSON array of PRs from stdin and prints the composed notes, so the + workflow can pipe to it. `.github/release.yml` stays the single source of truth for categories; + the workflow passes it (parsed) so the mapping is not duplicated in Python. + +- **`release.yml` change.** Replace the `generate-notes` call. Gather the PRs merged in + `PREV..HEAD`: + 1. `git log --format=%H <PREV>..HEAD` (or from start of history on the first release). + 2. For each commit, `gh api repos/${repo}/commits/<sha>/pulls -q '.[].number'`; collect the + unique PR numbers. + 3. For each number, `gh api repos/${repo}/pulls/<n> -q '{number,title,body,labels:[.labels[].name]}'`. + 4. Pipe the JSON array of PRs into `python scripts/release_notes.py`, which emits the grouped, + body-inlined markdown. + + The fixed preamble (build-from hash, install-by-path warning) is unchanged and still prepended. + +**Why a repo-level script, not a `keel/` module:** it is release tooling, never runtime; shipping +it in the wheel would bloat the artifact. Tests import it directly from `scripts/`. + +### 3.2 `config.yaml` as a live release asset (item 5) + +**Two config files, by intent.** + +- **`keel/templates/config.yaml`** (unchanged) — the **dev** template: `mode: paper`, shipped + inside the wheel (`pyproject` `artifacts = ["keel/templates/*.yaml"]`), written by + `keel init-config` / `keel init`. Paper mode places nothing. +- **`keel/templates/config.live.yaml`** (new, committed, reviewed) — the **production** template: + identical shape, but `auto_trade.mode: confirm`, real allowlist (`BTC`, `ETH`, `PAXG`), sensible + caps / `history_days`, and header comments naming exactly what to review before going live + (secrets in `.env`, caps, allowance, promoting rules). It is glob-matched by the existing + `artifacts` line, so it also ships in the wheel — enabling a local reproduction of the release + asset. + +**Release attachment.** In `release.yml`'s publish step, copy the live template to a +download-friendly name and attach it: + +``` +cp keel/templates/config.live.yaml config.yaml +gh release create "v<version>" dist/* config.yaml --title … --notes-file … +``` + +So the Release lists all wheels **plus** `config.yaml` (the production, confirm-mode config). + +**Local parity.** `keel init-config` gains a `--live` flag that writes `config.live.yaml`'s +contents instead of the dev template, so an operator can reproduce the exact release asset without +downloading it. Default (no flag) is unchanged (dev/paper). + +**Safety.** The production config is `mode: confirm` — the tool asks before every order. Going +autonomous remains a separate, deliberate edit (and, after workstream A, a tracked profile choice). +Nothing ships armed. A release-time check asserts the live template parses via `load_config` and is +`mode: confirm` (a red tripwire against accidentally committing an armed config). + +### 3.3 Seed + migration lifecycle (item 6) + +**Two distinct operations, never conflated:** + +- **Seed** = populate the strategy library on a *fresh* deployment. In this codebase the `rules` + table *is* the strategy library (`keel rules seed` inserts one `candidate` per (kind, product) + from each rule's constructor defaults). Seeding is first-run data, idempotent by (kind, + product_id). Seeded rules are `candidate` — they cannot trade until a deliberate `rules promote`. +- **Migrate** = evolve the *schema* of an *existing* database. `keel/data/db.py::migrate(conn)` + already does this incrementally against a `schema_version` table (`SCHEMA_VERSION = 6`), each step + guarded and idempotent. **Migration must never re-seed** — that would resurrect deleted/refuted + rules. + +**New: `keel migrate` command.** A thin, idempotent CLI wrapper over `db.migrate()`: +- Reads the current `schema_version` (0 if absent/fresh), calls `db.migrate(conn)`, reports + `migrated <from> -> <to>` (or `already at <SCHEMA_VERSION>, nothing to do`). +- No network, no authz gate, no seeding — schema only. Safe to run repeatedly and on a live DB. +- `--db` targets an explicit database path (defaults to the context `db_path`), so it can be + pointed at wherever the database lives — including a future server-mounted path. + +**Fresh-deploy seeding.** `keel init` already does `init-config` + `rules seed` (candidates). We +make the bootstrap explicit and release-documented: `keel init` first ensures the schema exists +(runs `db.migrate` on the fresh DB, which `_open_repo` already does on connect), then seeds the +candidate rule library. No behaviour change beyond documenting it as *the* fresh-deploy path in +`docs/RELEASING.md`. Seeding stays idempotent, so re-running `init` on an existing deployment +adds nothing. + +**Manual migration workflow — `.github/workflows/migrate.yml`.** `workflow_dispatch` only (never +on push/merge). Honest scaffold for the server future, useful today as a migration-integrity check: +- Input `db_path` (optional, default empty). +- Install via `uv sync`. +- **If `db_path` is provided** (future self-hosted-runner / server-mounted DB): run + `keel migrate --db "<db_path>"` and print the from→to report. +- **If `db_path` is empty** (today's default): run a **migration smoke test** — build a fresh DB + and a synthetic "old" DB (stamped at an earlier `schema_version`), run `db.migrate`, and assert + both reach `SCHEMA_VERSION`. This gives the workflow a real job now (catches a broken migration + chain) without inventing a production DB that does not exist. +- A comment in the file documents the deferred seam: release CI will call this with the server's + DB target once workstream A stands the server up. Release CI is **not** wired to it in this spec. + +--- + +## Components & interfaces (files touched) + +| File | Change | +|---|---| +| `scripts/release_notes.py` | **new** — pure `compose_release_notes` + `clean_pr_body`; `__main__` stdin→stdout glue | +| `tests/test_release_notes.py` | **new** — grouping, catch-all, `norelease` exclusion, body cleaning, empty-body | +| `.github/workflows/release.yml` | replace `generate-notes` step with PR-fetch + `scripts/release_notes.py`; attach `config.yaml`; add live-config parse/mode tripwire | +| `keel/templates/config.live.yaml` | **new** — production template, `mode: confirm`, real allowlist/caps, review-before-live header | +| `keel/cli.py` | `init-config --live` flag; **new** `keel migrate` command | +| `tests/test_cli_*.py` | `--live` writes the live template; `keel migrate` reports from→to and is idempotent | +| `.github/workflows/migrate.yml` | **new** — `workflow_dispatch`; `db_path` target or migration smoke test | +| `docs/RELEASING.md` | document the config asset, the fresh-deploy seed path, and `keel migrate` + the workflow | +| `keel/templates/config.yaml` | unchanged (stays dev/paper) | + +## Data flow + +- **Release notes:** `git log PREV..HEAD` → per-commit `gh api …/pulls` → unique PR numbers → per-PR + `gh api …/pulls/<n>` (title/body/labels) → JSON → `scripts/release_notes.py` → grouped markdown → + prepend preamble → `gh release create --notes-file`. +- **Config asset:** committed `config.live.yaml` → wheel (via `artifacts` glob) *and* → copied to + `config.yaml` → attached to the Release. +- **Fresh deploy:** `keel init` → `db.migrate` (fresh schema) + `rules seed` (candidate library) → + operator edits `config.yaml` + `.env`, promotes rules deliberately. +- **Existing deploy:** `keel migrate` (or the workflow) → `db.migrate` runs outstanding steps only. + +## Error handling & safety + +- **No red build ships:** the existing tests+ruff gate is unchanged; `scripts/release_notes.py` has + its own unit tests run by that gate. +- **Live-config tripwire:** the release asserts `config.live.yaml` parses and is `mode: confirm`; + a committed armed config fails the release loudly. +- **Migration never seeds; seeding never migrates schema beyond `db.migrate`'s idempotent DDL.** + `keel migrate` is safe on a live DB (idempotent, schema-only). Seeded rules are `candidate`. +- **Empty PR body** renders `_(no description)_` rather than a blank entry. +- **First release** (no previous tag) composes notes from the start of history, as today. + +## Testing + +- `test_release_notes.py`: category assignment incl. first-match-wins and `"*"` catch-all; + `norelease` dropped; footer/HTML-comment/`Co-Authored-By` stripping; blank-line collapse; + empty-body placeholder; a full end-to-end compose over a small fixture PR set. +- CLI tests: `init-config --live` writes the live template (asserts `mode: confirm`); `keel migrate` + on a fresh DB reports up-to-date, on a synthetic-old DB reports from→to and reaches + `SCHEMA_VERSION`, and is idempotent on a second run. +- The migrate workflow's smoke-test logic is exercised by the same synthetic-old-DB unit test, so CI + green ⇒ the workflow's default path is green. +- Manual/CI: full test suite + ruff stay green; the release workflow changes are validated on the + next real release cut (the notes composition can be dry-run locally by piping recorded PR JSON). + +## Future seams (explicitly deferred) + +- Release-CI-triggered migration against a **server** database (needs workstream A's hosting + + a DB target + secrets). `migrate.yml` is shaped to accept that target via `db_path`. +- Autonomous/profile mode (workstream A) will change `mode`'s vocabulary; the live config's + `mode: confirm` remains the safe default across that change. From 7100a877b4923456ef87a792c146cd97a29c2e06 Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim <eaitbrahim@gmail.com> Date: Tue, 21 Jul 2026 18:16:50 -0400 Subject: [PATCH 3/9] feat(release): compose release notes from PR bodies, not links Renders '### <title> (#N)' + the cleaned PR description, grouped by the categories in .github/release.yml (still the single source of truth). Strips the Claude Code footer, HTML comments and Co-Authored-By trailers. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../2026-07-21-release-packaging-bootstrap.md | 116 +++++++++++++ scripts/__init__.py | 1 + scripts/release_notes.py | 152 ++++++++++++++++++ tests/test_release_notes.py | 139 ++++++++++++++++ 4 files changed, 408 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-21-release-packaging-bootstrap.md create mode 100644 scripts/__init__.py create mode 100644 scripts/release_notes.py create mode 100644 tests/test_release_notes.py diff --git a/docs/superpowers/plans/2026-07-21-release-packaging-bootstrap.md b/docs/superpowers/plans/2026-07-21-release-packaging-bootstrap.md new file mode 100644 index 00000000..b170333d --- /dev/null +++ b/docs/superpowers/plans/2026-07-21-release-packaging-bootstrap.md @@ -0,0 +1,116 @@ +# Release packaging & bootstrap — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans or subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** Ship workstream B — self-contained PR-body release notes, a confirm-mode production `config.yaml` release asset, and a seed/migrate bootstrap lifecycle with a manually dispatchable migration workflow. + +**Architecture:** A tested pure module (`scripts/release_notes.py`) composes notes from PR JSON; `release.yml` fetches PRs and pipes to it, and attaches a committed `config.live.yaml`. A new `keel migrate` CLI wraps the existing idempotent `db.migrate`; `init-config --live` writes the production template; a `workflow_dispatch` `migrate.yml` targets a `db_path` or runs a migration smoke test. + +**Tech Stack:** Python 3.12, click, PyYAML (already a dep via config loading), pytest, GitHub Actions, `gh`/`jq`. + +## Global Constraints + +- Seeded rules stay `candidate`; production config stays `auto_trade.mode: confirm`. Nothing ships armed. +- `keel migrate` is schema-only and idempotent — it never seeds and never places orders. +- Release tooling (`scripts/*`) is NOT shipped in the wheel. +- Full suite green + `uv run ruff check keel tests packages` clean before each commit. +- `.github/release.yml` category order/labels are the single source of truth for grouping. + +--- + +### Task 1: `scripts/release_notes.py` — PR-body note composition + +**Files:** +- Create: `scripts/release_notes.py`, `scripts/__init__.py` +- Test: `tests/test_release_notes.py` + +**Interfaces:** +- Produces: `PullRequest(number:int, title:str, body:str, labels:tuple[str,...])`; `Category(title:str, labels:tuple[str,...])`; `clean_pr_body(body:str)->str`; `categorize(prs, categories)->list[tuple[Category,list[PullRequest]]]`; `compose_release_notes(prs, categories)->str`; `load_categories(path)->list[Category]`; `DEFAULT_CATEGORIES`. + +- [ ] **Step 1:** Write failing tests covering: `norelease` exclusion; first-match-wins category assignment; `"*"` catch-all; footer/HTML-comment/`Co-Authored-By` stripping; blank-line collapse; empty body → `_(no description)_`; `load_categories` against the real `.github/release.yml`. +- [ ] **Step 2:** Run `uv run pytest tests/test_release_notes.py -q` → FAIL (module missing). +- [ ] **Step 3:** Implement the module (dataclasses, regex cleaners, categorize, compose, `load_categories` via yaml, `__main__` reads a JSON array of PRs on stdin and prints notes using `load_categories(".github/release.yml")`). +- [ ] **Step 4:** Run tests → PASS; `ruff check`. +- [ ] **Step 5:** Commit `feat(release): compose release notes from PR bodies`. + +### Task 2: `keel migrate` command + +**Files:** +- Modify: `keel/cli.py` (new `migrate` command near `init`) +- Test: `tests/test_cli.py` (append) + +**Interfaces:** +- Consumes: `keel.data.db.{connect, migrate, SCHEMA_VERSION}`, `ctx.obj["db_path"]`. +- Produces: CLI `keel migrate [--db PATH]` printing `migrated <path>: schema <from> -> <to>` or `<path>: already at schema <n>, nothing to do`. + +- [ ] **Step 1:** Write failing tests: fresh DB → `0 -> SCHEMA_VERSION`; second run → `already at`; downgraded (`UPDATE schema_version SET version=1`) → `1 -> SCHEMA_VERSION`. +- [ ] **Step 2:** Run → FAIL. +- [ ] **Step 3:** Add `migrate_cmd` with a `_current_schema_version(conn)` helper (0 when the table is absent), `--db` defaulting to `ctx.obj["db_path"]`. +- [ ] **Step 4:** Run → PASS; ruff. +- [ ] **Step 5:** Commit `feat(cli): keel migrate -- idempotent schema-only migration`. + +### Task 3: `config.live.yaml` + `init-config --live` + +**Files:** +- Create: `keel/templates/config.live.yaml` +- Modify: `keel/cli.py` (`_template_config_text(live=False)`, `init-config --live`) +- Test: `tests/test_init_and_seed.py` (append) + +**Interfaces:** +- Produces: CLI `keel init-config --live` writes the production template; parses via `load_config` as `mode == "confirm"`. + +- [ ] **Step 1:** Write failing tests: `--live` writes a file that `load_config` reads as `auto_trade.mode == "confirm"`; default (no flag) stays `paper`; the shipped `config.live.yaml` parses and is `confirm`. +- [ ] **Step 2:** Run → FAIL. +- [ ] **Step 3:** Create `config.live.yaml` (dev template with a production header + `mode: confirm`); add `live` param to `_template_config_text` and a `--live` flag to `init-config`. +- [ ] **Step 4:** Run → PASS; ruff. +- [ ] **Step 5:** Commit `feat(cli): ship a confirm-mode production config template (--live)`. + +### Task 4: `scripts/migration_smoke.py` + +**Files:** +- Create: `scripts/migration_smoke.py` +- Test: `tests/test_release_notes.py` or a new `tests/test_migration_smoke.py` + +**Interfaces:** +- Produces: `main()` that asserts a fresh and a downgraded DB both reach `SCHEMA_VERSION`; exits 0 on success. + +- [ ] **Step 1:** Write a failing test importing `scripts.migration_smoke.main` and asserting it runs without raising. +- [ ] **Step 2:** Run → FAIL. +- [ ] **Step 3:** Implement `main()` (tempfile fresh DB → migrate → assert; downgrade to 1 → migrate → assert; cleanup). +- [ ] **Step 4:** Run → PASS; ruff. +- [ ] **Step 5:** Commit `feat(release): migration smoke test for the migrate workflow`. + +### Task 5: `release.yml` — PR-body notes + config asset + live tripwire + +**Files:** +- Modify: `.github/workflows/release.yml` + +- [ ] **Step 1:** Replace the "Compose release notes" step: derive `PREV`, collect unique PR numbers across `git log $RANGE` via `gh api commits/<sha>/pulls`, fetch each PR's `{number,title,body,labels}`, `jq -s` into an array, pipe to `python scripts/release_notes.py`; keep the fixed preamble. +- [ ] **Step 2:** Add a "Verify the live config asset" step: `uv run python` asserts `load_config("keel/templates/config.live.yaml").auto_trade.mode == "confirm"`, then `cp keel/templates/config.live.yaml config.yaml`. +- [ ] **Step 3:** Extend the publish step to attach `config.yaml`: `gh release create "v$V" dist/* config.yaml …`. +- [ ] **Step 4:** `actionlint` if available / manual YAML sanity (`python -c "import yaml,pathlib; yaml.safe_load(pathlib.Path('.github/workflows/release.yml').read_text())"`). +- [ ] **Step 5:** Commit `feat(release): inline PR bodies + attach the confirm-mode config asset`. + +### Task 6: `migrate.yml` workflow + `docs/RELEASING.md` + +**Files:** +- Create: `.github/workflows/migrate.yml` +- Modify: `docs/RELEASING.md` + +- [ ] **Step 1:** Create `migrate.yml`: `workflow_dispatch` with optional `db_path`; sync deps; if `db_path` set → `uv run keel migrate --db "$DB"`, else `uv run python scripts/migration_smoke.py`. Comment documents the deferred release-CI seam. +- [ ] **Step 2:** YAML sanity check both workflows. +- [ ] **Step 3:** Update `docs/RELEASING.md`: config-asset section, fresh-deploy seed path (`keel init`), `keel migrate` + the workflow, and PR-body notes note. +- [ ] **Step 4:** Commit `docs(release): config asset, seed/migrate lifecycle, PR-body notes`. + +### Task 7: Integration verification + +- [ ] **Step 1:** `uv run pytest -q` (full suite green, count up). +- [ ] **Step 2:** `uv run ruff check keel tests packages scripts` clean. +- [ ] **Step 3:** Dry-run notes locally: pipe a small hand-written PR JSON array to `scripts/release_notes.py` and eyeball the grouped, body-inlined output. +- [ ] **Step 4:** `uv run keel migrate` on a temp DB; `uv run keel init-config --live --config /tmp/live.yaml` then `load_config` it. + +## Self-Review + +- **Spec coverage:** item 4 → Tasks 1, 5; item 5 → Tasks 3, 5; item 6 → Tasks 2, 4, 6. All spec sections mapped. +- **Placeholders:** none — each task names exact files, functions, commands. +- **Type consistency:** `compose_release_notes(prs, categories)`, `clean_pr_body`, `_current_schema_version`, `_template_config_text(live=...)` used consistently across tasks. diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..8238caf0 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Repo-level release tooling. NOT shipped in the wheel -- see pyproject `artifacts`.""" diff --git a/scripts/release_notes.py b/scripts/release_notes.py new file mode 100644 index 00000000..961026a4 --- /dev/null +++ b/scripts/release_notes.py @@ -0,0 +1,152 @@ +"""Compose release notes that INLINE each merged PR's body, instead of linking to it. + +The release workflow previously called GitHub's `releases/generate-notes` API, which returns a +flat/categorised list of `* <title> by @author in #N` -- titles and links only, so a reader had +to click through every PR to learn what actually shipped. This module renders, per PR, a +`### <title> (#N)` heading followed by the PR's **cleaned** description. + +Grouping is driven by `.github/release.yml` (the single source of truth), so labels keep working +exactly as documented in `docs/RELEASING.md`: the first category whose labels intersect the PR's +labels wins, an unlabelled PR lands in the `"*"` catch-all, and `norelease` PRs are dropped. + +This is release tooling -- it is deliberately NOT part of the shipped `keel` package. + +Usage (from the release workflow): + + gh api ... | python scripts/release_notes.py > notes-body.md + +reading a JSON array of `{number, title, body, labels}` on stdin. +""" + +from __future__ import annotations + +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +import yaml + +#: PRs carrying this label never appear in the notes (mirrors `.github/release.yml`). +EXCLUDE_LABEL = "norelease" + +#: Rendered in place of an empty description, so an entry is never a silent blank. +NO_DESCRIPTION = "_(no description)_" + +# Everything from the Claude Code footer onward is tooling noise, not release content. +_FOOTER_MARKER = "🤖 Generated with" +_HTML_COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL) +_COAUTHOR_RE = re.compile(r"^[ \t]*Co-[Aa]uthored-[Bb]y:.*$", re.MULTILINE) +_MULTI_BLANK_RE = re.compile(r"\n{3,}") + + +@dataclass(frozen=True) +class PullRequest: + """One merged PR, as fetched from the GitHub API.""" + + number: int + title: str + body: str + labels: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Category: + """A notes section. `labels == ("*",)` marks the catch-all, which must come last.""" + + title: str + labels: tuple[str, ...] + + +def clean_pr_body(body: str | None) -> str: + """Strip tooling noise from a PR description. + + Removes the Claude Code footer (and everything after it), HTML comments, and + `Co-Authored-By:` trailers, then collapses runs of blank lines. Returns `""` for an + absent or whitespace-only body -- callers substitute `NO_DESCRIPTION`. + """ + if not body: + return "" + + text = body.replace("\r\n", "\n") + + marker = text.find(_FOOTER_MARKER) + if marker != -1: + text = text[:marker] + + text = _HTML_COMMENT_RE.sub("", text) + text = _COAUTHOR_RE.sub("", text) + text = _MULTI_BLANK_RE.sub("\n\n", text) + return text.strip() + + +def load_categories(path: str | Path = ".github/release.yml") -> list[Category]: + """Read the notes categories from `.github/release.yml`, preserving their order. + + Order matters twice: the first matching category wins, and the `"*"` catch-all is last so a + labelled PR never falls into it by accident. + """ + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} + categories = (raw.get("changelog") or {}).get("categories") or [] + return [ + Category(title=str(entry["title"]), labels=tuple(entry.get("labels") or ())) + for entry in categories + ] + + +def categorize( + prs: list[PullRequest], categories: list[Category] +) -> list[tuple[Category, list[PullRequest]]]: + """Bucket PRs into categories, dropping `norelease` and empty sections. + + Returns `(category, prs)` pairs in the category order given, omitting any category that + ended up empty. + """ + buckets: dict[str, list[PullRequest]] = {c.title: [] for c in categories} + + for pr in prs: + labels = set(pr.labels) + if EXCLUDE_LABEL in labels: + continue + for category in categories: + if "*" in category.labels or labels & set(category.labels): + buckets[category.title].append(pr) + break + + return [(c, buckets[c.title]) for c in categories if buckets[c.title]] + + +def compose_release_notes(prs: list[PullRequest], categories: list[Category]) -> str: + """Render the grouped, body-inlined change list as markdown.""" + sections: list[str] = [] + + for category, bucket in categorize(prs, categories): + lines = [f"## {category.title}", ""] + for pr in bucket: + body = clean_pr_body(pr.body) or NO_DESCRIPTION + lines += [f"### {pr.title} (#{pr.number})", "", body, ""] + sections.append("\n".join(lines).rstrip()) + + return "\n\n".join(sections) + + +def _pr_from_json(entry: dict) -> PullRequest: + return PullRequest( + number=int(entry["number"]), + title=str(entry.get("title") or ""), + body=str(entry.get("body") or ""), + labels=tuple(entry.get("labels") or ()), + ) + + +def main() -> None: + """Read a JSON array of PRs on stdin; print the composed notes on stdout.""" + payload = json.load(sys.stdin) or [] + prs = [_pr_from_json(entry) for entry in payload] + categories = load_categories(Path(__file__).resolve().parent.parent / ".github" / "release.yml") + sys.stdout.write(compose_release_notes(prs, categories)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_release_notes.py b/tests/test_release_notes.py new file mode 100644 index 00000000..3e2556da --- /dev/null +++ b/tests/test_release_notes.py @@ -0,0 +1,139 @@ +"""`scripts/release_notes.py` -- composing release notes from PR BODIES, not links. + +The release workflow used to call GitHub's `generate-notes` API, which emits +`* <title> by @author in #N` -- titles and links only. These notes inline each merged PR's +cleaned body so a reader never has to click through to know what shipped. + +Only the pure composition is tested here; fetching the PRs is `gh` glue in the workflow. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scripts.release_notes import ( # noqa: E402 + Category, + PullRequest, + clean_pr_body, + compose_release_notes, + load_categories, +) + +CATS = [ + Category(title="⚠️ Breaking changes", labels=("breaking",)), + Category(title="Features", labels=("feature", "enhancement")), + Category(title="Fixes", labels=("bug", "fix")), + Category(title="Other changes", labels=("*",)), +] + + +def _pr(number, title="A change", body="Body text.", labels=()): + return PullRequest(number=number, title=title, body=body, labels=tuple(labels)) + + +# -- body cleaning ------------------------------------------------------------- + + +def test_the_claude_code_footer_is_stripped_with_everything_after_it(): + body = "Real content.\n\n🤖 Generated with [Claude Code](https://claude.com/claude-code)\n\ntrailing" + assert clean_pr_body(body) == "Real content." + + +def test_html_comments_are_stripped(): + body = "Visible.\n<!-- a reviewer checklist nobody wants in the notes -->\nAlso visible." + cleaned = clean_pr_body(body) + assert "reviewer checklist" not in cleaned + assert "Visible." in cleaned and "Also visible." in cleaned + + +def test_co_authored_by_trailers_are_stripped(): + body = "Content.\n\nCo-Authored-By: Someone <a@b.c>" + assert "Co-Authored-By" not in clean_pr_body(body) + assert "Content." in clean_pr_body(body) + + +def test_runs_of_blank_lines_collapse(): + assert clean_pr_body("A.\n\n\n\n\nB.") == "A.\n\nB." + + +def test_an_empty_or_whitespace_body_cleans_to_empty_string(): + assert clean_pr_body("") == "" + assert clean_pr_body(" \n\n ") == "" + assert clean_pr_body(None) == "" + + +# -- categorisation ------------------------------------------------------------ + + +def test_norelease_prs_are_excluded_entirely(): + out = compose_release_notes([_pr(1, title="Hidden", labels=["norelease"])], CATS) + assert "Hidden" not in out + assert out == "" + + +def test_the_first_matching_category_wins(): + """A PR labelled both `feature` and `fix` belongs to Features -- it is listed first.""" + out = compose_release_notes([_pr(7, title="Dual", labels=["fix", "feature"])], CATS) + assert "## Features" in out + assert "## Fixes" not in out + + +def test_an_unlabelled_pr_falls_into_the_catch_all(): + out = compose_release_notes([_pr(9, title="Unlabelled")], CATS) + assert "## Other changes" in out + assert "### Unlabelled (#9)" in out + + +def test_empty_categories_are_omitted(): + out = compose_release_notes([_pr(3, labels=["feature"])], CATS) + assert "## Features" in out + assert "## Fixes" not in out + assert "Breaking" not in out + + +# -- composition --------------------------------------------------------------- + + +def test_the_pr_body_is_inlined_under_a_titled_heading(): + out = compose_release_notes( + [_pr(12, title="Add the thing", body="It does X.\nAnd Y.", labels=["feature"])], CATS + ) + assert "### Add the thing (#12)" in out + assert "It does X." in out + assert "And Y." in out + + +def test_a_pr_with_no_body_renders_a_placeholder_not_a_blank(): + out = compose_release_notes([_pr(5, title="Terse", body="", labels=["feature"])], CATS) + assert "### Terse (#5)" in out + assert "_(no description)_" in out + + +def test_prs_are_grouped_under_their_category_in_category_order(): + out = compose_release_notes( + [ + _pr(2, title="Fixed it", labels=["bug"]), + _pr(1, title="Built it", labels=["feature"]), + _pr(3, title="Broke it", labels=["breaking"]), + ], + CATS, + ) + assert out.index("Breaking changes") < out.index("## Features") < out.index("## Fixes") + + +# -- categories come from .github/release.yml ---------------------------------- + + +def test_load_categories_reads_the_repo_release_yml(): + """`.github/release.yml` stays the single source of truth for grouping.""" + cats = load_categories(REPO_ROOT / ".github" / "release.yml") + assert cats, "expected categories from .github/release.yml" + assert cats[0].title == "⚠️ Breaking changes" + assert cats[-1].labels == ("*",), "the catch-all must be last" + titles = [c.title for c in cats] + assert "Features" in titles and "Compliance & rails" in titles From e41b17a5e5e6671c91d5cfe76d7cdcd798d6d8cf Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim <eaitbrahim@gmail.com> Date: Tue, 21 Jul 2026 18:17:37 -0400 Subject: [PATCH 4/9] feat(cli): keel migrate -- idempotent, schema-only migration Counterpart to 'keel init': init bootstraps a FRESH deployment (config + candidate rule library); migrate evolves an EXISTING database's schema and never seeds, so deliberately deleted/refuted rules are not resurrected. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- keel/cli.py | 48 +++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/keel/cli.py b/keel/cli.py index 6f53d8e2..43e0ed85 100644 --- a/keel/cli.py +++ b/keel/cli.py @@ -320,6 +320,54 @@ def init_cmd(ctx: click.Context, config_path: str, force: bool) -> None: ctx.invoke(rules_seed, products=None, kinds=None, force=False, status="candidate") +# -- migrate (schema evolution for an EXISTING database) ------------------------------------- + + +def _current_schema_version(conn: Any) -> int: + """The stored schema version, or 0 when the database has no schema at all yet.""" + present = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'" + ).fetchone() + if present is None: + return 0 + row = conn.execute("SELECT version FROM schema_version").fetchone() + return int(row["version"]) if row is not None else 0 + + +@cli.command("migrate") +@click.option( + "--db", + "db_override", + default=None, + help="Database file to migrate (default: the global --db / keel.db).", +) +@click.pass_context +def migrate_cmd(ctx: click.Context, db_override: str | None) -> None: + """Apply outstanding schema migrations to an existing database (idempotent, schema-only). + + This is the counterpart to `keel init`, and the two are deliberately NOT the same thing: + + * `keel init` bootstraps a FRESH deployment -- it writes a config and seeds the strategy + (rules) library as `candidate`s. + * `keel migrate` evolves the SCHEMA of an EXISTING database and **never seeds**. Re-seeding + on migrate would resurrect rules that were deliberately deleted or refuted. + + Runs `keel.data.db.migrate`, which steps the stored `schema_version` up incrementally and is + safe to call repeatedly. No network, no authz gate, no orders -- safe against a live database. + """ + path = db_override or ctx.obj["db_path"] + conn = connect(path) + + before = _current_schema_version(conn) + migrate(conn) + after = _current_schema_version(conn) + + if after > before: + click.echo(f"migrated {path}: schema {before} -> {after}") + else: + click.echo(f"{path}: already at schema {after}, nothing to do") + + # -- db import ------------------------------------------------------------------------------ diff --git a/tests/test_cli.py b/tests/test_cli.py index a5b6b7b9..98fa2245 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1165,3 +1165,61 @@ def test_record_flow_rejects_a_non_finite_amount(tmp_path): assert result.exit_code != 0 assert _repo_at(db_path).get_state("equity_high_water_mark") == Decimal("10000") + + +# -- keel migrate (schema-only, idempotent) ------------------------------------ + + +def test_migrate_brings_a_fresh_db_up_to_head(tmp_path): + """A fresh file has no schema at all: migrate must create it and report 0 -> HEAD.""" + from keel.data.db import SCHEMA_VERSION + + db = tmp_path / "fresh.db" + result = CliRunner().invoke(cli, ["--db", str(db), "migrate"]) + assert result.exit_code == 0, result.output + assert f"0 -> {SCHEMA_VERSION}" in result.output + conn = connect(str(db)) + row = conn.execute("SELECT version FROM schema_version").fetchone() + assert int(row["version"]) == SCHEMA_VERSION + + +def test_migrate_is_idempotent(tmp_path): + """Running it twice must be safe -- it is the command CI/an operator re-runs.""" + db = tmp_path / "twice.db" + CliRunner().invoke(cli, ["--db", str(db), "migrate"]) + again = CliRunner().invoke(cli, ["--db", str(db), "migrate"]) + assert again.exit_code == 0, again.output + assert "nothing to do" in again.output + + +def test_migrate_advances_a_downgraded_db(tmp_path): + """The real job: an existing DB stamped below HEAD is stepped up to HEAD.""" + from keel.data.db import SCHEMA_VERSION + + db = tmp_path / "old.db" + conn = connect(str(db)) + migrate(conn) + conn.execute("UPDATE schema_version SET version = 1") + conn.commit() + + result = CliRunner().invoke(cli, ["--db", str(db), "migrate"]) + assert result.exit_code == 0, result.output + assert f"1 -> {SCHEMA_VERSION}" in result.output + check = connect(str(db)) + assert int(check.execute("SELECT version FROM schema_version").fetchone()["version"]) == ( + SCHEMA_VERSION + ) + + +def test_migrate_honours_an_explicit_db_option(tmp_path): + """--db targets a database directly, so it can point at wherever the DB lives.""" + from keel.data.db import SCHEMA_VERSION + + target = tmp_path / "explicit.db" + result = CliRunner().invoke(cli, ["migrate", "--db", str(target)]) + assert result.exit_code == 0, result.output + assert str(target) in result.output + conn = connect(str(target)) + assert int(conn.execute("SELECT version FROM schema_version").fetchone()["version"]) == ( + SCHEMA_VERSION + ) From 1ad73c5ec1165def5927e04054e86237a63038bc Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim <eaitbrahim@gmail.com> Date: Tue, 21 Jul 2026 18:18:43 -0400 Subject: [PATCH 5/9] feat(cli): ship a confirm-mode production config template (--live) keel/templates/config.live.yaml is the config.yaml attached to a Release: real allowlist/caps in mode: confirm, so it is ready for live use but never trades unattended off a fresh download. Dev template stays mode: paper. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- keel/cli.py | 31 ++++++-- keel/templates/config.live.yaml | 130 ++++++++++++++++++++++++++++++++ tests/test_init_and_seed.py | 45 +++++++++++ 3 files changed, 199 insertions(+), 7 deletions(-) create mode 100644 keel/templates/config.live.yaml diff --git a/keel/cli.py b/keel/cli.py index 43e0ed85..04b37ba2 100644 --- a/keel/cli.py +++ b/keel/cli.py @@ -276,11 +276,17 @@ def cli( # -- init (scaffold a working directory) ---------------------------------------------------- -def _template_config_text() -> str: - """The default config.yaml shipped inside the wheel (see pyproject `artifacts`).""" +def _template_config_text(live: bool = False) -> str: + """A config.yaml template shipped inside the wheel (see pyproject `artifacts`). + + `live=False` returns the dev template (`mode: paper` -- places nothing). `live=True` returns + the production template (`mode: confirm` -- previews every order and waits for approval), + which is also the `config.yaml` attached to a GitHub Release. + """ from importlib.resources import files - return (files("keel.templates") / "config.yaml").read_text(encoding="utf-8") + name = "config.live.yaml" if live else "config.yaml" + return (files("keel.templates") / name).read_text(encoding="utf-8") @cli.command("init-config") @@ -289,17 +295,28 @@ def _template_config_text() -> str: help="Where to write the config file.", ) @click.option("--force", is_flag=True, default=False, help="Overwrite an existing config.") -def init_config(config_path: str, force: bool) -> None: +@click.option( + "--live", + is_flag=True, + default=False, + help="Write the PRODUCTION template (mode: confirm) instead of the dev one (mode: paper).", +) +def init_config(config_path: str, force: bool, live: bool) -> None: """Write a default `config.yaml` into the current directory, ready to edit. - The installed wheel ships this template so a fresh working directory has a config to start + The installed wheel ships both templates so a fresh working directory has a config to start from -- edit `allowlist`, `caps`, and `auto_trade.mode` before running anything live. + + `--live` writes the same production config that is attached to a GitHub Release: real + allowlist/caps in `mode: confirm`, which previews every order and waits for your approval. + Without it you get the dev template in `mode: paper`, which places nothing at all. """ path = Path(config_path) if path.exists() and not force: raise click.ClickException(f"{path} already exists; pass --force to overwrite") - path.write_text(_template_config_text(), encoding="utf-8") - click.echo(f"wrote {path}. Edit allowlist/caps/auto_trade.mode before going live.") + path.write_text(_template_config_text(live=live), encoding="utf-8") + which = "production/confirm" if live else "dev/paper" + click.echo(f"wrote {path} [{which}]. Review allowlist/caps/auto_trade before going live.") @cli.command("init") diff --git a/keel/templates/config.live.yaml b/keel/templates/config.live.yaml new file mode 100644 index 00000000..a1840e98 --- /dev/null +++ b/keel/templates/config.live.yaml @@ -0,0 +1,130 @@ +# keel PRODUCTION configuration -- the `config.yaml` attached to a GitHub Release. +# +# Shipped in `auto_trade.mode: confirm`: keel previews every order and waits for your explicit +# approval before placing it. It is ready for live use, but it will never trade unattended off a +# fresh download -- that is a separate, deliberate decision. +# +# REVIEW BEFORE GOING LIVE: +# 1. `.env` -- CDP_API_KEY / CDP_API_SECRET for your Coinbase Advanced Trade key. +# 2. `allowlist` -- only assets you have screened and attested (`keel assets screen|attest`). +# 3. `caps` -- max_exposure_usd / max_per_asset_pct are your risk ceilings. +# 4. `subscription` -- attest your venue tier (`keel subscription attest`); rail 14 caps spend. +# 5. rules -- `keel init` seeds them as `candidate`; nothing trades until you promote. +# 6. `auto_trade.enabled` -- still false; set it true to start the scheduled loop. +# +# `allowlist` and `caps` are required and validated by keel.config.load_config; missing or +# invalid values raise ConfigError naming the offending key rather than silently defaulting. + +allowlist: + - BTC + - ETH + - PAXG + +target_weights: + BTC: 0.40 + ETH: 0.30 + PAXG: 0.30 + +risk_pct: 0.01 + +caps: + # max_per_order_usd / max_per_day_usd are OPTIONAL internal RISK limits, not real Coinbase + # limits -- Coinbase One's only subscription constraint is monthly fee-free trading VOLUME + # (see `subscription:` below). They default to a non-binding $1B when omitted (Issue #85); + # set explicit values here only if you want an extra per-order/per-day risk ceiling tighter + # than the exposure/concentration caps below. Left at their non-binding default here so + # risk-sized rule orders ($400-24k typical) aren't silently rejected. + max_exposure_usd: 5000 + max_per_asset_pct: 0.50 + +market_data: + granularities: + - ONE_DAY + - ONE_HOUR + - FIFTEEN_MINUTE + history_days: 365 + +auto_trade: + # confirm = preview + explicit approval for every order (the safe live default). + mode: confirm + enabled: false + interval_sec: 900 + # how long a `keel arm-bypass` token stays valid (seconds) before bypass mode requires + # re-arming (Issue #60) -- `keel agent --bypass` still needs a fresh armed token even with + # the CLI passphrase gate satisfied. + bypass_arm_ttl_sec: 3600 + +promotion: + min_trades: 100 + min_expectancy: 0.0 + min_rr: 1.5 + min_win_rate: 0.55 + +money_mgmt: + profit_trigger_pct: 0.10 + acceleration_pct: 0.05 + max_total_dd_pct: 0.20 + max_weekly_dd_pct: 0.08 + # Rail 16 (consecutive-loss breaker) — DISABLED by default (0 = off). + # Set from a backtest sweep, and set it ABOVE the strategy's tested max losing streak: + # turtle_breakout's max streak is 5, so a threshold of 3 would fire on normal variance. + max_consecutive_losses: 0 + streak_cooloff_days: 0 + +dca: + budget_usd: 50 + cadence_days: 7 + +# quote-currency the executor draws BUY notional from (rail 13, USDC-funding) -- never bank/ACH. +quote_currency: USDC + +subscription: + # The SIMULATOR's assumed fee-free monthly volume. The LIVE rail-14 cap is not set here -- + # it comes from the attested record: `keel subscription attest --venue coinbase --tier <t>`. + assumed_free_volume_usd: 500 + # What rail 14 permits on a venue that is unattested, suspect, lapsed, or overdue. + # 0 means such a venue cannot buy at all until it is attested. + unsubscribed_allowance_usd: 0 + pacing: opportunistic # opportunistic (monthly cap only) | even_daily (also paces per business day) + +# Coinbase One subscription tiers (Issue #86) -- fee-free monthly TRADING VOLUME (buys + sells) +# allowance per tier, used by `keel simulate`'s tier/fee analysis matrix to compare staying +# within a tier's free volume (throttled, 0 trading fees, but you still pay the subscription) +# against trading freely and paying the taker fee on volume EXCEEDING it. +# free_volume_usd: null means unlimited (Premium -- always fee-free, no cap to exceed). +tiers: + - name: Basic + free_volume_usd: 500 + subscription_usd_month: 4.99 + - name: Preferred + free_volume_usd: 10000 + subscription_usd_month: 29.99 + - name: Premium + free_volume_usd: null + subscription_usd_month: 299.99 + +# Coinbase Advanced trading fees applied to volume beyond a tier's free allowance, for a +# <$1k-30d-volume account (Coinbase's published fee schedule). taker_pct is the sim's default -- +# it fills market-style at next-bar open; maker_pct is exposed for a caller that wants to model +# limit-order fills instead. +fees: + taker_pct: 0.012 + maker_pct: 0.006 + +# Engine-activity logging. verbose=false (default) means only errors/exceptions are ever logged +# (the "keel" logger stays at ERROR level); set verbose: true (or `keel -v`) to also log major +# operations/decisions (INFO level) -- cycle starts, signals, guard vetoes, order outcomes, etc. +# file_count is the TOTAL number of files kept (the active log + rotated backups), each capped +# at max_file_mb. +logging: + verbose: false + file: logs/keel.log + max_file_mb: 25 + file_count: 5 + +# G4 overfitting gate (KB §78). NEVER tune these to obtain a desired verdict -- doing so is +# the exact Strathern misuse the gate exists to prevent (§78.7). slope_floor is calibrated +# from §78.8's worked cases: real strategy -0.35, pure random walk -0.61, overfit -0.75. +research: + pbo_max: 0.05 + slope_floor: -0.5 diff --git a/tests/test_init_and_seed.py b/tests/test_init_and_seed.py index 7516b6dd..58c5a0f5 100644 --- a/tests/test_init_and_seed.py +++ b/tests/test_init_and_seed.py @@ -109,3 +109,48 @@ def test_seed_rejects_an_unknown_status(tmp_path): cli, ["--db", str(tmp_path / "t.db"), "rules", "seed", "--status", "bogus"] ) assert result.exit_code != 0 + + +# -- the LIVE (production) template -------------------------------------------- + + +def test_the_live_template_is_a_VALID_config(tmp_path): + p = tmp_path / "live.yaml" + p.write_text(_template_config_text(live=True)) + load_config(str(p)) # raises ConfigError if invalid + + +def test_the_live_template_is_CONFIRM_mode_never_armed(tmp_path): + """The release asset must ask before every order. This is the tripwire against + ever shipping a config that trades unattended straight off a download.""" + p = tmp_path / "live.yaml" + p.write_text(_template_config_text(live=True)) + assert load_config(str(p)).auto_trade.mode == "confirm" + + +def test_the_dev_template_stays_paper(): + p = Path(__file__).resolve().parent.parent / "config.yaml" + assert load_config(str(p)).auto_trade.mode == "paper" + + +def test_the_two_templates_have_the_same_top_level_keys(): + """Catches drift: a key added to the dev config must reach the live one too.""" + import yaml + + dev = yaml.safe_load(_template_config_text()) + live = yaml.safe_load(_template_config_text(live=True)) + assert set(dev) == set(live), "live/dev config templates have drifted apart" + + +def test_init_config_live_writes_the_confirm_mode_template(tmp_path): + out = tmp_path / "config.yaml" + result = CliRunner().invoke(cli, ["init-config", "--config", str(out), "--live"]) + assert result.exit_code == 0, result.output + assert load_config(str(out)).auto_trade.mode == "confirm" + + +def test_init_config_without_live_still_writes_the_paper_template(tmp_path): + out = tmp_path / "config.yaml" + result = CliRunner().invoke(cli, ["init-config", "--config", str(out)]) + assert result.exit_code == 0, result.output + assert load_config(str(out)).auto_trade.mode == "paper" From 9b5f72e66ba3513121bf4dbff6f33f5467cecdd0 Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim <eaitbrahim@gmail.com> Date: Tue, 21 Jul 2026 18:19:07 -0400 Subject: [PATCH 6/9] feat(release): migration smoke test for the migrate workflow Verifies a fresh DB and a DB stamped below HEAD both reach SCHEMA_VERSION. Gives the manual migrate workflow a real job until a hosted DB exists. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- scripts/migration_smoke.py | 51 +++++++++++++++++++++++++++++++++++ tests/test_migration_smoke.py | 20 ++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 scripts/migration_smoke.py create mode 100644 tests/test_migration_smoke.py diff --git a/scripts/migration_smoke.py b/scripts/migration_smoke.py new file mode 100644 index 00000000..2607b4ef --- /dev/null +++ b/scripts/migration_smoke.py @@ -0,0 +1,51 @@ +"""Prove the migration chain reaches HEAD from both a fresh and a downgraded database. + +This is the default job of `.github/workflows/migrate.yml`: until a hosted database exists to +point the workflow at, the useful thing it can do on demand is verify that the migration chain +in `keel.data.db` is not broken -- a fresh DB lands on `SCHEMA_VERSION`, and a DB stamped below +HEAD is stepped all the way up to it. + +Release tooling: deliberately NOT shipped in the wheel. +""" + +from __future__ import annotations + +import os +import sqlite3 +import tempfile + +from keel.data.db import SCHEMA_VERSION, connect, migrate + + +def _version(conn: sqlite3.Connection) -> int: + return int(conn.execute("SELECT version FROM schema_version").fetchone()["version"]) + + +def main() -> None: + """Raise `AssertionError` if the migration chain does not reach `SCHEMA_VERSION`.""" + handle, path = tempfile.mkstemp(suffix=".db") + os.close(handle) + try: + conn = connect(path) + + migrate(conn) + fresh = _version(conn) + assert fresh == SCHEMA_VERSION, f"fresh DB stopped at schema {fresh}, want {SCHEMA_VERSION}" + + # Stamp it back to the oldest version and prove every step re-runs cleanly. + conn.execute("UPDATE schema_version SET version = 1") + conn.commit() + migrate(conn) + upgraded = _version(conn) + assert upgraded == SCHEMA_VERSION, ( + f"downgraded DB stopped at schema {upgraded}, want {SCHEMA_VERSION}" + ) + + conn.close() + print(f"migration smoke test OK: fresh and downgraded both reach schema {SCHEMA_VERSION}") + finally: + os.unlink(path) + + +if __name__ == "__main__": + main() diff --git a/tests/test_migration_smoke.py b/tests/test_migration_smoke.py new file mode 100644 index 00000000..68221e53 --- /dev/null +++ b/tests/test_migration_smoke.py @@ -0,0 +1,20 @@ +"""`scripts/migration_smoke.py` -- the default job of the manual migrate workflow. + +Running it here means CI green implies the workflow's no-target path is green too. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scripts.migration_smoke import main # noqa: E402 + + +def test_the_migration_chain_reaches_head(capsys): + main() # raises AssertionError if a fresh or downgraded DB stops short of SCHEMA_VERSION + assert "migration smoke test OK" in capsys.readouterr().out From 80b0a471d49adc22d335e66ed31dd74771d178ee Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim <eaitbrahim@gmail.com> Date: Tue, 21 Jul 2026 18:21:11 -0400 Subject: [PATCH 7/9] feat(release): inline PR bodies, attach the config asset, add migrate workflow - release.yml composes notes via scripts/release_notes.py (PR bodies, not links) - verifies keel/templates/config.live.yaml is mode: confirm, then attaches it to the Release as config.yaml -- an armed config fails the release loudly - migrate.yml: manual-only; migrates a given db_path, else verifies the migration chain. CI has no DB to reach until the app is server-hosted. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .github/workflows/migrate.yml | 59 +++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 56 ++++++++++++++++++++++++++++----- docs/RELEASING.md | 55 ++++++++++++++++++++++++++++---- 3 files changed, 156 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/migrate.yml diff --git a/.github/workflows/migrate.yml b/.github/workflows/migrate.yml new file mode 100644 index 00000000..a3707f6e --- /dev/null +++ b/.github/workflows/migrate.yml @@ -0,0 +1,59 @@ +name: Migrate database + +# MANUAL ONLY. Schema migration for an EXISTING database -- never runs on push or merge. +# +# Migration is deliberately separate from seeding: `keel init` seeds the strategy (rules) library +# on a FRESH deployment, while `keel migrate` only evolves an existing database's schema. Seeding +# on migrate would resurrect rules that were deliberately deleted or refuted. +# +# DEFERRED SEAM: today `keel.db` is local, git-ignored and single-user, so CI has no database to +# reach. Once the app is server-hosted, that deployment's database becomes the `db_path` target +# (via a self-hosted runner or a mounted volume) and the release workflow can call this job. Until +# then, dispatching this with no target runs a migration-integrity check instead of pretending to +# migrate something that is not there. +on: + workflow_dispatch: + inputs: + db_path: + description: "Database to migrate. Leave empty to run the migration smoke test instead." + required: false + type: string + default: "" + +permissions: + contents: read + +jobs: + migrate: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.12 + + - name: Sync dependencies + run: uv sync --all-extras --dev + + - name: Migrate, or smoke-test the migration chain + run: | + set -euo pipefail + DB="${{ inputs.db_path }}" + if [ -n "$DB" ]; then + if [ ! -f "$DB" ]; then + echo "::error::no database at '$DB' -- refusing to create one here." + echo "::error::A fresh deployment is bootstrapped with 'keel init', not this workflow." + exit 1 + fi + echo "migrating $DB" + uv run keel migrate --db "$DB" + else + echo "no db_path given -- verifying the migration chain instead" + uv run python scripts/migration_smoke.py + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e3d3945b..81671b7d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -98,6 +98,15 @@ jobs: printf '%s' "$OUT" | grep -q "DIRTY" && { echo "::error::artifact reports a dirty tree"; exit 1; } || true + # The release ships a ready-for-live config as a downloadable asset. It must be in + # `confirm` mode: a config that trades unattended straight off a download is exactly what + # this project refuses to ship. Fail the release loudly rather than publish an armed config. + - name: Verify and stage the live config asset + run: | + set -euo pipefail + uv run python -c "from keel.config import load_config; m = load_config('keel/templates/config.live.yaml').auto_trade.mode; assert m == 'confirm', f'live config must be confirm mode, got {m!r}'; print('live config OK: mode=confirm')" + cp keel/templates/config.live.yaml config.yaml + - name: Tag run: | set -euo pipefail @@ -111,17 +120,35 @@ jobs: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - # Auto-generate the change list from merged PRs since the previous tag, categorised by - # .github/release.yml. The tag already exists (previous step), so the API can range on it. + # The change list INLINES each merged PR's body -- a reader should never have to click + # through to a PR to learn what shipped. Grouping still comes from .github/release.yml. PREV="$(git describe --tags --abbrev=0 "v${{ inputs.version }}^" 2>/dev/null || true)" if [ -n "$PREV" ]; then - GENERATED="$(gh api "repos/${{ github.repository }}/releases/generate-notes" \ - -f tag_name="v${{ inputs.version }}" -f previous_tag_name="$PREV" -q .body)" + RANGE="$PREV..HEAD" else - # First release: no previous tag, generate from the start of history. - GENERATED="$(gh api "repos/${{ github.repository }}/releases/generate-notes" \ - -f tag_name="v${{ inputs.version }}" -q .body)" + # First release: no previous tag, so walk from the start of history. + RANGE="HEAD" fi + + # Every PR whose commits land in this range, de-duplicated. + : > /tmp/pr-numbers.txt + for sha in $(git log --format=%H "$RANGE"); do + gh api "repos/${{ github.repository }}/commits/$sha/pulls" \ + -q '.[].number' 2>/dev/null >> /tmp/pr-numbers.txt || true + done + sort -u -n /tmp/pr-numbers.txt -o /tmp/pr-numbers.txt + echo "found $(wc -l < /tmp/pr-numbers.txt) PRs in $RANGE" + + # Fetch each PR's title/body/labels, then compose. jq -s folds the stream into an array. + : > /tmp/prs.ndjson + while read -r n; do + [ -n "$n" ] || continue + gh api "repos/${{ github.repository }}/pulls/$n" \ + -q '{number:.number,title:.title,body:(.body // ""),labels:[.labels[].name]}' \ + >> /tmp/prs.ndjson + done < /tmp/pr-numbers.txt + jq -s '.' /tmp/prs.ndjson > /tmp/prs.json + GENERATED="$(uv run python scripts/release_notes.py < /tmp/prs.json)" { echo "Built from $(git rev-parse --short=12 HEAD). Version binds to this hash:" echo "\`keel --version\` reports \`keel ${{ inputs.version }}+$(git rev-parse --short=12 HEAD) [release]\`." @@ -139,6 +166,18 @@ jobs: echo "someone else's package. A build reporting **DIRTY** or **[checkout]** is not this" echo "release and must not be run against live funds." echo + echo "## Configure" + echo + echo "\`config.yaml\` is attached to this release: the production config, in" + echo "\`auto_trade.mode: confirm\` — keel previews every order and waits for your" + echo "approval. Drop it beside the install (or run \`keel init-config --live\`), put" + echo "your CDP key in a git-ignored \`.env\`, then:" + echo + echo ' keel migrate # existing database: apply schema migrations' + echo ' keel init # fresh deployment: write config + seed candidate rules' + echo + echo "Seeded rules start as \`candidate\` and trade nothing until you promote them." + echo echo "$GENERATED" } > /tmp/release-notes.md echo "composed $(wc -l < /tmp/release-notes.md) lines of notes" @@ -147,7 +186,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - gh release create "v${{ inputs.version }}" dist/* \ + # dist/* = every workspace wheel; config.yaml = the confirm-mode production config. + gh release create "v${{ inputs.version }}" dist/* config.yaml \ --title "keel v${{ inputs.version }}" \ --notes-file /tmp/release-notes.md echo "published v${{ inputs.version }}" diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 036190ab..d5a010e9 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -28,15 +28,58 @@ against live funds**; `keel --version` warns loudly when so. 3. The workflow: validates the input is semver and matches `pyproject.toml` and no such tag exists → runs tests + ruff → stamps the commit into `keel/_build_info.py` → `uv build --all-packages` → installs the wheel into a clean venv **by path** and asserts it self-identifies as a clean - `[release]` → tags `v<version>` → composes release notes → publishes the GitHub Release with all - wheels attached. + `[release]` → verifies the live config asset is `mode: confirm` → tags `v<version>` → composes + release notes → publishes the GitHub Release with all wheels **and `config.yaml`** attached. + +## Release assets + +| asset | what it is | +|---|---| +| `keel_trader-<version>-py3-none-any.whl` | the CLI. Install **by path**, never by bare name. | +| `keel_core-*`, `keel_broker_*` wheels | workspace members `keel` depends on; download them all. | +| `config.yaml` | the **production** config: real allowlist/caps in `auto_trade.mode: confirm`. | + +`config.yaml` is `keel/templates/config.live.yaml`, committed and reviewed like any other code. +It ships in **confirm** mode — keel previews every order and waits for your approval — so a fresh +download is ready for live use but can never trade unattended. The release **fails loudly** if +that file is ever anything other than `mode: confirm`. + +Both templates also ship inside the wheel: `keel init-config` writes the dev one (`mode: paper`, +places nothing) and `keel init-config --live` writes the exact release asset. + +## Bootstrapping a deployment + +Seeding and migrating are deliberately **separate** operations: + +``` +keel init # FRESH deployment: write config.yaml + seed the strategy (rules) library +keel migrate # EXISTING database: apply outstanding schema migrations. Never seeds. +``` + +- **`keel init`** = `init-config` + `rules seed`. Rules are seeded as `candidate`, so nothing + trades until you deliberately `keel rules promote` them. +- **`keel migrate`** is idempotent and schema-only — safe to re-run, and safe against a live + database. It never re-seeds, because that would resurrect rules deliberately deleted or refuted. + `--db <path>` targets a database directly. + +The **Migrate database** workflow (Actions → Migrate database → Run workflow) is manual-only. Give +it a `db_path` to migrate that database; leave it empty and it verifies the migration chain +instead (a fresh DB and a downgraded DB both reach `SCHEMA_VERSION`). CI has no database to reach +while `keel.db` is local and git-ignored — once the app is server-hosted, that deployment's +database becomes the `db_path` target and the release can call this job. ## Release notes come from PRs -The change list in each release is **auto-generated from the PRs merged since the previous tag** -(`.github/release.yml`). The unit is the **pull request** — a clear PR title is all that is needed -for a useful entry. Issues and issue↔commit linking are **not** required and are not enforced: -good PRs are the source. +The change list in each release is **auto-generated from the PRs merged since the previous tag**. +The unit is the **pull request**. Issues and issue↔commit linking are **not** required and are not +enforced: good PRs are the source. + +Each entry **inlines the PR's description**, not a link to it — a reader should never have to +click through to learn what shipped. So the PR body *is* the release note: write it for someone +reading the release page. `scripts/release_notes.py` composes them (unit-tested in +`tests/test_release_notes.py`), stripping the Claude Code footer, HTML comments and +`Co-Authored-By:` trailers. A PR with an empty body renders as `_(no description)_` — visible, +so it gets fixed. Labels are **optional** and only affect grouping. Without them the notes are a flat "What's Changed" list of PR titles, which is fine. With them, PRs are grouped into sections: From e50ec117c59528b37f9fb36647b32fcb61b4fde6 Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim <eaitbrahim@gmail.com> Date: Tue, 21 Jul 2026 18:28:47 -0400 Subject: [PATCH 8/9] ci: map the pre-existing 'documentation' label into the docs category The categories referenced 11 labels that did not exist in the repo; they are now created. 'documentation' already existed, so it is treated as a synonym of 'docs' rather than duplicated -- the same pattern as feature/enhancement. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .github/release.yml | 2 +- docs/RELEASING.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/release.yml b/.github/release.yml index 0878d1a1..6f80b488 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -18,7 +18,7 @@ changelog: - title: Research & validation labels: [research, experiment] - title: Docs, CI & tooling - labels: [docs, ci, tooling] + labels: [docs, documentation, ci, tooling] # Catch-all LAST so a labelled PR never lands here by accident. - title: Other changes labels: ["*"] diff --git a/docs/RELEASING.md b/docs/RELEASING.md index d5a010e9..86c6c689 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -90,7 +90,7 @@ Changed" list of PR titles, which is fine. With them, PRs are grouped into secti | `bug`, `fix` | Fixes | | `compliance`, `rails` | Compliance & rails | | `research`, `experiment` | Research & validation | -| `docs`, `ci`, `tooling` | Docs, CI & tooling | +| `docs`, `documentation`, `ci`, `tooling` | Docs, CI & tooling | | `breaking` | ⚠️ Breaking changes | | `norelease` | *excluded from notes* | From 127beb0d4fa952210d79e12ff182969d4e5f660c Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim <eaitbrahim@gmail.com> Date: Tue, 21 Jul 2026 18:35:54 -0400 Subject: [PATCH 9/9] fix(release): demote PR-body headings and bound the notes to GitHub's limit Both found by composing the REAL 70 merged PRs, not by unit tests: 1. PR bodies carry their own '##' headings, which rendered as siblings of the category headings and flattened the outline. Headings in a body are now shifted so the shallowest becomes h4, preserving relative depth, capped at h6, and ignoring '#' inside fenced code blocks. 2. The composed notes were 168,066 chars against GitHub's 125,000 limit -- 'gh release create' would have rejected the release outright. The budget is now shared across entries so EVERY PR stays listed and only bodies give ground; truncation cuts at a paragraph boundary and re-closes an orphaned code fence. Real output is now 98,718 chars with all 70 entries. Truncation only binds on this first all-history release: at <=20 PRs per release the budget exceeds the largest body (3,686 chars) and nothing is cut. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- scripts/release_notes.py | 127 ++++++++++++++++++++++++++++++++++-- tests/test_release_notes.py | 82 +++++++++++++++++++++++ 2 files changed, 202 insertions(+), 7 deletions(-) diff --git a/scripts/release_notes.py b/scripts/release_notes.py index 961026a4..74fcd3d1 100644 --- a/scripts/release_notes.py +++ b/scripts/release_notes.py @@ -40,6 +40,19 @@ _COAUTHOR_RE = re.compile(r"^[ \t]*Co-[Aa]uthored-[Bb]y:.*$", re.MULTILINE) _MULTI_BLANK_RE = re.compile(r"\n{3,}") +_FENCE_RE = re.compile(r"^\s*(?:```|~~~)") +_HEADING_RE = re.compile(r"^(#{1,6})(\s+)(.*)$") + +#: GitHub rejects a release body longer than this. The real first release composed 168k +#: characters of PR bodies, which `gh release create` would have refused outright. +GITHUB_RELEASE_BODY_LIMIT = 125_000 + +#: Room left for the workflow's fixed preamble (install/configure instructions). +_PREAMBLE_RESERVE = 6_000 + +#: Never trim an entry below this -- a stub that says nothing is worse than a link. +_MIN_BODY_CHARS = 300 + @dataclass(frozen=True) class PullRequest: @@ -81,6 +94,73 @@ def clean_pr_body(body: str | None) -> str: return text.strip() +def demote_headings(body: str, min_level: int = 4) -> str: + """Shift every heading in a PR body down so it nests UNDER that PR's entry. + + Entries render as `## <category>` / `### <title>`, so a body containing its own `##` + headings would render them as SIBLINGS of the category headings and flatten the whole + document outline. Relative depth is preserved (the shallowest heading becomes + `min_level`), headings never go past h6, and `#` inside a fenced code block is left + alone -- a shell comment is not a heading. + """ + lines = body.split("\n") + + levels: list[int] = [] + in_fence = False + for line in lines: + if _FENCE_RE.match(line): + in_fence = not in_fence + continue + if in_fence: + continue + match = _HEADING_RE.match(line) + if match: + levels.append(len(match.group(1))) + + if not levels: + return body + shift = min_level - min(levels) + if shift <= 0: + return body + + out: list[str] = [] + in_fence = False + for line in lines: + if _FENCE_RE.match(line): + in_fence = not in_fence + out.append(line) + continue + match = None if in_fence else _HEADING_RE.match(line) + if match: + level = min(6, len(match.group(1)) + shift) + out.append("#" * level + match.group(2) + match.group(3)) + else: + out.append(line) + return "\n".join(out) + + +def truncate_body(body: str, limit: int | None, number: int) -> str: + """Trim `body` to roughly `limit` characters, pointing at the PR for the rest. + + Cuts at a paragraph (then line) boundary where one is available, and re-closes a code + fence left open by the cut -- an orphaned fence would swallow everything after it on the + release page. + """ + if limit is None or len(body) <= limit: + return body + + cut = body.rfind("\n\n", 0, limit) + if cut < limit // 2: + cut = body.rfind("\n", 0, limit) + if cut < limit // 2: + cut = limit + + kept = body[:cut].rstrip() + if kept.count("```") % 2: + kept += "\n```" + return f"{kept}\n\n*…truncated — full description in #{number}.*" + + def load_categories(path: str | Path = ".github/release.yml") -> list[Category]: """Read the notes categories from `.github/release.yml`, preserving their order. @@ -117,18 +197,47 @@ def categorize( return [(c, buckets[c.title]) for c in categories if buckets[c.title]] -def compose_release_notes(prs: list[PullRequest], categories: list[Category]) -> str: - """Render the grouped, body-inlined change list as markdown.""" - sections: list[str] = [] +def compose_release_notes( + prs: list[PullRequest], + categories: list[Category], + *, + body_limit: int | None = None, + total_limit: int | None = None, +) -> str: + """Render the grouped, body-inlined change list as markdown. - for category, bucket in categorize(prs, categories): + `total_limit` caps the whole document (GitHub refuses an over-long release body). The + budget is shared evenly across entries, so **every PR stays listed** and only the bodies + give ground -- dropping entries would silently hide shipped work. + """ + grouped = categorize(prs, categories) + entries = sum(len(bucket) for _, bucket in grouped) + + if total_limit is not None and entries: + # Headings, blank lines and the truncation marker all cost characters too. + overhead = sum(len(category.title) + 8 for category, _ in grouped) + sum( + len(pr.title) + 70 for _, bucket in grouped for pr in bucket + ) + budget = max(_MIN_BODY_CHARS, (total_limit - overhead) // entries) + body_limit = budget if body_limit is None else min(body_limit, budget) + + sections: list[str] = [] + for category, bucket in grouped: lines = [f"## {category.title}", ""] for pr in bucket: - body = clean_pr_body(pr.body) or NO_DESCRIPTION + body = clean_pr_body(pr.body) + body = truncate_body(demote_headings(body), body_limit, pr.number) if body else ( + NO_DESCRIPTION + ) lines += [f"### {pr.title} (#{pr.number})", "", body, ""] sections.append("\n".join(lines).rstrip()) - return "\n\n".join(sections) + out = "\n\n".join(sections) + + # Belt and braces: a pathological set of titles could still overrun the budget. + if total_limit is not None and len(out) > total_limit: + out = out[: max(0, total_limit - 60)].rstrip() + "\n\n*…release notes truncated.*" + return out def _pr_from_json(entry: dict) -> PullRequest: @@ -145,7 +254,11 @@ def main() -> None: payload = json.load(sys.stdin) or [] prs = [_pr_from_json(entry) for entry in payload] categories = load_categories(Path(__file__).resolve().parent.parent / ".github" / "release.yml") - sys.stdout.write(compose_release_notes(prs, categories)) + sys.stdout.write( + compose_release_notes( + prs, categories, total_limit=GITHUB_RELEASE_BODY_LIMIT - _PREAMBLE_RESERVE + ) + ) if __name__ == "__main__": diff --git a/tests/test_release_notes.py b/tests/test_release_notes.py index 3e2556da..a0473990 100644 --- a/tests/test_release_notes.py +++ b/tests/test_release_notes.py @@ -137,3 +137,85 @@ def test_load_categories_reads_the_repo_release_yml(): assert cats[-1].labels == ("*",), "the catch-all must be last" titles = [c.title for c in cats] assert "Features" in titles and "Compliance & rails" in titles + + +# -- heading levels: PR bodies must nest UNDER their entry, not beside it ------ + + +def test_body_headings_are_demoted_below_the_pr_heading(): + """A PR body full of `##` headings would otherwise render as siblings of the + category headings and destroy the document outline. Found on real data.""" + from scripts.release_notes import demote_headings + + out = demote_headings("## Top\n\ntext\n\n### Nested\n") + assert "#### Top" in out + assert "##### Nested" in out + + +def test_demotion_preserves_relative_heading_depth(): + from scripts.release_notes import demote_headings + + out = demote_headings("# A\n## B\n### C\n") + assert "#### A" in out and "##### B" in out and "###### C" in out + + +def test_demotion_never_exceeds_h6(): + from scripts.release_notes import demote_headings + + assert "####### " not in demote_headings("###### Deep\n") + + +def test_hashes_inside_code_fences_are_not_headings(): + from scripts.release_notes import demote_headings + + out = demote_headings("```bash\n# not a heading\n```\n\n## real heading\n") + assert "# not a heading" in out + assert "#### real heading" in out + + +def test_a_body_with_no_headings_is_untouched(): + from scripts.release_notes import demote_headings + + assert demote_headings("just prose\n") == "just prose\n" + + +def test_composed_notes_demote_real_body_headings(): + out = compose_release_notes( + [_pr(4, title="T", body="## Section\n\nprose", labels=["feature"])], CATS + ) + assert "## Features" in out + assert "### T (#4)" in out + assert "#### Section" in out + + +# -- size: GitHub rejects a release body over 125k characters ------------------ + + +def test_a_long_body_is_truncated_with_a_pointer_to_the_pr(): + from scripts.release_notes import truncate_body + + out = truncate_body("x" * 5000, 1000, 42) + assert len(out) < 1200 + assert "#42" in out and "truncated" in out + + +def test_truncation_closes_an_orphaned_code_fence(): + """Cutting mid-fence would break every following block in the release page.""" + from scripts.release_notes import truncate_body + + out = truncate_body("intro\n\n```python\n" + "y = 1\n" * 500, 200, 7) + assert out.count("```") % 2 == 0, "left an unclosed code fence" + + +def test_a_short_body_is_never_truncated(): + from scripts.release_notes import truncate_body + + assert truncate_body("short", 1000, 1) == "short" + + +def test_total_output_respects_a_total_limit(): + """The real first release composed 168k chars against a 125k platform limit.""" + prs = [_pr(n, title=f"PR {n}", body="z" * 9000, labels=["feature"]) for n in range(60)] + out = compose_release_notes(prs, CATS, total_limit=40_000) + assert len(out) <= 40_000, f"composed {len(out)} chars, over the limit" + assert "PR 59" in out, "every PR must still be listed, even if its body is trimmed"