From 92ead94f5dbe9cda619b0fcc70992e199d7afa83 Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim Date: Tue, 28 Jul 2026 16:30:58 -0400 Subject: [PATCH] fix(turtle): drop the daily bar only when it is really forming The forming-bar guard keyed off the mere PRESENCE of an ONE_HOUR key. That is the account sim's shape (portfolio_sim hands the rule a hourly window inside the current, still-forming day), but the live agent passes ONE_HOUR too -- and market_feed persists only CLOSED candles, so its newest daily bar has already closed. The agent was therefore discarding a completed day and deciding on a bar up to 48h old: a full day of lag on every breakout entry and channel exit. Decide by where the hourly series sits instead of whether it exists: the newest daily bar is forming unless the newest hourly bar opens at or after that day's close. In the sim that is never true -- it slices daily with bisect_right(daily_ts, t), so its hourly bar is by construction inside the last daily bar -- so the sim's guard is provably unchanged. Against the live paper DB this moves the decision bar from 2026-07-26 to the newest closed day, 2026-07-27. Co-Authored-By: Claude Opus 5 (1M context) --- keel/strategy/rules/turtle_breakout.py | 58 ++++++++++---- tests/strategy/test_turtle_breakout.py | 105 +++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 16 deletions(-) diff --git a/keel/strategy/rules/turtle_breakout.py b/keel/strategy/rules/turtle_breakout.py index 4201a09f..b558c6e0 100644 --- a/keel/strategy/rules/turtle_breakout.py +++ b/keel/strategy/rules/turtle_breakout.py @@ -26,15 +26,18 @@ lookback longer than 20 beat 20 out-of-sample; 40 was the most robust across held-out years). The 2:1 asymmetric ratio, ADX(14) gate, and 20-day ATR "N" are unchanged. -**Forming-bar lookahead guard.** The two backtest passes present the daily series differently: +**Forming-bar lookahead guard.** Three callers present the daily series differently: - The *edge* backtester (`strategy.backtest.backtest`) drives the rule on its native series only -- no `ONE_HOUR` key -- and every daily bar in it is already closed. - The *account* simulator (`sim.portfolio_sim`) iterates hourly and hands the rule BOTH an `ONE_HOUR` window AND the `ONE_DAY` series, where the last daily bar is the CURRENT, still- forming day (its DB OHLC is the completed day = lookahead if consumed intraday). -`detect()`/`exit_signal()` therefore drop the last daily bar iff an `ONE_HOUR` key is present, -so decisions are made only on completed days in the account pass while using every (closed) bar -in the edge pass. +- The *live agent* (`agent.run_once`) also passes an `ONE_HOUR` key, but `data.market_feed` + persists only CLOSED candles, so its last daily bar has already closed. +`detect()`/`exit_signal()` therefore drop the last daily bar only when it is genuinely still +forming -- decided by where the hourly series SITS, not by whether it exists (`_completed_days`). +Keying it on the mere presence of `ONE_HOUR` cost the live agent a full day of lag on every +breakout, since it discarded a completed day the sim never had. """ from __future__ import annotations @@ -45,6 +48,35 @@ from keel.strategy.rules.base import Rule, Setup from keel.types import Candle, Granularity +_DAY_SECONDS = 24 * 60 * 60 + + +def _completed_days(candles_by_tf: dict[Granularity, list[Candle]]) -> list[Candle]: + """The `ONE_DAY` series with any still-FORMING last bar dropped. + + The account sim (`sim.portfolio_sim`) slices its daily series with + `bisect_right(daily_ts, t)` against the current hourly bar `t`, so its last daily bar + always CONTAINS that hourly bar -- it is the current, still-forming day, and its DB OHLC is + the completed day (lookahead if consumed intraday). + + The live agent (`agent.run_once`) passes an `ONE_HOUR` key too, but `data.market_feed` + persists only CLOSED candles, so its last daily bar has already closed. Keying the drop on + the mere PRESENCE of `ONE_HOUR` therefore threw away a completed day there and left the + rule deciding on a bar up to 48h old -- a day of lag on every breakout. + + So the test is where the hourly series SITS, not whether it exists: the newest daily bar is + still forming unless the newest hourly bar opens at or after that day's close. In the sim + that is never true (its hourly bar is by construction inside the day), so the sim's guard is + unchanged; in the live agent it is true from the first full hour of the next UTC day. + """ + daily = candles_by_tf.get(Granularity.ONE_DAY, []) + hourly = candles_by_tf.get(Granularity.ONE_HOUR) + if not hourly or not daily: + return daily + if hourly[-1].ts < daily[-1].ts + _DAY_SECONDS: + return daily[:-1] + return daily + class TurtleBreakout(Rule): """Donchian-breakout trend-follower: ADX-gated entry, asymmetric channel exit, 2xATR stop. @@ -125,13 +157,10 @@ def detect(self, candles_by_tf: dict[Granularity, list[Candle]]) -> Setup | None this target; it only exists to clear the evaluation engine's rr>=1 kill-zone gate and let winners run past a fixed 1:1/2:1 cap). """ - daily = candles_by_tf.get(Granularity.ONE_DAY, []) - # Account sim (portfolio_sim) includes the current *forming* daily bar alongside an - # ONE_HOUR window -> drop it to decide only on completed days (no lookahead). The - # daily-only edge backtest has no ONE_HOUR key and every daily bar is already closed, - # so use them all. - if candles_by_tf.get(Granularity.ONE_HOUR): - daily = daily[:-1] + # Drop the last daily bar only when it is genuinely still forming -- see + # `_completed_days`. The daily-only edge backtest has no ONE_HOUR key and every daily + # bar is already closed, so it uses them all. + daily = _completed_days(candles_by_tf) entry_lookback = self.params["entry_lookback"] exit_lookback = self.params["exit_lookback"] @@ -229,11 +258,8 @@ def exit_signal( nominal target are the backtester/account-sim's job to enforce separately. """ del held - daily = candles_by_tf.get(Granularity.ONE_DAY, []) - # Same forming-bar guard as detect(): the account sim carries the current forming daily - # bar alongside an ONE_HOUR window -> decide on completed days only. - if candles_by_tf.get(Granularity.ONE_HOUR): - daily = daily[:-1] + # Same forming-bar guard as detect() -- see `_completed_days`. + daily = _completed_days(candles_by_tf) exit_lookback = self.params["exit_lookback"] if len(daily) <= exit_lookback + 1: diff --git a/tests/strategy/test_turtle_breakout.py b/tests/strategy/test_turtle_breakout.py index e5b2e3cb..7a32f234 100644 --- a/tests/strategy/test_turtle_breakout.py +++ b/tests/strategy/test_turtle_breakout.py @@ -13,6 +13,7 @@ from __future__ import annotations +from dataclasses import replace from decimal import Decimal from keel import agent @@ -122,6 +123,26 @@ def _falling_candles(n: int = 10, start: float = 200.0, decr: float = 3.0) -> li return _trending_base(n, start=start, incr=-decr) +_DAY = 24 * 60 * 60 +_HOUR = 60 * 60 +#: An epoch second that is exactly a UTC day boundary (86400 * 20660). The forming-bar guard +#: compares real bar timestamps, so the day-shaped fixtures need day-aligned ones. +_DAY_ZERO = 1_785_024_000 + + +def _day_candle(ts: int, o: float, h: float, low: float, c: float) -> Candle: + return _candle(ts, o, h, low, c) + + +def _on_days(candles: list[Candle], start_ts: int = _DAY_ZERO) -> list[Candle]: + """Re-stamp a hand-built series onto consecutive UTC day boundaries. + + The shared fixtures number their bars 0, 1, 2...; the live-agent shape needs timestamps a + real daily series would have, since that is what the guard reasons about. + """ + return [replace(c, ts=start_ts + i * _DAY) for i, c in enumerate(candles)] + + def _rule(**overrides) -> TurtleBreakout: params = {"product_id": "BTC-USD", **_SMALL_PARAMS} params.update(overrides) @@ -270,6 +291,90 @@ def test_completed_breakout_fires_despite_non_triggering_forming_bar(self) -> No assert setup.ts == 20 +class TestCompletedDailyBarIsUsedInTheLiveAgentPath: + """The LIVE agent (`agent.run_once`) hands the rule an `ONE_HOUR` key too, but its daily + series has NO forming bar: `data.market_feed` only ever persists CLOSED candles. Dropping + the newest daily bar there is not a lookahead guard, it is a full day of lag on every + breakout -- the rule decides on a bar that closed up to 48h ago. + + What separates the two callers is not the presence of `ONE_HOUR` but where the hourly + series SITS: the account sim's hourly window is always inside the last daily bar's own + period (`portfolio_sim` slices daily with `bisect_right(daily_ts, t)`), while the live + agent's has advanced into a later day. + """ + + def test_breakout_on_the_newest_closed_daily_bar_fires(self) -> None: + rule = _rule() + # Penultimate completed day pulls back -- no breakout there -- so only the NEWEST daily + # bar can produce a setup. If it is dropped, detect() returns None. + base = _on_days(_trending_base(19)) + pullback_price = float(base[-3].close) + base.append( + _day_candle( + base[-1].ts + _DAY, + pullback_price - 0.5, pullback_price + 0.5, pullback_price - 0.5, pullback_price, + ) + ) + breakout_price = float(base[-2].close) + 20.0 + breakout_ts = base[-1].ts + _DAY + base.append( + _day_candle( + breakout_ts, + breakout_price - 0.5, breakout_price + 0.5, breakout_price - 0.5, breakout_price, + ) + ) + # The live agent's newest CLOSED hourly bar sits in the day AFTER the newest daily bar, + # which is only possible once that daily bar has itself closed. + hourly = [_day_candle(breakout_ts + _DAY, 1, 1, 1, 1)] + + setup = rule.detect({Granularity.ONE_HOUR: hourly, Granularity.ONE_DAY: base}) + + assert setup is not None + assert setup.ts == breakout_ts + + def test_account_sim_forming_bar_is_still_dropped(self) -> None: + """The guard must keep working for `portfolio_sim`, whose hourly bar sits INSIDE the + last daily bar's period -- the one shape where that bar really is still forming. + """ + rule = _rule() + base = _on_days(_trending_base(19)) + pullback_price = float(base[-3].close) + base.append( + _day_candle( + base[-1].ts + _DAY, + pullback_price - 0.5, pullback_price + 0.5, pullback_price - 0.5, pullback_price, + ) + ) + forming_ts = base[-1].ts + _DAY + breakout_price = float(base[-2].close) + 20.0 + base.append( + _day_candle( + forming_ts, + breakout_price - 0.5, breakout_price + 0.5, breakout_price - 0.5, breakout_price, + ) + ) + # Mid-day hourly bar, inside the last daily bar's own period = still forming. + hourly = [_day_candle(forming_ts + 12 * _HOUR, 1, 1, 1, 1)] + + assert rule.detect({Granularity.ONE_HOUR: hourly, Granularity.ONE_DAY: base}) is None + + def test_exit_fires_on_the_newest_closed_daily_bar(self) -> None: + rule = _rule() + # A rising base, then a single sharp drop below the prior 3-day channel low ON THE + # NEWEST bar. No earlier bar breaks its own channel, so dropping the newest = no exit. + base = _on_days(_trending_base(20)) + break_price = float(min(c.low for c in base[-3:])) - 5.0 + break_ts = base[-1].ts + _DAY + base.append( + _day_candle( + break_ts, break_price + 0.5, break_price + 0.5, break_price - 0.5, break_price + ) + ) + hourly = [_day_candle(break_ts + _DAY, 1, 1, 1, 1)] + + assert rule.exit_signal(None, {Granularity.ONE_HOUR: hourly, Granularity.ONE_DAY: base}) + + class TestExitSignal: def test_fires_when_close_breaks_below_prior_channel_low(self) -> None: rule = _rule()