From 95dcc22ae5b34aea7ae1210da4005573b351e996 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Thu, 9 Jul 2026 08:59:13 +0800 Subject: [PATCH] engine: opt-in security_range_start_na_warmup flag for TV HTF warmup parity TradingView's deep backtest truncates every request.security series at the applied range start, and TV's *built-in* ta.ema returns na until `length` values accumulate, then seeds with the SMA of those first `length` values -- unlike this engine's default EMA, which src-seeds on the first bar and is never na. A strategy gating direction on a long-lookback HTF ta.* therefore diverges from TV across the whole warmup window: TV's gate is na (comparisons -> false) for `length` HTF bars while the engine's gate has a real value from day one. Add an opt-in run flag `security_range_start_na_warmup` carrying the range start in epoch-ms, plumbed on the existing syminfo_metadata channel (the same path as bar_index_offset -- no new C ABI or run_strategy.py surface). When set, each request.security series: (a) aggregates from range_start_ms instead of the feed start (pre-range input bars are dropped from the security aggregation); (b) its embedded lookback ta.ema na-warms per TV built-in semantics -- na until `length` values accumulate, then seed = SMA of the first `length` values, then the ordinary EMA recursion; and (c) plain security expressions read na until the first completed HTF bar from the range start (a consequence of (a), counting the partial first bucket as HTF bar 1). Scope is the security-evaluator path only: the na-warm mode is a thread-local flag raised solely around request.security evaluation, so security-embedded EMAs latch it on their first compute() while chart-timeframe EMAs keep the default src-seed. ta.sma/ta.rma/ta.highest already na-warm from series start and are untouched. Behavior is byte-identical when the flag is unset. Tests: a new engine-level test pins na-until-length + SMA-first-length seed + range-start trim (flag on) and byte-identical src-seed (flag off); it fails without the fix. Full ctest 82/82. Corpus byte-identical with the flag unused (252 ok / 0 fail; excellent=239 strong=12 anomaly=1). Co-Authored-By: Claude Opus 4.8 --- include/pineforge/engine.hpp | 22 +++ include/pineforge/ta.hpp | 31 +++ src/engine_security.cpp | 26 +++ src/ta_moving_averages.cpp | 43 ++++- tests/CMakeLists.txt | 1 + tests/test_security_range_start_na_warmup.cpp | 177 ++++++++++++++++++ 6 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 tests/test_security_range_start_na_warmup.cpp diff --git a/include/pineforge/engine.hpp b/include/pineforge/engine.hpp index 090f7a7..c5c6d5c 100644 --- a/include/pineforge/engine.hpp +++ b/include/pineforge/engine.hpp @@ -535,6 +535,13 @@ class BacktestEngine { Bar current_bar_; int bar_index_ = 0; int bar_index_offset_ = 0; + // Opt-in KI-55 HTF warmup parity (see set_syminfo_metadata, + // "security_range_start_na_warmup"). When enabled, request.security series + // aggregate from security_range_start_ms_ instead of the feed start and + // their embedded ta.ema na-warm per TV built-in semantics. Default OFF → + // byte-identical behavior; touched only through feed_security_eval_state. + bool security_range_start_na_warmup_ = false; + int64_t security_range_start_ms_ = 0; int64_t next_order_seq_ = 1; // TV: at most one priced ENTRY "open" event per bar; persists across // multiple process_pending_orders calls (bar magnifier) and dual-pass @@ -1908,6 +1915,21 @@ class BacktestEngine { ? static_cast(std::llround(value)) : 0; } + // Opt-in KI-55 HTF warmup parity. The value is the TV deep-backtest + // range start in epoch-ms (exactly representable as a double for any + // realistic date — 2025 is ~1.7e12 << 2^53). A positive, finite value + // enables it; anything else (0 / NaN / negative) is the disabled + // default, so a run that never sets this key is byte-identical. + if (key == "security_range_start_na_warmup") { + if (std::isfinite(value) && value > 0.0) { + security_range_start_na_warmup_ = true; + security_range_start_ms_ = + static_cast(std::llround(value)); + } else { + security_range_start_na_warmup_ = false; + security_range_start_ms_ = 0; + } + } // "qty_step" is the per-instrument lot increment used by the forced- // liquidation quantizer. Route it onto the dedicated member so the // codegen run(const Bar*, int) path (which never overwrites it) keeps diff --git a/include/pineforge/ta.hpp b/include/pineforge/ta.hpp index eea925b..c68e139 100644 --- a/include/pineforge/ta.hpp +++ b/include/pineforge/ta.hpp @@ -10,6 +10,26 @@ namespace pineforge { namespace ta { +// Opt-in TradingView-built-in EMA warmup toggle (thread-local, default false). +// +// TradingView's *built-in* ``ta.ema`` returns ``na`` until ``length`` values +// have accumulated, then seeds with the SMA of those first ``length`` values — +// unlike the documented ``pine_ema`` reference impl (and this engine's default), +// which seed ``ema := src`` on the first bar and are therefore never ``na``. +// The difference only bites a ``request.security`` HTF series read under a +// range-start-truncated feed (KI-55): the security's embedded ``ta.ema`` must +// be ``na`` for its whole warmup window to match TV, exactly as ``ta.rma`` / +// ``ta.sma`` already are. +// +// An ``EMA`` instance latches this flag on its first ``compute()`` and keeps +// that mode for life. The engine raises the flag ONLY around +// ``request.security`` evaluation while the ``security_range_start_na_warmup`` +// run flag is active (see BacktestEngine::feed_security_eval_state), so +// security-embedded EMAs latch ``true`` and chart-timeframe EMAs latch +// ``false``. When the run flag is never set the toggle stays ``false`` and +// every EMA is byte-identical to the prior src-seed behavior. +bool& ema_na_warmup_flag(); + class RMA { double output_val; double sum; @@ -92,12 +112,23 @@ class EMA { double alpha; double sum; int bar_count; + // Retained only for the opt-in TradingView-built-in na-warmup path (below), + // which must count `length` values before seeding with their SMA. The + // default src-seed recursion uses `alpha` alone and never reads this. + int length_; // saved state for recompute double saved_output_val_; double saved_sum_; int saved_bar_count_; + // TradingView-built-in warmup latch (see ema_na_warmup_flag above). Latched + // once on the first compute() and never revisited, so recompute()'s + // restore() (which only rewinds output_val/sum/bar_count) cannot flip the + // mode mid-series. Deliberately NOT part of save()/restore(). + bool na_warmup_ = false; + bool warmup_latched_ = false; + public: explicit EMA(int length); double compute(double src); diff --git a/src/engine_security.cpp b/src/engine_security.cpp index f0946b6..4bad85b 100644 --- a/src/engine_security.cpp +++ b/src/engine_security.cpp @@ -4,6 +4,8 @@ #include "engine_internal.hpp" +#include + #include #include #include @@ -269,6 +271,30 @@ bool BacktestEngine::security_series_slot_is_new(int sec_id) const { void BacktestEngine::feed_security_eval_state(SecurityEvalState& state, const Bar& input_bar) { + // Opt-in KI-55 HTF warmup parity (security_range_start_na_warmup run flag): + // (a) start every request.security aggregation at range_start_ms, not the + // feed start — drop pre-range input bars so the aggregator, its TA + // members, and the exposed history all begin at the range start; + // (b) its embedded lookback ta.ema na-warms per TV built-in semantics — + // scoped by raising ta::ema_na_warmup_flag() for the duration of this + // call, which covers every evaluate_security() dispatch below (each of + // which is the only place the security's EMA members compute()); + // (c) plain security expressions (e.g. `close`) read na until the first + // COMPLETED HTF bar from the range start — a consequence of (a): under + // lookahead_off no evaluate_security() fires until the first bucket + // completes, and the partial first bucket counts as HTF bar 1. + // All three collapse to no-ops when the flag is unset (byte-identical). + if (security_range_start_na_warmup_ + && input_bar.timestamp < security_range_start_ms_) { + return; + } + struct SecurityNaWarmupScope { + bool prev_; + explicit SecurityNaWarmupScope(bool on) + : prev_(ta::ema_na_warmup_flag()) { ta::ema_na_warmup_flag() = on; } + ~SecurityNaWarmupScope() { ta::ema_na_warmup_flag() = prev_; } + } _na_warmup_scope(security_range_start_na_warmup_); + if (state.lower_tf_use_input) { // Buffer raw input bars until we accumulate one full script-TF // chunk, then aggregate (if req > input) and dispatch each diff --git a/src/ta_moving_averages.cpp b/src/ta_moving_averages.cpp index f5623ac..725afa9 100644 --- a/src/ta_moving_averages.cpp +++ b/src/ta_moving_averages.cpp @@ -19,6 +19,15 @@ namespace pineforge { namespace ta { +// Thread-local so parallel in-process engines never cross-contaminate. Default +// false → src-seed EMA (byte-identical to prior behavior); the engine raises it +// only around request.security evaluation under the security_range_start_na_warmup +// run flag. See the declaration in for the full rationale. +bool& ema_na_warmup_flag() { + static thread_local bool flag = false; + return flag; +} + RMA::RMA(int length) : output_val(na()), sum(0.0), length(length), bar_count(0), // Mirror the initial committed state so a recompute() issued before @@ -118,7 +127,7 @@ double SMA::compute(double src) { EMA::EMA(int length) : output_val(na()), alpha(2.0 / (length + 1)), sum(0.0), - bar_count(0), + bar_count(0), length_(length), // Mirror the initial committed state (see RMA::RMA) so a recompute() // issued before the first compute() restores a well-defined pristine // state instead of reading uninitialized save-state members. @@ -126,11 +135,43 @@ EMA::EMA(int length) double EMA::compute(double src) { save(); + // Latch the warmup mode once, on the first compute(). A security-embedded + // EMA's first compute() always runs inside the engine's request.security + // evaluation (where the flag is raised under security_range_start_na_warmup), + // and a chart EMA's never does — so the latch is consistent for the + // instance's whole life without threading any per-instance wiring through + // codegen. + if (!warmup_latched_) { + na_warmup_ = ema_na_warmup_flag(); + warmup_latched_ = true; + } if (is_na(src)) { // Pine semantics: ignore na inputs and keep prior EMA value. return output_val; } + if (na_warmup_) { + // TradingView *built-in* ta.ema warmup: return na until `length` values + // have accumulated since series start, then seed with the SMA of those + // first `length` values, then run the ordinary EMA recursion. Mirrors + // ta.rma/ta.sma warmup (RMA::compute above) so a range-start-truncated + // request.security(ta.ema(...)) reads na for its whole warmup window, + // matching TV (KI-55). Once output_val is non-na the series is seeded. + if (is_na(output_val)) { + sum += src; + bar_count++; + if (bar_count < length_) { + return na(); + } + // bar_count == length_: seed = SMA of the first `length_` values. + output_val = sum / static_cast(length_); + return output_val; + } + output_val = alpha * src + (1.0 - alpha) * output_val; + bar_count++; + return output_val; + } + // Pine ta.ema reference: // ema := na(ema[1]) ? src : alpha * src + (1 - alpha) * ema[1] // Seed from the first non-na source value (not SMA warmup). diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 551cf08..baa4676 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,6 +21,7 @@ set(TEST_SOURCES test_dmi_parity test_integration test_request_security + test_security_range_start_na_warmup test_security_tf_validation test_security_lower_tf_input_passthrough test_ltf_buffer_no_leak diff --git a/tests/test_security_range_start_na_warmup.cpp b/tests/test_security_range_start_na_warmup.cpp new file mode 100644 index 0000000..c2bbf45 --- /dev/null +++ b/tests/test_security_range_start_na_warmup.cpp @@ -0,0 +1,177 @@ +// test_security_range_start_na_warmup — pins the opt-in KI-55 HTF warmup flag. +// +// The engine run flag ``security_range_start_na_warmup`` (carried through the +// existing syminfo_metadata channel as an epoch-ms range start) makes every +// ``request.security`` series: +// (a) start aggregation at the range start (drop pre-range input bars); +// (b) na-warm its embedded ``ta.ema`` per TradingView's *built-in* semantics +// — na until ``length`` HTF bars accumulate, then seed with the SMA of +// those first ``length`` values, then recurse (NOT the engine's default +// src-seed); and +// (c) expose plain security expressions (here the completed HTF close) only +// from the first COMPLETED HTF bar at/after the range start. +// +// This fixture drives one HTF ("60" aggregated from "15") request.security with +// an embedded ta.ema(close, 3) end-to-end through the engine and asserts the +// exact per-completed-bar sequence with the flag ON, then asserts the flag OFF +// run is byte-identical to the prior src-seed behavior (opt-in / no regression). +// +// It FAILS without the fix: without the flag machinery the ON run would drop +// nothing and src-seed the EMA, so both the trimmed-count and the na-warm-seed +// assertions below would fail. + +#include +#include +#include + +#include +#include +#include +#include + +using namespace pineforge; + +static int failures = 0; + +// Bit-exact: the seed/recursion reference values are chosen to be exact +// doubles, so any drift (or a wrong warmup shape) is a hard mismatch. +static bool exact_eq(double a, double b) { + if (is_na(a) && is_na(b)) return true; + if (is_na(a) || is_na(b)) return false; + return a == b; +} + +#define CHECK(cond, tag) do { \ + if (!(cond)) { \ + std::printf("FAIL: %s (line %d)\n", (tag), __LINE__); \ + ++failures; \ + } \ +} while (0) + +// One HTF request.security ("60" from "15"), lookahead_off, with an embedded +// ta.ema(close, 3). Records every completed HTF bar's plain close and embedded +// EMA value, in order. +class RangeStartWarmupHarness : public BacktestEngine { +public: + ta::EMA sec_ema_{3}; + std::vector htf_close_seq; + std::vector htf_ema_seq; + + RangeStartWarmupHarness() { + register_security_eval(0, "60", "15", /*lookahead_on=*/false, + /*gaps_on=*/false); + } + + void evaluate_security(int sec_id, const Bar& bar, bool is_complete) override { + if (sec_id != 0 || !is_complete) { + return; + } + htf_close_seq.push_back(bar.close); + htf_ema_seq.push_back(sec_ema_.compute(bar.close)); + } + + void on_bar(const Bar&) override {} +}; + +// 8 hours of 15m bars (4 bars/hour). Hour k's four bars all carry close +// hour_close[k], so the aggregated hourly close is unambiguously hour_close[k]. +// Hour 0 (ts 0..2.7e6) lies BEFORE the range start (3.6e6) and is dropped when +// the flag is on. Hour 7 never completes (no following bar), matching the +// aggregator's boundary-flush rule. +static std::vector make_feed() { + const double hour_close[8] = {999.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0}; + std::vector bars; + for (int h = 0; h < 8; ++h) { + for (int q = 0; q < 4; ++q) { + int64_t ts = static_cast(h) * 3600000 + + static_cast(q) * 900000; + double c = hour_close[h]; + bars.push_back(Bar{c, c, c, c, 100.0, ts}); + } + } + return bars; +} + +// range start = start of hour 1 (drops hour 0). +static const double kRangeStartMs = 3600000.0; + +void test_flag_on_na_warm_and_trim() { + RangeStartWarmupHarness strat; + strat.set_syminfo_metadata("security_range_start_na_warmup", kRangeStartMs); + auto bars = make_feed(); + strat.run(bars.data(), static_cast(bars.size()), "15", "15"); + + // (a)+(c): hour 0 trimmed -> completed HTF bars are hours 1..7 (7 of them; + // hour 7 is flushed as the final bucket at end-of-run). First completed + // close is hour 1's (10), NOT hour 0's (999). + const double exp_close[7] = {10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0}; + CHECK(strat.htf_close_seq.size() == 7, + "flag-on: hour0 trimmed -> 7 completed HTF bars"); + for (std::size_t i = 0; i < strat.htf_close_seq.size() && i < 7; ++i) { + CHECK(exact_eq(strat.htf_close_seq[i], exp_close[i]), + "flag-on: completed HTF close sequence"); + } + + // (b): ta.ema(3) na-warm. na, na, then SMA(10,20,30)=20 at the 3rd HTF bar, + // then EMA recursion with alpha = 2/(3+1) = 0.5: + // bar4: 0.5*40 + 0.5*20 = 30 + // bar5: 0.5*50 + 0.5*30 = 40 + // bar6: 0.5*60 + 0.5*40 = 50 + // bar7: 0.5*70 + 0.5*50 = 60 + const double na_ = na(); + const double exp_ema[7] = {na_, na_, 20.0, 30.0, 40.0, 50.0, 60.0}; + CHECK(strat.htf_ema_seq.size() == 7, "flag-on: 7 embedded-EMA values"); + for (std::size_t i = 0; i < strat.htf_ema_seq.size() && i < 7; ++i) { + CHECK(exact_eq(strat.htf_ema_seq[i], exp_ema[i]), + "flag-on: embedded ta.ema na-warm + SMA-first-length seed"); + } + + std::printf("test_flag_on_na_warm_and_trim: %s\n", + failures ? "FAIL" : "ok"); +} + +void test_flag_off_byte_identical_srcseed() { + int before = failures; + RangeStartWarmupHarness strat; // flag NOT set + auto bars = make_feed(); + strat.run(bars.data(), static_cast(bars.size()), "15", "15"); + + // No trim: hour 0 (close 999) is a valid HTF bar -> hours 0..7 complete (8; + // hour 7 flushed as the final bucket at end-of-run). + const double exp_close[8] = {999.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0}; + CHECK(strat.htf_close_seq.size() == 8, + "flag-off: hour0 NOT trimmed -> 8 completed HTF bars"); + for (std::size_t i = 0; i < strat.htf_close_seq.size() && i < 8; ++i) { + CHECK(exact_eq(strat.htf_close_seq[i], exp_close[i]), + "flag-off: completed HTF close sequence"); + } + + // Default src-seed EMA: never na; seeds at the first HTF close (999), then + // bar2: 0.5*10 + 0.5*999 = 504.5 + // bar3: 0.5*20 + 0.5*504.5 = 262.25 + const double exp_ema[4] = { + 999.0, 504.5, 262.25, + 0.5 * 30.0 + 0.5 * 262.25, + }; + CHECK(strat.htf_ema_seq.size() == 8, "flag-off: 8 embedded-EMA values"); + CHECK(!strat.htf_ema_seq.empty() && !is_na(strat.htf_ema_seq.front()), + "flag-off: EMA non-na from the first HTF bar (src-seed)"); + for (std::size_t i = 0; i < strat.htf_ema_seq.size() && i < 4; ++i) { + CHECK(exact_eq(strat.htf_ema_seq[i], exp_ema[i]), + "flag-off: EMA src-seed recursion (byte-identical to prior)"); + } + + std::printf("test_flag_off_byte_identical_srcseed: %s\n", + (failures > before) ? "FAIL" : "ok"); +} + +int main() { + test_flag_on_na_warm_and_trim(); + test_flag_off_byte_identical_srcseed(); + if (failures) { + std::printf("%d check(s) FAILED\n", failures); + return 1; + } + std::printf("test_security_range_start_na_warmup passed.\n"); + return 0; +}