diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c9cde9..37b781a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Migration guides will be provided for all breaking changes - Semantic versioning (MAJOR.MINOR.PATCH) is strictly followed +## [Unreleased] + +### 🐛 Fixed + +- **Realtime bar schema mismatch**: extra fields in the bars API response made + the historical seed wider than the six-column bars the realtime data manager + constructs, so appending a new realtime bar failed with a Polars width + mismatch (`unable to append to a DataFrame of width N with a DataFrame of + width 6`) and froze realtime timeframes. The realtime data manager now + appends bars with diagonal concatenation, tolerating the wider historical + schema (extra fields become null on realtime bars). + +### ✨ Added + +- **Preserve extra bar fields**: `get_bars` now keeps any additional fields the + bars API returns (canonical `timestamp, open, high, low, close, volume` + first), instead of dropping them, so callers retain access to the extra data. + ## [3.5.8] - 2025-09-02 ### 🐛 Fixed diff --git a/src/project_x_py/client/market_data.py b/src/project_x_py/client/market_data.py index 55fbb17..8fd02e8 100644 --- a/src/project_x_py/client/market_data.py +++ b/src/project_x_py/client/market_data.py @@ -554,6 +554,14 @@ async def get_bars( ) ) + # Order the canonical OHLCV columns first, but preserve any additional + # fields the bars API returns (e.g. trade count) rather than dropping + # them. The realtime data manager tolerates the wider historical schema + # via diagonal concatenation, so callers keep access to the extra data. + canonical = ["timestamp", "open", "high", "low", "close", "volume"] + extra = [column for column in data.columns if column not in canonical] + data = data.select(canonical + extra) + # Handle datetime conversion robustly # Try the simple approach first (fastest for consistent data) try: diff --git a/src/project_x_py/realtime_data_manager/core.py b/src/project_x_py/realtime_data_manager/core.py index ed67b27..8fec30b 100644 --- a/src/project_x_py/realtime_data_manager/core.py +++ b/src/project_x_py/realtime_data_manager/core.py @@ -1306,7 +1306,12 @@ async def _check_and_create_empty_bars(self) -> None: } ) - self.data[tf_key] = pl.concat([current_data, new_bar]) + # Diagonal so a six-column realtime bar can extend a + # wider historical frame (extra API fields become + # null) instead of raising a width mismatch. + self.data[tf_key] = pl.concat( + [current_data, new_bar], how="diagonal_relaxed" + ) self.last_bar_times[tf_key] = expected_bar_time self.logger.debug( @@ -1379,7 +1384,12 @@ async def _check_and_create_empty_bars(self) -> None: } ) - self.data[tf_key] = pl.concat([current_data, new_bar]) + # Diagonal so a six-column realtime bar can extend a + # wider historical frame (extra API fields become + # null) instead of raising a width mismatch. + self.data[tf_key] = pl.concat( + [current_data, new_bar], how="diagonal_relaxed" + ) self.last_bar_times[tf_key] = expected_bar_time self.logger.debug( diff --git a/src/project_x_py/realtime_data_manager/data_processing.py b/src/project_x_py/realtime_data_manager/data_processing.py index 5ed1bd3..58ec30f 100644 --- a/src/project_x_py/realtime_data_manager/data_processing.py +++ b/src/project_x_py/realtime_data_manager/data_processing.py @@ -734,7 +734,12 @@ async def _update_timeframe_data( } ) - self.data[tf_key] = pl.concat([current_data, new_bar]) + # Diagonal so a six-column realtime bar can extend a wider + # historical frame (extra API fields become null on the new + # bar) instead of raising a width mismatch. + self.data[tf_key] = pl.concat( + [current_data, new_bar], how="diagonal_relaxed" + ) self.last_bar_times[tf_key] = bar_time # Track new bar creation with new statistics system diff --git a/tests/client/test_market_data.py b/tests/client/test_market_data.py index 89291b3..31417c6 100644 --- a/tests/client/test_market_data.py +++ b/tests/client/test_market_data.py @@ -279,6 +279,72 @@ async def test_get_bars( cache_key = "MGC_5_5_2_True" assert cache_key in client._opt_market_data_cache + @pytest.mark.asyncio + async def test_get_bars_preserves_extra_api_columns( + self, + mock_httpx_client, + mock_auth_response, + mock_instrument_response, + mock_response, + ): + """Extra fields in the bars API response are preserved, canonical first. + + The realtime data manager tolerates the wider historical schema via + diagonal concatenation, so get_bars keeps additional fields instead of + dropping them. + """ + auth_response, accounts_response = mock_auth_response + now = datetime.datetime.now(pytz.UTC) + bars_data = [ + { + "t": (now - datetime.timedelta(minutes=i * 5)).isoformat(), + "o": 1900.0 + i, + "h": 1905.0 + i, + "l": 1895.0 + i, + "c": 1902.0 + i, + "v": 100 + i, + # Additional fields the bars API may include. + "symbol": "MGC", + "n": i, + } + for i in range(3) + ] + bars_response = mock_response(json_data={"success": True, "bars": bars_data}) + mock_httpx_client.request.side_effect = [ + auth_response, + accounts_response, + mock_instrument_response, + bars_response, + ] + + with patch("httpx.AsyncClient", return_value=mock_httpx_client): + async with ProjectX("testuser", "test-api-key") as client: + client.api_call_count = 0 + client._opt_instrument_cache = {} + client._opt_instrument_cache_time = {} + client._opt_market_data_cache = {} + client._opt_market_data_cache_time = {} + client.cache_ttl = 300 + client.last_cache_cleanup = time.time() + client.cache_hit_count = 0 + client.rate_limiter = RateLimiter(max_requests=100, window_seconds=60) + await client.authenticate() + + bars = await client.get_bars("MGC", days=5, interval=5) + + # Canonical OHLCV columns come first, in order... + assert bars.columns[:6] == [ + "timestamp", + "open", + "high", + "low", + "close", + "volume", + ] + # ...and the extra API fields are preserved, not dropped. + assert "symbol" in bars.columns + assert "n" in bars.columns + @pytest.mark.asyncio async def test_get_bars_from_cache(self, mock_httpx_client, mock_auth_response): """Test getting bars from cache."""