feat: preserve extra bar fields; diagonal concat for realtime bars

CHANGELOG.md

@@ -14,6 +14,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Migration guides will be provided for all breaking changes
 - Semantic versioning (MAJOR.MINOR.PATCH) is strictly followed
 
+## [Unreleased]
+
+### 🐛 Fixed
+
+- **Realtime bar schema mismatch**: extra fields in the bars API response made
+  the historical seed wider than the six-column bars the realtime data manager
+  constructs, so appending a new realtime bar failed with a Polars width
+  mismatch (`unable to append to a DataFrame of width N with a DataFrame of
+  width 6`) and froze realtime timeframes. The realtime data manager now
+  appends bars with diagonal concatenation, tolerating the wider historical
+  schema (extra fields become null on realtime bars).
+
+### ✨ Added
+
+- **Preserve extra bar fields**: `get_bars` now keeps any additional fields the
+  bars API returns (canonical `timestamp, open, high, low, close, volume`
+  first), instead of dropping them, so callers retain access to the extra data.
+
 ## [3.5.8] - 2025-09-02
 
 ### 🐛 Fixed

src/project_x_py/client/market_data.py

@@ -554,6 +554,14 @@ async def get_bars(
             )
         )
 
+        # Order the canonical OHLCV columns first, but preserve any additional
+        # fields the bars API returns (e.g. trade count) rather than dropping
+        # them. The realtime data manager tolerates the wider historical schema
+        # via diagonal concatenation, so callers keep access to the extra data.
+        canonical = ["timestamp", "open", "high", "low", "close", "volume"]
+        extra = [column for column in data.columns if column not in canonical]
+        data = data.select(canonical + extra)
+
         # Handle datetime conversion robustly
         # Try the simple approach first (fastest for consistent data)
         try:

src/project_x_py/realtime_data_manager/core.py

@@ -1306,7 +1306,12 @@ async def _check_and_create_empty_bars(self) -> None:
                                 }
                             )
 
-                            self.data[tf_key] = pl.concat([current_data, new_bar])
+                            # Diagonal so a six-column realtime bar can extend a
+                            # wider historical frame (extra API fields become
+                            # null) instead of raising a width mismatch.
+                            self.data[tf_key] = pl.concat(
+                                [current_data, new_bar], how="diagonal_relaxed"
+                            )
                             self.last_bar_times[tf_key] = expected_bar_time
 
                             self.logger.debug(
@@ -1379,7 +1384,12 @@ async def _check_and_create_empty_bars(self) -> None:
                                 }
                             )
 
-                            self.data[tf_key] = pl.concat([current_data, new_bar])
+                            # Diagonal so a six-column realtime bar can extend a
+                            # wider historical frame (extra API fields become
+                            # null) instead of raising a width mismatch.
+                            self.data[tf_key] = pl.concat(
+                                [current_data, new_bar], how="diagonal_relaxed"
+                            )
                             self.last_bar_times[tf_key] = expected_bar_time
 
                             self.logger.debug(

src/project_x_py/realtime_data_manager/data_processing.py

@@ -734,7 +734,12 @@ async def _update_timeframe_data(
                         }
                     )
 
-                    self.data[tf_key] = pl.concat([current_data, new_bar])
+                    # Diagonal so a six-column realtime bar can extend a wider
+                    # historical frame (extra API fields become null on the new
+                    # bar) instead of raising a width mismatch.
+                    self.data[tf_key] = pl.concat(
+                        [current_data, new_bar], how="diagonal_relaxed"
+                    )
                     self.last_bar_times[tf_key] = bar_time
 
                     # Track new bar creation with new statistics system

tests/client/test_market_data.py

@@ -279,6 +279,72 @@ async def test_get_bars(
                 cache_key = "MGC_5_5_2_True"
                 assert cache_key in client._opt_market_data_cache
 
+    @pytest.mark.asyncio
+    async def test_get_bars_preserves_extra_api_columns(
+        self,
+        mock_httpx_client,
+        mock_auth_response,
+        mock_instrument_response,
+        mock_response,
+    ):
+        """Extra fields in the bars API response are preserved, canonical first.
+
+        The realtime data manager tolerates the wider historical schema via
+        diagonal concatenation, so get_bars keeps additional fields instead of
+        dropping them.
+        """
+        auth_response, accounts_response = mock_auth_response
+        now = datetime.datetime.now(pytz.UTC)
+        bars_data = [
+            {
+                "t": (now - datetime.timedelta(minutes=i * 5)).isoformat(),
+                "o": 1900.0 + i,
+                "h": 1905.0 + i,
+                "l": 1895.0 + i,
+                "c": 1902.0 + i,
+                "v": 100 + i,
+                # Additional fields the bars API may include.
+                "symbol": "MGC",
+                "n": i,
+            }
+            for i in range(3)
+        ]
+        bars_response = mock_response(json_data={"success": True, "bars": bars_data})
+        mock_httpx_client.request.side_effect = [
+            auth_response,
+            accounts_response,
+            mock_instrument_response,
+            bars_response,
+        ]
+
+        with patch("httpx.AsyncClient", return_value=mock_httpx_client):
+            async with ProjectX("testuser", "test-api-key") as client:
+                client.api_call_count = 0
+                client._opt_instrument_cache = {}
+                client._opt_instrument_cache_time = {}
+                client._opt_market_data_cache = {}
+                client._opt_market_data_cache_time = {}
+                client.cache_ttl = 300
+                client.last_cache_cleanup = time.time()
+                client.cache_hit_count = 0
+                client.rate_limiter = RateLimiter(max_requests=100, window_seconds=60)
+                await client.authenticate()
+
+                bars = await client.get_bars("MGC", days=5, interval=5)
+
+                # Canonical OHLCV columns come first, in order...
+                assert bars.columns[:6] == [
+                    "timestamp",
+                    "open",
+                    "high",
+                    "low",
+                    "close",
+                    "volume",
+                ]
+                # ...and the extra API fields are preserved, not dropped.
+                assert "symbol" in bars.columns
+                assert "n" in bars.columns
+
     @pytest.mark.asyncio
     async def test_get_bars_from_cache(self, mock_httpx_client, mock_auth_response):
         """Test getting bars from cache."""