diff --git a/src/ml4t/data/providers/yahoo.py b/src/ml4t/data/providers/yahoo.py index 04fab6a..b9c1aa8 100644 --- a/src/ml4t/data/providers/yahoo.py +++ b/src/ml4t/data/providers/yahoo.py @@ -53,6 +53,22 @@ def _chunks(lst: list[Any], n: int) -> Iterator[list[Any]]: yield lst[i : i + n] +def _drop_priceless_rows(df: pl.DataFrame) -> pl.DataFrame: + """Drop bars that carry no price at all. + + From the US close until Yahoo consolidates the daily bar, ``yf.download`` returns a + final row for the current exchange date with volume and null OHLC. It is not a + session: keeping it writes null prices into storage and into everything derived from + them. A row with prices and zero volume is a real halted session and is kept. + """ + price_columns = ("open", "high", "low", "close") + present = [column for column in price_columns if column in df.columns] + if not present: + return df + has_price = pl.any_horizontal(pl.col(column).is_not_null() for column in present) + return df.filter(has_price) + + class YahooFinanceProvider(BaseProvider): """ Thin wrapper around yfinance for API consistency and incremental updates. @@ -194,6 +210,14 @@ def _fetch_and_transform_data( # Convert to Polars with symbol column df = self._convert_to_polars(df_pandas, symbol) + if df.is_empty(): + # Every row was the current session's placeholder; the window holds no bar. + raise SymbolNotFoundError( + "yahoo", + symbol, + details={"start": start, "end": end_str, "frequency": frequency}, + ) + logger.info("Successfully fetched data", symbol=symbol, rows=len(df)) return df @@ -279,6 +303,7 @@ def _convert_to_polars(self, df_pandas: pd.DataFrame, symbol: str) -> pl.DataFra pl.col("volume").cast(pl.Float64), ) .with_columns(pl.lit(symbol.upper()).alias("symbol")) + .pipe(_drop_priceless_rows) .select(["timestamp", "symbol", "open", "high", "low", "close", "volume"]) ) @@ -587,10 +612,7 @@ def _convert_batch_to_polars( ] ) - # Drop rows where all OHLCV are null (symbol had no data for that date) - df_symbol = df_symbol.filter( - pl.col("close").is_not_null() | pl.col("open").is_not_null() - ) + df_symbol = _drop_priceless_rows(df_symbol) if len(df_symbol) > 0: records.append(df_symbol) diff --git a/tests/futures/test_continuous.py b/tests/futures/test_continuous.py index 59fc626..6bfc54a 100644 --- a/tests/futures/test_continuous.py +++ b/tests/futures/test_continuous.py @@ -8,7 +8,7 @@ import pytest from ml4t.data.core.config import resolve_storage_path -from ml4t.data.futures.adjustment import BackAdjustment, RatioAdjustment +from ml4t.data.futures.adjustment import BackAdjustment, NoAdjustment, RatioAdjustment from ml4t.data.futures.continuous import ( ContinuousContractBuilder, build_continuous_contract, @@ -557,3 +557,71 @@ def test_storage_path_as_pathlib(self, sample_raw_data, sample_continuous_data): ) mock_raw.assert_called_once() + + +class TestContinuousContractRemovesRollGap: + """End-to-end: the series a reader gets back carries no artificial roll-day move.""" + + @staticmethod + def _two_contract_panel() -> pl.DataFrame: + """Front month at 100 rising by 1/day, deferred 10 points above it in contango. + + Volume moves to the deferred contract on 2024-01-04, so a roll follows. + """ + dates = [date(2024, 1, day) for day in range(1, 7)] + front_close = [100.0, 101.0, 102.0, 103.0, 104.0, 105.0] + back_close = [close + 10.0 for close in front_close] + front_volume = [10000.0, 10000.0, 10000.0, 1000.0, 1000.0, 1000.0] + back_volume = [1000.0, 1000.0, 1000.0, 10000.0, 10000.0, 10000.0] + return pl.DataFrame( + { + "date": dates * 2, + "symbol": ["ESH24"] * 6 + ["ESM24"] * 6, + "open": front_close + back_close, + "high": front_close + back_close, + "low": front_close + back_close, + "close": front_close + back_close, + "volume": front_volume + back_volume, + } + ) + + def _build(self, adjustment_method) -> pl.DataFrame: + builder = ContinuousContractBuilder( + roll_strategy=VolumeBasedRoll(min_days_between_rolls=0), + adjustment_method=adjustment_method, + ) + with patch("ml4t.data.futures.continuous.parse_quandl_chris_raw") as mock_raw: + mock_raw.return_value = self._two_contract_panel() + return builder.build("ES") + + def test_unadjusted_series_carries_the_roll_gap(self): + """The premise: without adjustment the roll day shows a jump no contract made.""" + result = self._build(NoAdjustment()) + roll_index = result["is_roll_date"].to_list().index(True) + + assert result["is_roll_date"].sum() == 1 + raw_move = result["adjusted_close"][roll_index] - result["adjusted_close"][roll_index - 1] + assert raw_move == pytest.approx(11.0) # 10.0 of spread, 1.0 of genuine move + + def test_back_adjusted_roll_day_moves_by_the_old_contract_only(self): + result = self._build(BackAdjustment()) + roll_index = result["is_roll_date"].to_list().index(True) + + move = result["adjusted_close"][roll_index] - result["adjusted_close"][roll_index - 1] + assert move == pytest.approx(1.0) + + def test_ratio_adjusted_roll_day_returns_the_old_contract_return(self): + result = self._build(RatioAdjustment()) + roll_index = result["is_roll_date"].to_list().index(True) + + adjusted_return = ( + result["adjusted_close"][roll_index] / result["adjusted_close"][roll_index - 1] - 1 + ) + # The outgoing contract rose 1 point off its own prior close of 103 on the roll day. + assert adjusted_return == pytest.approx(1.0 / 103.0) + + def test_adjusted_series_ends_on_the_traded_price(self): + """Back-adjustment anchors the most recent bar; only history is shifted.""" + for method in (BackAdjustment(), RatioAdjustment()): + result = self._build(method) + assert result["adjusted_close"][-1] == pytest.approx(result["close"][-1]) diff --git a/tests/test_yahoo_provider.py b/tests/test_yahoo_provider.py index a7a97b1..ed77aac 100644 --- a/tests/test_yahoo_provider.py +++ b/tests/test_yahoo_provider.py @@ -6,6 +6,7 @@ import polars as pl import pytest +from ml4t.data.core.exceptions import SymbolNotFoundError from ml4t.data.providers.yahoo import YahooFinanceProvider @@ -215,3 +216,69 @@ def test_frequency_mapping(self, mock_download: MagicMock) -> None: # Check that the correct interval was used call_kwargs = mock_download.call_args[1] assert call_kwargs["interval"] == expected_interval + + +class TestYahooCurrentSessionPlaceholderBar: + """From the US close until Yahoo consolidates the daily bar it returns a row with + volume and no prices. The single-symbol and batch paths must agree about that row.""" + + @staticmethod + def _frame_with_placeholder() -> pd.DataFrame: + """The frame observed on 2026-09-03: a final row carrying volume and NaN OHLC.""" + return pd.DataFrame( + { + ("Close", "AAPL"): [324.959991, float("nan")], + ("High", "AAPL"): [328.399994, float("nan")], + ("Low", "AAPL"): [323.529999, float("nan")], + ("Open", "AAPL"): [326.869995, float("nan")], + ("Volume", "AAPL"): [33776400, 37197362], + }, + index=pd.DatetimeIndex( + [pd.Timestamp("2026-09-02"), pd.Timestamp("2026-09-03")], name="Date" + ), + ) + + @patch("ml4t.data.providers.yahoo.yf.download") + def test_fetch_ohlcv_drops_the_placeholder_bar(self, mock_download: MagicMock) -> None: + mock_download.return_value = self._frame_with_placeholder() + + df = YahooFinanceProvider().fetch_ohlcv("AAPL", "2026-09-02", "2026-09-03", "daily") + + assert len(df) == 1 + assert df["timestamp"].dt.date().to_list() == [pd.Timestamp("2026-09-02").date()] + assert df["open"].to_list() == [326.869995] + + @patch("ml4t.data.providers.yahoo.yf.download") + def test_fetch_and_batch_agree_on_the_placeholder_bar(self, mock_download: MagicMock) -> None: + """The defect was the disagreement: batch_load succeeded where update() raised.""" + provider = YahooFinanceProvider() + + mock_download.return_value = self._frame_with_placeholder() + single = provider.fetch_ohlcv("AAPL", "2026-09-02", "2026-09-03", "daily") + + mock_download.return_value = self._frame_with_placeholder() + batch = provider.fetch_batch_ohlcv(["AAPL"], "2026-09-02", "2026-09-03", "daily") + + assert single["timestamp"].dt.date().to_list() == batch["timestamp"].dt.date().to_list() + + @patch("ml4t.data.providers.yahoo.yf.download") + def test_a_row_with_prices_and_no_volume_is_kept(self, mock_download: MagicMock) -> None: + """Only a priceless row is a placeholder. A halted session with zero volume is a bar.""" + frame = self._frame_with_placeholder() + frame.loc[pd.Timestamp("2026-09-03")] = [325.0, 325.0, 325.0, 325.0, 0] + mock_download.return_value = frame + + df = YahooFinanceProvider().fetch_ohlcv("AAPL", "2026-09-02", "2026-09-03", "daily") + + assert len(df) == 2 + + @patch("ml4t.data.providers.yahoo.yf.download") + def test_a_response_of_nothing_but_placeholders_is_not_data( + self, mock_download: MagicMock + ) -> None: + """Dropping every row must not report an empty fetch as a successful one.""" + frame = self._frame_with_placeholder().iloc[1:] + mock_download.return_value = frame + + with pytest.raises(SymbolNotFoundError): + YahooFinanceProvider().fetch_ohlcv("AAPL", "2026-09-03", "2026-09-03", "daily") diff --git a/uv.lock b/uv.lock index 3a02c2b..6e86b6a 100644 --- a/uv.lock +++ b/uv.lock @@ -821,14 +821,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.58" +version = "3.1.62" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/26/d6/5f358ff283325580c2003a6d953aea18cfe10ae87b46f5ebc80fa3a386dc/gitpython-3.1.58.tar.gz", hash = "sha256:621416df10ef3fd0e19fabf9172ddeed0fa704d353d04f194eec56a625a95b22", size = 228498, upload-time = "2026-08-04T15:05:49.47Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/db/3ca813cbacb23ab6fe46ff38a9b5ef8e73e970c8051f2ce903aacafe0446/gitpython-3.1.62.tar.gz", hash = "sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af", size = 231728, upload-time = "2026-09-07T02:57:21.155Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/0c/9d8752098bc442f0726e64aa6135940b3a96809915d1aa4206c1bb97881d/gitpython-3.1.58-py3-none-any.whl", hash = "sha256:d331e722577f0fd7fc1f857419b3ecc07af66282b933d2a4d95f84a042fdd50f", size = 220183, upload-time = "2026-08-04T15:05:48.025Z" }, + { url = "https://files.pythonhosted.org/packages/d6/0b/29d7965215f8ef830a7ca1f42997fe13e5693d85e9edb18f938d063ef5f2/gitpython-3.1.62-py3-none-any.whl", hash = "sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e", size = 222753, upload-time = "2026-09-07T02:57:19.762Z" }, ] [[package]]