diff --git a/src/_schemas.py b/src/_schemas.py index e9e47e90..8fdc161b 100644 --- a/src/_schemas.py +++ b/src/_schemas.py @@ -117,10 +117,9 @@ class PolymarketFetchFrame(QuestionFrame): class YfinanceFetchFrame(QuestionFrame): - """Output of YfinanceSource.fetch(). QuestionFrame plus transient fields for update().""" + """Output of YfinanceSource.fetch(). QuestionFrame plus a transient field for update().""" fetch_datetime: Series[str] - probability: Series[object] = pa.Field(nullable=True) class ManifoldFetchFrame(pa.DataFrameModel): diff --git a/src/sources/yfinance.py b/src/sources/yfinance.py index e545a1d1..fdb35e2e 100644 --- a/src/sources/yfinance.py +++ b/src/sources/yfinance.py @@ -53,9 +53,11 @@ def fetch( """Fetch S&P 500 stock data from Yahoo Finance. The ticker universe is the union of the current S&P 500 constituents and any tickers - already in the question bank. Tickers that are still in the question pool but have dropped - out of the S&P 500 and can no longer be fetched are marked resolved (delisted) using their - existing question row. + already in the question bank, minus the curated nullified (known-delisted) tickers, which + are never fetched: they 404 on every run and the noise hides genuinely-new delistings. + Nullified tickers still in the pool are carried forward as resolved using their existing + question row. Tickers that are still in the pool, have dropped out of the S&P 500, and can + no longer be fetched (but are not yet curated as nullified) are likewise marked resolved. Args: dfq (DataFrame[QuestionFrame] | None): Existing question bank. @@ -63,17 +65,35 @@ def fetch( top_500 = self._get_sp500_tickers() set_top_500 = set(top_500) set_current = set(dfq["id"].unique()) if dfq is not None and "id" in dfq.columns else set() - all_tickers = list(set_top_500 | set_current) + + # Known-delisted (nullified) tickers 404 on every fetch and only add log noise that masks + # genuinely-new delistings. Never fetch them: drop them from the universe up front and + # carry their existing question rows forward as resolved (the same row the delisted + # heuristic below would have emitted, minus the wasted request and the 404). + nullified_ids = self.get_nullified_ids() + nullified_in_pool = sorted(set_current & nullified_ids) + all_tickers = list((set_top_500 | set_current) - nullified_ids) logger.info( - f"Stock tickers not in top 500 but in current stocks: {set_current - set_top_500}" + "Stock tickers not in top 500 but in current stocks (excluding known-delisted): " + f"{set_current - set_top_500 - nullified_ids}" ) + if nullified_in_pool: + logger.info( + f"Skipping fetch for {len(nullified_in_pool)} known-delisted (nullified) tickers; " + f"carrying them forward as resolved: {nullified_in_pool}" + ) # Pin 'today' once for this run so all downstream date logic is consistent. self._today = dates.get_date_today() current_time = dates.get_datetime_now() rows = [] + + # Carry forward known-delisted tickers as resolved without hitting the API. + for ticker_symbol in nullified_in_pool: + rows.append(self._carry_forward_resolved(ticker_symbol, dfq, current_time)) + for ticker_symbol in all_tickers: time.sleep(1) # Avoid YFRateLimitError company_name, hist = self._fetch_one_stock(ticker_symbol) @@ -101,7 +121,6 @@ def fetch( "resolved": False, "market_info_resolution_datetime": "N/A", "fetch_datetime": current_time, - "probability": current_price, "forecast_horizons": constants.FORECAST_HORIZONS_IN_DAYS, "freeze_datetime_value": current_price, "freeze_datetime_value_explanation": ( @@ -115,24 +134,39 @@ def fetch( and ticker_symbol in set_current and ticker_symbol not in set_top_500 ): - # Delisted: still in the question pool but no longer fetchable and out of the - # S&P 500. Carry forward the existing question row, marked resolved. - existing = dfq[dfq["id"] == ticker_symbol].iloc[0].to_dict() - existing.update( - { - "resolved": True, - "fetch_datetime": current_time, - "probability": float("nan"), - "freeze_datetime_value": "N/A", - } - ) - rows.append(existing) + # Newly delisted: still in the question pool but no longer fetchable and out of + # the S&P 500, yet not in the curated nullified list. Carry the existing question + # row forward as resolved and warn so it can be added to nullified_questions. + rows.append(self._carry_forward_resolved(ticker_symbol, dfq, current_time)) logger.warning( - f"{ticker_symbol} detected as delisted (not in S&P 500 and fetch failed)" + f"{ticker_symbol} detected as delisted (not in S&P 500 and fetch failed); " + "consider adding it to nullified_questions" ) return pd.DataFrame(rows) + @staticmethod + def _carry_forward_resolved(ticker_symbol: str, dfq: pd.DataFrame, current_time: str) -> dict: + """Return a delisted ticker's existing question row, marked resolved. + + Shared by the curated-nullified skip (before fetch) and the runtime delisted heuristic + (fetch returned nothing). freeze_datetime_value gets the delisted marker. + + Args: + ticker_symbol (str): Ticker whose existing question row to carry forward. + dfq (pd.DataFrame): Existing question bank (must contain ``ticker_symbol``). + current_time (str): Fetch timestamp to stamp on the carried-forward row. + """ + existing = dfq[dfq["id"] == ticker_symbol].iloc[0].to_dict() + existing.update( + { + "resolved": True, + "fetch_datetime": current_time, + "freeze_datetime_value": "N/A", + } + ) + return existing + # ------------------------------------------------------------------ # Public: update # ------------------------------------------------------------------ @@ -148,6 +182,10 @@ def update( ) -> UpdateResult: """Process fetched stock data into updated questions and resolution files. + Curated nullified (known-delisted) tickers are never sent to the API here either — they + 404 forever and their final close is fixed — so their resolution files are forward-filled + from existing data instead of re-fetched. + Args: dfq (DataFrame[QuestionFrame]): Existing questions. dff (DataFrame[YfinanceFetchFrame]): Freshly fetched data. @@ -166,6 +204,7 @@ def update( ) renamed_tickers = {entry["original_ticker"] for entry in self.ticker_renames} + nullified_ids = self.get_nullified_ids() for question in dff.to_dict("records"): question_id = str(question["id"]) @@ -173,6 +212,13 @@ def update( if question_id in renamed_tickers: # Resolution file is rebuilt from the replacement ticker below. logger.info(f"Skipping {question_id} (renamed ticker, handled separately)") + elif question_id in nullified_ids: + # Known-delisted (nullified): never hit the API (it 404s). The final close is + # fixed, so just forward-fill the existing resolution file to yesterday so that + # newly-arriving resolution dates still find an exact-date row. + df_res = self._forward_fill_existing(existing_resolution_files.get(question_id)) + if df_res is not None: + resolution_files[question_id] = df_res else: df_res = self._build_resolution_df( question=question, @@ -185,7 +231,6 @@ def update( # Strip transient fetch-only fields (not part of QuestionFrame) del question["fetch_datetime"] - del question["probability"] # Upsert into dfq if question["id"] in dfq["id"].values: @@ -370,6 +415,25 @@ def _finalize_resolution_file(self, df: pd.DataFrame) -> pd.DataFrame: return df[["id", "date", "value"]].astype(dtype=constants.RESOLUTION_FILE_COLUMN_DTYPE) + def _forward_fill_existing(self, existing_df: pd.DataFrame | None) -> pd.DataFrame | None: + """Forward-fill an existing resolution file to yesterday without fetching. + + For curated nullified (known-delisted) tickers, whose price is fixed and which 404 on the + API. Mirrors the resolved path of ``_build_resolution_df`` minus the doomed request. + + Args: + existing_df (pd.DataFrame | None): Existing resolution data, or None. + + Returns: + The forward-filled DataFrame, or None when there is no existing file or nothing changed. + """ + if existing_df is None or existing_df.empty: + return None + df_new = self._finalize_resolution_file(existing_df) + if existing_df.equals(df_new): + return None + return df_new + def _build_resolution_df( self, question: dict, diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 0af0d240..e6d4e9c9 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -306,7 +306,6 @@ def make_yfinance_fetch_df(rows): "market_info_close_datetime": "N/A", "market_info_resolution_datetime": "N/A", "fetch_datetime": "2026-03-18T00:00:00+00:00", - "probability": 100.0, } df = pd.DataFrame(rows) for col, default in defaults.items(): diff --git a/src/tests/test_yfinance.py b/src/tests/test_yfinance.py index de4324ec..c3b04cd3 100644 --- a/src/tests/test_yfinance.py +++ b/src/tests/test_yfinance.py @@ -299,7 +299,7 @@ def test_builds_valid_fetch_frame( row = dff[dff["id"] == "AAPL"].iloc[0] assert bool(row["resolved"]) is False assert row["url"] == "https://finance.yahoo.com/quote/AAPL" - assert float(row["probability"]) == 254.23 + assert float(row["freeze_datetime_value"]) == 254.23 @patch("sources.yfinance.yf.Ticker") @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=[]) @@ -316,7 +316,6 @@ def test_delisted_ticker_marked_resolved( row = dff[dff["id"] == "OLDCO"].iloc[0] assert bool(row["resolved"]) is True assert row["freeze_datetime_value"] == "N/A" - assert pd.isna(row["probability"]) assert row["question"] == "legacy question" # original question text preserved @patch("sources.yfinance.yf.Ticker") @@ -356,6 +355,76 @@ def test_not_in_sp500_but_fetch_succeeds_not_resolved( assert bool(outco["resolved"]) is False +class TestSourceFetchSkipsNullified: + """Nullified (known-delisted) tickers are never fetched, only carried forward.""" + + @staticmethod + def _ok_hist(): + return pd.DataFrame({"Close": [254.23], "Date": pd.to_datetime(["2026-03-17"])}) + + @patch("sources.yfinance.yf.Ticker") + @patch.object(YfinanceSource, "_fetch_one_stock") + @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL"]) + def test_nullified_in_pool_skipped_and_carried_forward( + self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today + ): + """A nullified pool ticker is never sent to the API and is carried forward as resolved.""" + freeze_today(date(2026, 3, 18)) + mock_fetch_one.return_value = ("Apple Inc.", self._ok_hist()) + mock_ticker_cls.return_value.info.get.return_value = "N/A" + + dfq = make_question_df( + [ + {"id": "AAPL", "question": "aapl question"}, + {"id": "ANSS", "question": "legacy ANSS question"}, + ] + ) + dff = yfinance_source.fetch(dfq=dfq) + + fetched = [call.args[0] for call in mock_fetch_one.call_args_list] + assert "ANSS" not in fetched, "nullified ticker must never be fetched" + assert "AAPL" in fetched + + anss = dff[dff["id"] == "ANSS"].iloc[0] + assert bool(anss["resolved"]) is True + assert anss["freeze_datetime_value"] == "N/A" + assert anss["question"] == "legacy ANSS question" # original row preserved + YfinanceFetchFrame.validate(dff) # carry-forward row is schema-valid + + @patch("sources.yfinance.yf.Ticker") + @patch.object(YfinanceSource, "_fetch_one_stock") + @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL", "ANSS"]) + def test_nullified_never_fetched_even_if_in_sp500_scrape( + self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today + ): + """A nullified ticker is dropped from the universe even if the S&P 500 scrape lists it.""" + freeze_today(date(2026, 3, 18)) + mock_fetch_one.return_value = ("Apple Inc.", self._ok_hist()) + mock_ticker_cls.return_value.info.get.return_value = "N/A" + + dff = yfinance_source.fetch(dfq=make_question_df([{"id": "AAPL"}])) + + fetched = [call.args[0] for call in mock_fetch_one.call_args_list] + assert "ANSS" not in fetched + assert "ANSS" not in dff["id"].values # not in pool -> not carried forward either + + @patch("sources.yfinance.yf.Ticker") + @patch.object(YfinanceSource, "_fetch_one_stock") + @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL"]) + def test_carry_forward_does_not_404( + self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today + ): + """Carrying a nullified ticker forward must not invoke yf.Ticker for it (no 404 noise).""" + freeze_today(date(2026, 3, 18)) + mock_fetch_one.return_value = ("Apple Inc.", self._ok_hist()) + mock_ticker_cls.return_value.info.get.return_value = "N/A" + + yfinance_source.fetch(dfq=make_question_df([{"id": "AAPL"}, {"id": "WBA"}])) + + ticker_calls = [call.args[0] for call in mock_ticker_cls.call_args_list] + assert "WBA" not in ticker_calls, "nullified ticker must not be passed to yf.Ticker" + + class TestSourceBuildResolutionDf: """Tests for YfinanceSource._build_resolution_df.""" @@ -520,14 +589,19 @@ def fake_fetch(symbol, period): assert replacement in seen @patch.object(YfinanceSource, "_fetch_historical_prices") - def test_delisted_stock_preserves_existing_question_fields( + def test_nullified_stock_preserves_question_fields_and_is_not_fetched( self, mock_fetch, yfinance_source, freeze_today ): - """Updating a delisted (resolved) ticker keeps its existing question text/background.""" + """A nullified (HES) ticker keeps its question fields, is never fetched, and is forward + filled from its existing resolution file.""" freeze_today(date(2026, 3, 18)) - mock_fetch.return_value = pd.DataFrame( - {"date": pd.to_datetime(["2026-03-13"]), "value": [149.0]} - ) + seen = [] + + def fake_fetch(symbol, period): + seen.append(symbol) + return pd.DataFrame({"date": pd.to_datetime(["2026-03-13"]), "value": [149.0]}) + + mock_fetch.side_effect = fake_fetch dfq = make_question_df( [{"id": "HES", "question": "Will HES go up?", "background": "Hess Corporation."}] ) @@ -539,20 +613,47 @@ def test_delisted_stock_preserves_existing_question_fields( "background": "Hess Corporation.", "resolved": True, "freeze_datetime_value": "N/A", - "probability": float("nan"), } ] ) + existing = { + "HES": make_resolution_df([{"id": "HES", "date": "2026-03-10", "value": 149.0}]) + } - result = yfinance_source.update(dfq, dff) + result = yfinance_source.update(dfq, dff, existing_resolution_files=existing) hes = result.dfq[result.dfq["id"] == "HES"].iloc[0] assert bool(hes["resolved"]) is True assert hes["question"] == "Will HES go up?" assert hes["background"] == "Hess Corporation." assert hes["freeze_datetime_value"] == "N/A" - # Resolved tickers are forward-filled and uploaded. + # Nullified: never fetched (no 404), resolution file forward-filled from existing data. + assert "HES" not in seen assert "HES" in result.resolution_files + out = result.resolution_files["HES"] + assert pd.to_datetime(out["date"]).max().date() == date(2026, 3, 17) # yesterday + assert float(out["value"].iloc[-1]) == 149.0 # final close carried forward + + @patch.object(YfinanceSource, "_fetch_historical_prices") + def test_nullified_stock_without_existing_file_writes_nothing( + self, mock_fetch, yfinance_source, freeze_today + ): + """A nullified ticker with no existing resolution file is not fetched and writes no file.""" + freeze_today(date(2026, 3, 18)) + seen = [] + + def fake_fetch(symbol, period): + seen.append(symbol) + return pd.DataFrame({"date": pd.to_datetime(["2026-03-17"]), "value": [1.0]}) + + mock_fetch.side_effect = fake_fetch + dfq = make_question_df([{"id": "HES"}]) + dff = make_yfinance_fetch_df([{"id": "HES", "resolved": True}]) + + result = yfinance_source.update(dfq, dff) # no existing_resolution_files + + assert "HES" not in seen # never fetched + assert "HES" not in result.resolution_files # nothing to forward-fill class TestSourceFinalizeResolutionFile: