fix: never fetch delisted yfinance stocks

src/_schemas.py

@@ -117,10 +117,9 @@ class PolymarketFetchFrame(QuestionFrame):
 
 
 class YfinanceFetchFrame(QuestionFrame):
-    """Output of YfinanceSource.fetch(). QuestionFrame plus transient fields for update()."""
+    """Output of YfinanceSource.fetch(). QuestionFrame plus a transient field for update()."""
 
     fetch_datetime: Series[str]
-    probability: Series[object] = pa.Field(nullable=True)
 
 
 class ManifoldFetchFrame(pa.DataFrameModel):

src/sources/yfinance.py

@@ -53,27 +53,47 @@ def fetch(
         """Fetch S&P 500 stock data from Yahoo Finance.
 
         The ticker universe is the union of the current S&P 500 constituents and any tickers
-        already in the question bank. Tickers that are still in the question pool but have dropped
-        out of the S&P 500 and can no longer be fetched are marked resolved (delisted) using their
-        existing question row.
+        already in the question bank, minus the curated nullified (known-delisted) tickers, which
+        are never fetched: they 404 on every run and the noise hides genuinely-new delistings.
+        Nullified tickers still in the pool are carried forward as resolved using their existing
+        question row. Tickers that are still in the pool, have dropped out of the S&P 500, and can
+        no longer be fetched (but are not yet curated as nullified) are likewise marked resolved.
 
         Args:
             dfq (DataFrame[QuestionFrame] | None): Existing question bank.
         """
         top_500 = self._get_sp500_tickers()
         set_top_500 = set(top_500)
         set_current = set(dfq["id"].unique()) if dfq is not None and "id" in dfq.columns else set()
-        all_tickers = list(set_top_500 | set_current)
+
+        # Known-delisted (nullified) tickers 404 on every fetch and only add log noise that masks
+        # genuinely-new delistings. Never fetch them: drop them from the universe up front and
+        # carry their existing question rows forward as resolved (the same row the delisted
+        # heuristic below would have emitted, minus the wasted request and the 404).
+        nullified_ids = self.get_nullified_ids()
+        nullified_in_pool = sorted(set_current & nullified_ids)
+        all_tickers = list((set_top_500 | set_current) - nullified_ids)
 
         logger.info(
-            f"Stock tickers not in top 500 but in current stocks: {set_current - set_top_500}"
+            "Stock tickers not in top 500 but in current stocks (excluding known-delisted): "
+            f"{set_current - set_top_500 - nullified_ids}"
         )
+        if nullified_in_pool:
+            logger.info(
+                f"Skipping fetch for {len(nullified_in_pool)} known-delisted (nullified) tickers; "
+                f"carrying them forward as resolved: {nullified_in_pool}"
+            )
 
         # Pin 'today' once for this run so all downstream date logic is consistent.
         self._today = dates.get_date_today()
         current_time = dates.get_datetime_now()
 
         rows = []
+
+        # Carry forward known-delisted tickers as resolved without hitting the API.
+        for ticker_symbol in nullified_in_pool:
+            rows.append(self._carry_forward_resolved(ticker_symbol, dfq, current_time))
+
         for ticker_symbol in all_tickers:
             time.sleep(1)  # Avoid YFRateLimitError
             company_name, hist = self._fetch_one_stock(ticker_symbol)
@@ -101,7 +121,6 @@ def fetch(
                         "resolved": False,
                         "market_info_resolution_datetime": "N/A",
                         "fetch_datetime": current_time,
-                        "probability": current_price,
                         "forecast_horizons": constants.FORECAST_HORIZONS_IN_DAYS,
                         "freeze_datetime_value": current_price,
                         "freeze_datetime_value_explanation": (
@@ -115,24 +134,39 @@ def fetch(
                 and ticker_symbol in set_current
                 and ticker_symbol not in set_top_500
             ):
-                # Delisted: still in the question pool but no longer fetchable and out of the
-                # S&P 500. Carry forward the existing question row, marked resolved.
-                existing = dfq[dfq["id"] == ticker_symbol].iloc[0].to_dict()
-                existing.update(
-                    {
-                        "resolved": True,
-                        "fetch_datetime": current_time,
-                        "probability": float("nan"),
-                        "freeze_datetime_value": "N/A",
-                    }
-                )
-                rows.append(existing)
+                # Newly delisted: still in the question pool but no longer fetchable and out of
+                # the S&P 500, yet not in the curated nullified list. Carry the existing question
+                # row forward as resolved and warn so it can be added to nullified_questions.
+                rows.append(self._carry_forward_resolved(ticker_symbol, dfq, current_time))
                 logger.warning(
-                    f"{ticker_symbol} detected as delisted (not in S&P 500 and fetch failed)"
+                    f"{ticker_symbol} detected as delisted (not in S&P 500 and fetch failed); "
+                    "consider adding it to nullified_questions"
                 )
 
         return pd.DataFrame(rows)
 
+    @staticmethod
+    def _carry_forward_resolved(ticker_symbol: str, dfq: pd.DataFrame, current_time: str) -> dict:
+        """Return a delisted ticker's existing question row, marked resolved.
+
+        Shared by the curated-nullified skip (before fetch) and the runtime delisted heuristic
+        (fetch returned nothing). freeze_datetime_value gets the delisted marker.
+
+        Args:
+            ticker_symbol (str): Ticker whose existing question row to carry forward.
+            dfq (pd.DataFrame): Existing question bank (must contain ``ticker_symbol``).
+            current_time (str): Fetch timestamp to stamp on the carried-forward row.
+        """
+        existing = dfq[dfq["id"] == ticker_symbol].iloc[0].to_dict()
+        existing.update(
+            {
+                "resolved": True,
+                "fetch_datetime": current_time,
+                "freeze_datetime_value": "N/A",
+            }
+        )
+        return existing
+
     # ------------------------------------------------------------------
     # Public: update
     # ------------------------------------------------------------------
@@ -148,6 +182,10 @@ def update(
     ) -> UpdateResult:
         """Process fetched stock data into updated questions and resolution files.
 
+        Curated nullified (known-delisted) tickers are never sent to the API here either — they
+        404 forever and their final close is fixed — so their resolution files are forward-filled
+        from existing data instead of re-fetched.
+
         Args:
             dfq (DataFrame[QuestionFrame]): Existing questions.
             dff (DataFrame[YfinanceFetchFrame]): Freshly fetched data.
@@ -166,13 +204,21 @@ def update(
         )
 
         renamed_tickers = {entry["original_ticker"] for entry in self.ticker_renames}
+        nullified_ids = self.get_nullified_ids()
 
         for question in dff.to_dict("records"):
             question_id = str(question["id"])
 
             if question_id in renamed_tickers:
                 # Resolution file is rebuilt from the replacement ticker below.
                 logger.info(f"Skipping {question_id} (renamed ticker, handled separately)")
+            elif question_id in nullified_ids:
+                # Known-delisted (nullified): never hit the API (it 404s). The final close is
+                # fixed, so just forward-fill the existing resolution file to yesterday so that
+                # newly-arriving resolution dates still find an exact-date row.
+                df_res = self._forward_fill_existing(existing_resolution_files.get(question_id))
+                if df_res is not None:
+                    resolution_files[question_id] = df_res
             else:
                 df_res = self._build_resolution_df(
                     question=question,
@@ -185,7 +231,6 @@ def update(
 
             # Strip transient fetch-only fields (not part of QuestionFrame)
             del question["fetch_datetime"]
-            del question["probability"]
 
             # Upsert into dfq
             if question["id"] in dfq["id"].values:
@@ -370,6 +415,25 @@ def _finalize_resolution_file(self, df: pd.DataFrame) -> pd.DataFrame:
 
         return df[["id", "date", "value"]].astype(dtype=constants.RESOLUTION_FILE_COLUMN_DTYPE)
 
+    def _forward_fill_existing(self, existing_df: pd.DataFrame | None) -> pd.DataFrame | None:
+        """Forward-fill an existing resolution file to yesterday without fetching.
+
+        For curated nullified (known-delisted) tickers, whose price is fixed and which 404 on the
+        API. Mirrors the resolved path of ``_build_resolution_df`` minus the doomed request.
+
+        Args:
+            existing_df (pd.DataFrame | None): Existing resolution data, or None.
+
+        Returns:
+            The forward-filled DataFrame, or None when there is no existing file or nothing changed.
+        """
+        if existing_df is None or existing_df.empty:
+            return None
+        df_new = self._finalize_resolution_file(existing_df)
+        if existing_df.equals(df_new):
+            return None
+        return df_new
+
     def _build_resolution_df(
         self,
         question: dict,

src/tests/conftest.py

@@ -306,7 +306,6 @@ def make_yfinance_fetch_df(rows):
         "market_info_close_datetime": "N/A",
         "market_info_resolution_datetime": "N/A",
         "fetch_datetime": "2026-03-18T00:00:00+00:00",
-        "probability": 100.0,
     }
     df = pd.DataFrame(rows)
     for col, default in defaults.items():

src/tests/test_yfinance.py

@@ -299,7 +299,7 @@ def test_builds_valid_fetch_frame(
         row = dff[dff["id"] == "AAPL"].iloc[0]
         assert bool(row["resolved"]) is False
         assert row["url"] == "https://finance.yahoo.com/quote/AAPL"
-        assert float(row["probability"]) == 254.23
+        assert float(row["freeze_datetime_value"]) == 254.23
 
     @patch("sources.yfinance.yf.Ticker")
     @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=[])
@@ -316,7 +316,6 @@ def test_delisted_ticker_marked_resolved(
         row = dff[dff["id"] == "OLDCO"].iloc[0]
         assert bool(row["resolved"]) is True
         assert row["freeze_datetime_value"] == "N/A"
-        assert pd.isna(row["probability"])
         assert row["question"] == "legacy question"  # original question text preserved
 
     @patch("sources.yfinance.yf.Ticker")
@@ -356,6 +355,76 @@ def test_not_in_sp500_but_fetch_succeeds_not_resolved(
         assert bool(outco["resolved"]) is False
 
 
+class TestSourceFetchSkipsNullified:
+    """Nullified (known-delisted) tickers are never fetched, only carried forward."""
+
+    @staticmethod
+    def _ok_hist():
+        return pd.DataFrame({"Close": [254.23], "Date": pd.to_datetime(["2026-03-17"])})
+
+    @patch("sources.yfinance.yf.Ticker")
+    @patch.object(YfinanceSource, "_fetch_one_stock")
+    @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL"])
+    def test_nullified_in_pool_skipped_and_carried_forward(
+        self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today
+    ):
+        """A nullified pool ticker is never sent to the API and is carried forward as resolved."""
+        freeze_today(date(2026, 3, 18))
+        mock_fetch_one.return_value = ("Apple Inc.", self._ok_hist())
+        mock_ticker_cls.return_value.info.get.return_value = "N/A"
+
+        dfq = make_question_df(
+            [
+                {"id": "AAPL", "question": "aapl question"},
+                {"id": "ANSS", "question": "legacy ANSS question"},
+            ]
+        )
+        dff = yfinance_source.fetch(dfq=dfq)
+
+        fetched = [call.args[0] for call in mock_fetch_one.call_args_list]
+        assert "ANSS" not in fetched, "nullified ticker must never be fetched"
+        assert "AAPL" in fetched
+
+        anss = dff[dff["id"] == "ANSS"].iloc[0]
+        assert bool(anss["resolved"]) is True
+        assert anss["freeze_datetime_value"] == "N/A"
+        assert anss["question"] == "legacy ANSS question"  # original row preserved
+        YfinanceFetchFrame.validate(dff)  # carry-forward row is schema-valid
+
+    @patch("sources.yfinance.yf.Ticker")
+    @patch.object(YfinanceSource, "_fetch_one_stock")
+    @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL", "ANSS"])
+    def test_nullified_never_fetched_even_if_in_sp500_scrape(
+        self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today
+    ):
+        """A nullified ticker is dropped from the universe even if the S&P 500 scrape lists it."""
+        freeze_today(date(2026, 3, 18))
+        mock_fetch_one.return_value = ("Apple Inc.", self._ok_hist())
+        mock_ticker_cls.return_value.info.get.return_value = "N/A"
+
+        dff = yfinance_source.fetch(dfq=make_question_df([{"id": "AAPL"}]))
+
+        fetched = [call.args[0] for call in mock_fetch_one.call_args_list]
+        assert "ANSS" not in fetched
+        assert "ANSS" not in dff["id"].values  # not in pool -> not carried forward either
+
+    @patch("sources.yfinance.yf.Ticker")
+    @patch.object(YfinanceSource, "_fetch_one_stock")
+    @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL"])
+    def test_carry_forward_does_not_404(
+        self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today
+    ):
+        """Carrying a nullified ticker forward must not invoke yf.Ticker for it (no 404 noise)."""
+        freeze_today(date(2026, 3, 18))
+        mock_fetch_one.return_value = ("Apple Inc.", self._ok_hist())
+        mock_ticker_cls.return_value.info.get.return_value = "N/A"
+
+        yfinance_source.fetch(dfq=make_question_df([{"id": "AAPL"}, {"id": "WBA"}]))
+
+        ticker_calls = [call.args[0] for call in mock_ticker_cls.call_args_list]
+        assert "WBA" not in ticker_calls, "nullified ticker must not be passed to yf.Ticker"
+
+
 class TestSourceBuildResolutionDf:
     """Tests for YfinanceSource._build_resolution_df."""
 
@@ -520,14 +589,19 @@ def fake_fetch(symbol, period):
         assert replacement in seen
 
     @patch.object(YfinanceSource, "_fetch_historical_prices")
-    def test_delisted_stock_preserves_existing_question_fields(
+    def test_nullified_stock_preserves_question_fields_and_is_not_fetched(
         self, mock_fetch, yfinance_source, freeze_today
     ):
-        """Updating a delisted (resolved) ticker keeps its existing question text/background."""
+        """A nullified (HES) ticker keeps its question fields, is never fetched, and is forward
+        filled from its existing resolution file."""
         freeze_today(date(2026, 3, 18))
-        mock_fetch.return_value = pd.DataFrame(
-            {"date": pd.to_datetime(["2026-03-13"]), "value": [149.0]}
-        )
+        seen = []
+
+        def fake_fetch(symbol, period):
+            seen.append(symbol)
+            return pd.DataFrame({"date": pd.to_datetime(["2026-03-13"]), "value": [149.0]})
+
+        mock_fetch.side_effect = fake_fetch
         dfq = make_question_df(
             [{"id": "HES", "question": "Will HES go up?", "background": "Hess Corporation."}]
         )
@@ -539,20 +613,47 @@ def test_delisted_stock_preserves_existing_question_fields(
                     "background": "Hess Corporation.",
                     "resolved": True,
                     "freeze_datetime_value": "N/A",
-                    "probability": float("nan"),
                 }
             ]
         )
+        existing = {
+            "HES": make_resolution_df([{"id": "HES", "date": "2026-03-10", "value": 149.0}])
+        }
 
-        result = yfinance_source.update(dfq, dff)
+        result = yfinance_source.update(dfq, dff, existing_resolution_files=existing)
 
         hes = result.dfq[result.dfq["id"] == "HES"].iloc[0]
         assert bool(hes["resolved"]) is True
         assert hes["question"] == "Will HES go up?"
         assert hes["background"] == "Hess Corporation."
         assert hes["freeze_datetime_value"] == "N/A"
-        # Resolved tickers are forward-filled and uploaded.
+        # Nullified: never fetched (no 404), resolution file forward-filled from existing data.
+        assert "HES" not in seen
         assert "HES" in result.resolution_files
+        out = result.resolution_files["HES"]
+        assert pd.to_datetime(out["date"]).max().date() == date(2026, 3, 17)  # yesterday
+        assert float(out["value"].iloc[-1]) == 149.0  # final close carried forward
+
+    @patch.object(YfinanceSource, "_fetch_historical_prices")
+    def test_nullified_stock_without_existing_file_writes_nothing(
+        self, mock_fetch, yfinance_source, freeze_today
+    ):
+        """A nullified ticker with no existing resolution file is not fetched and writes no file."""
+        freeze_today(date(2026, 3, 18))
+        seen = []
+
+        def fake_fetch(symbol, period):
+            seen.append(symbol)
+            return pd.DataFrame({"date": pd.to_datetime(["2026-03-17"]), "value": [1.0]})
+
+        mock_fetch.side_effect = fake_fetch
+        dfq = make_question_df([{"id": "HES"}])
+        dff = make_yfinance_fetch_df([{"id": "HES", "resolved": True}])
+
+        result = yfinance_source.update(dfq, dff)  # no existing_resolution_files
+
+        assert "HES" not in seen  # never fetched
+        assert "HES" not in result.resolution_files  # nothing to forward-fill
 
 
 class TestSourceFinalizeResolutionFile: