From 563ac59ba95d624dd1efd4c22183956558bb8b51 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 18 Jun 2026 10:46:40 +0300 Subject: [PATCH 1/4] fix: delete in yfinance fetch --- src/_schemas.py | 3 +-- src/sources/yfinance.py | 3 --- src/tests/conftest.py | 1 - src/tests/test_yfinance.py | 4 +--- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/_schemas.py b/src/_schemas.py index e9e47e90..8fdc161b 100644 --- a/src/_schemas.py +++ b/src/_schemas.py @@ -117,10 +117,9 @@ class PolymarketFetchFrame(QuestionFrame): class YfinanceFetchFrame(QuestionFrame): - """Output of YfinanceSource.fetch(). QuestionFrame plus transient fields for update().""" + """Output of YfinanceSource.fetch(). QuestionFrame plus a transient field for update().""" fetch_datetime: Series[str] - probability: Series[object] = pa.Field(nullable=True) class ManifoldFetchFrame(pa.DataFrameModel): diff --git a/src/sources/yfinance.py b/src/sources/yfinance.py index e545a1d1..f430eed7 100644 --- a/src/sources/yfinance.py +++ b/src/sources/yfinance.py @@ -101,7 +101,6 @@ def fetch( "resolved": False, "market_info_resolution_datetime": "N/A", "fetch_datetime": current_time, - "probability": current_price, "forecast_horizons": constants.FORECAST_HORIZONS_IN_DAYS, "freeze_datetime_value": current_price, "freeze_datetime_value_explanation": ( @@ -122,7 +121,6 @@ def fetch( { "resolved": True, "fetch_datetime": current_time, - "probability": float("nan"), "freeze_datetime_value": "N/A", } ) @@ -185,7 +183,6 @@ def update( # Strip transient fetch-only fields (not part of QuestionFrame) del question["fetch_datetime"] - del question["probability"] # Upsert into dfq if question["id"] in dfq["id"].values: diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 0af0d240..e6d4e9c9 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -306,7 +306,6 @@ def make_yfinance_fetch_df(rows): "market_info_close_datetime": "N/A", "market_info_resolution_datetime": "N/A", "fetch_datetime": "2026-03-18T00:00:00+00:00", - "probability": 100.0, } df = pd.DataFrame(rows) for col, default in defaults.items(): diff --git a/src/tests/test_yfinance.py b/src/tests/test_yfinance.py index de4324ec..6aa17a86 100644 --- a/src/tests/test_yfinance.py +++ b/src/tests/test_yfinance.py @@ -299,7 +299,7 @@ def test_builds_valid_fetch_frame( row = dff[dff["id"] == "AAPL"].iloc[0] assert bool(row["resolved"]) is False assert row["url"] == "https://finance.yahoo.com/quote/AAPL" - assert float(row["probability"]) == 254.23 + assert float(row["freeze_datetime_value"]) == 254.23 @patch("sources.yfinance.yf.Ticker") @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=[]) @@ -316,7 +316,6 @@ def test_delisted_ticker_marked_resolved( row = dff[dff["id"] == "OLDCO"].iloc[0] assert bool(row["resolved"]) is True assert row["freeze_datetime_value"] == "N/A" - assert pd.isna(row["probability"]) assert row["question"] == "legacy question" # original question text preserved @patch("sources.yfinance.yf.Ticker") @@ -539,7 +538,6 @@ def test_delisted_stock_preserves_existing_question_fields( "background": "Hess Corporation.", "resolved": True, "freeze_datetime_value": "N/A", - "probability": float("nan"), } ] ) From f6ec0880ea5815370c5b0784a8fa762931378ce4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 18 Jun 2026 11:39:27 +0300 Subject: [PATCH 2/4] fix: never fetch delisted yfinance stocks --- src/sources/yfinance.py | 101 ++++++++++++++++++++++++++------ src/tests/test_yfinance.py | 117 ++++++++++++++++++++++++++++++++++--- 2 files changed, 194 insertions(+), 24 deletions(-) diff --git a/src/sources/yfinance.py b/src/sources/yfinance.py index f430eed7..fdb35e2e 100644 --- a/src/sources/yfinance.py +++ b/src/sources/yfinance.py @@ -53,9 +53,11 @@ def fetch( """Fetch S&P 500 stock data from Yahoo Finance. The ticker universe is the union of the current S&P 500 constituents and any tickers - already in the question bank. Tickers that are still in the question pool but have dropped - out of the S&P 500 and can no longer be fetched are marked resolved (delisted) using their - existing question row. + already in the question bank, minus the curated nullified (known-delisted) tickers, which + are never fetched: they 404 on every run and the noise hides genuinely-new delistings. + Nullified tickers still in the pool are carried forward as resolved using their existing + question row. Tickers that are still in the pool, have dropped out of the S&P 500, and can + no longer be fetched (but are not yet curated as nullified) are likewise marked resolved. Args: dfq (DataFrame[QuestionFrame] | None): Existing question bank. @@ -63,17 +65,35 @@ def fetch( top_500 = self._get_sp500_tickers() set_top_500 = set(top_500) set_current = set(dfq["id"].unique()) if dfq is not None and "id" in dfq.columns else set() - all_tickers = list(set_top_500 | set_current) + + # Known-delisted (nullified) tickers 404 on every fetch and only add log noise that masks + # genuinely-new delistings. Never fetch them: drop them from the universe up front and + # carry their existing question rows forward as resolved (the same row the delisted + # heuristic below would have emitted, minus the wasted request and the 404). + nullified_ids = self.get_nullified_ids() + nullified_in_pool = sorted(set_current & nullified_ids) + all_tickers = list((set_top_500 | set_current) - nullified_ids) logger.info( - f"Stock tickers not in top 500 but in current stocks: {set_current - set_top_500}" + "Stock tickers not in top 500 but in current stocks (excluding known-delisted): " + f"{set_current - set_top_500 - nullified_ids}" ) + if nullified_in_pool: + logger.info( + f"Skipping fetch for {len(nullified_in_pool)} known-delisted (nullified) tickers; " + f"carrying them forward as resolved: {nullified_in_pool}" + ) # Pin 'today' once for this run so all downstream date logic is consistent. self._today = dates.get_date_today() current_time = dates.get_datetime_now() rows = [] + + # Carry forward known-delisted tickers as resolved without hitting the API. + for ticker_symbol in nullified_in_pool: + rows.append(self._carry_forward_resolved(ticker_symbol, dfq, current_time)) + for ticker_symbol in all_tickers: time.sleep(1) # Avoid YFRateLimitError company_name, hist = self._fetch_one_stock(ticker_symbol) @@ -114,23 +134,39 @@ def fetch( and ticker_symbol in set_current and ticker_symbol not in set_top_500 ): - # Delisted: still in the question pool but no longer fetchable and out of the - # S&P 500. Carry forward the existing question row, marked resolved. - existing = dfq[dfq["id"] == ticker_symbol].iloc[0].to_dict() - existing.update( - { - "resolved": True, - "fetch_datetime": current_time, - "freeze_datetime_value": "N/A", - } - ) - rows.append(existing) + # Newly delisted: still in the question pool but no longer fetchable and out of + # the S&P 500, yet not in the curated nullified list. Carry the existing question + # row forward as resolved and warn so it can be added to nullified_questions. + rows.append(self._carry_forward_resolved(ticker_symbol, dfq, current_time)) logger.warning( - f"{ticker_symbol} detected as delisted (not in S&P 500 and fetch failed)" + f"{ticker_symbol} detected as delisted (not in S&P 500 and fetch failed); " + "consider adding it to nullified_questions" ) return pd.DataFrame(rows) + @staticmethod + def _carry_forward_resolved(ticker_symbol: str, dfq: pd.DataFrame, current_time: str) -> dict: + """Return a delisted ticker's existing question row, marked resolved. + + Shared by the curated-nullified skip (before fetch) and the runtime delisted heuristic + (fetch returned nothing). freeze_datetime_value gets the delisted marker. + + Args: + ticker_symbol (str): Ticker whose existing question row to carry forward. + dfq (pd.DataFrame): Existing question bank (must contain ``ticker_symbol``). + current_time (str): Fetch timestamp to stamp on the carried-forward row. + """ + existing = dfq[dfq["id"] == ticker_symbol].iloc[0].to_dict() + existing.update( + { + "resolved": True, + "fetch_datetime": current_time, + "freeze_datetime_value": "N/A", + } + ) + return existing + # ------------------------------------------------------------------ # Public: update # ------------------------------------------------------------------ @@ -146,6 +182,10 @@ def update( ) -> UpdateResult: """Process fetched stock data into updated questions and resolution files. + Curated nullified (known-delisted) tickers are never sent to the API here either — they + 404 forever and their final close is fixed — so their resolution files are forward-filled + from existing data instead of re-fetched. + Args: dfq (DataFrame[QuestionFrame]): Existing questions. dff (DataFrame[YfinanceFetchFrame]): Freshly fetched data. @@ -164,6 +204,7 @@ def update( ) renamed_tickers = {entry["original_ticker"] for entry in self.ticker_renames} + nullified_ids = self.get_nullified_ids() for question in dff.to_dict("records"): question_id = str(question["id"]) @@ -171,6 +212,13 @@ def update( if question_id in renamed_tickers: # Resolution file is rebuilt from the replacement ticker below. logger.info(f"Skipping {question_id} (renamed ticker, handled separately)") + elif question_id in nullified_ids: + # Known-delisted (nullified): never hit the API (it 404s). The final close is + # fixed, so just forward-fill the existing resolution file to yesterday so that + # newly-arriving resolution dates still find an exact-date row. + df_res = self._forward_fill_existing(existing_resolution_files.get(question_id)) + if df_res is not None: + resolution_files[question_id] = df_res else: df_res = self._build_resolution_df( question=question, @@ -367,6 +415,25 @@ def _finalize_resolution_file(self, df: pd.DataFrame) -> pd.DataFrame: return df[["id", "date", "value"]].astype(dtype=constants.RESOLUTION_FILE_COLUMN_DTYPE) + def _forward_fill_existing(self, existing_df: pd.DataFrame | None) -> pd.DataFrame | None: + """Forward-fill an existing resolution file to yesterday without fetching. + + For curated nullified (known-delisted) tickers, whose price is fixed and which 404 on the + API. Mirrors the resolved path of ``_build_resolution_df`` minus the doomed request. + + Args: + existing_df (pd.DataFrame | None): Existing resolution data, or None. + + Returns: + The forward-filled DataFrame, or None when there is no existing file or nothing changed. + """ + if existing_df is None or existing_df.empty: + return None + df_new = self._finalize_resolution_file(existing_df) + if existing_df.equals(df_new): + return None + return df_new + def _build_resolution_df( self, question: dict, diff --git a/src/tests/test_yfinance.py b/src/tests/test_yfinance.py index 6aa17a86..c3b04cd3 100644 --- a/src/tests/test_yfinance.py +++ b/src/tests/test_yfinance.py @@ -355,6 +355,76 @@ def test_not_in_sp500_but_fetch_succeeds_not_resolved( assert bool(outco["resolved"]) is False +class TestSourceFetchSkipsNullified: + """Nullified (known-delisted) tickers are never fetched, only carried forward.""" + + @staticmethod + def _ok_hist(): + return pd.DataFrame({"Close": [254.23], "Date": pd.to_datetime(["2026-03-17"])}) + + @patch("sources.yfinance.yf.Ticker") + @patch.object(YfinanceSource, "_fetch_one_stock") + @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL"]) + def test_nullified_in_pool_skipped_and_carried_forward( + self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today + ): + """A nullified pool ticker is never sent to the API and is carried forward as resolved.""" + freeze_today(date(2026, 3, 18)) + mock_fetch_one.return_value = ("Apple Inc.", self._ok_hist()) + mock_ticker_cls.return_value.info.get.return_value = "N/A" + + dfq = make_question_df( + [ + {"id": "AAPL", "question": "aapl question"}, + {"id": "ANSS", "question": "legacy ANSS question"}, + ] + ) + dff = yfinance_source.fetch(dfq=dfq) + + fetched = [call.args[0] for call in mock_fetch_one.call_args_list] + assert "ANSS" not in fetched, "nullified ticker must never be fetched" + assert "AAPL" in fetched + + anss = dff[dff["id"] == "ANSS"].iloc[0] + assert bool(anss["resolved"]) is True + assert anss["freeze_datetime_value"] == "N/A" + assert anss["question"] == "legacy ANSS question" # original row preserved + YfinanceFetchFrame.validate(dff) # carry-forward row is schema-valid + + @patch("sources.yfinance.yf.Ticker") + @patch.object(YfinanceSource, "_fetch_one_stock") + @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL", "ANSS"]) + def test_nullified_never_fetched_even_if_in_sp500_scrape( + self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today + ): + """A nullified ticker is dropped from the universe even if the S&P 500 scrape lists it.""" + freeze_today(date(2026, 3, 18)) + mock_fetch_one.return_value = ("Apple Inc.", self._ok_hist()) + mock_ticker_cls.return_value.info.get.return_value = "N/A" + + dff = yfinance_source.fetch(dfq=make_question_df([{"id": "AAPL"}])) + + fetched = [call.args[0] for call in mock_fetch_one.call_args_list] + assert "ANSS" not in fetched + assert "ANSS" not in dff["id"].values # not in pool -> not carried forward either + + @patch("sources.yfinance.yf.Ticker") + @patch.object(YfinanceSource, "_fetch_one_stock") + @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL"]) + def test_carry_forward_does_not_404( + self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today + ): + """Carrying a nullified ticker forward must not invoke yf.Ticker for it (no 404 noise).""" + freeze_today(date(2026, 3, 18)) + mock_fetch_one.return_value = ("Apple Inc.", self._ok_hist()) + mock_ticker_cls.return_value.info.get.return_value = "N/A" + + yfinance_source.fetch(dfq=make_question_df([{"id": "AAPL"}, {"id": "WBA"}])) + + ticker_calls = [call.args[0] for call in mock_ticker_cls.call_args_list] + assert "WBA" not in ticker_calls, "nullified ticker must not be passed to yf.Ticker" + + class TestSourceBuildResolutionDf: """Tests for YfinanceSource._build_resolution_df.""" @@ -519,14 +589,19 @@ def fake_fetch(symbol, period): assert replacement in seen @patch.object(YfinanceSource, "_fetch_historical_prices") - def test_delisted_stock_preserves_existing_question_fields( + def test_nullified_stock_preserves_question_fields_and_is_not_fetched( self, mock_fetch, yfinance_source, freeze_today ): - """Updating a delisted (resolved) ticker keeps its existing question text/background.""" + """A nullified (HES) ticker keeps its question fields, is never fetched, and is forward + filled from its existing resolution file.""" freeze_today(date(2026, 3, 18)) - mock_fetch.return_value = pd.DataFrame( - {"date": pd.to_datetime(["2026-03-13"]), "value": [149.0]} - ) + seen = [] + + def fake_fetch(symbol, period): + seen.append(symbol) + return pd.DataFrame({"date": pd.to_datetime(["2026-03-13"]), "value": [149.0]}) + + mock_fetch.side_effect = fake_fetch dfq = make_question_df( [{"id": "HES", "question": "Will HES go up?", "background": "Hess Corporation."}] ) @@ -541,16 +616,44 @@ def test_delisted_stock_preserves_existing_question_fields( } ] ) + existing = { + "HES": make_resolution_df([{"id": "HES", "date": "2026-03-10", "value": 149.0}]) + } - result = yfinance_source.update(dfq, dff) + result = yfinance_source.update(dfq, dff, existing_resolution_files=existing) hes = result.dfq[result.dfq["id"] == "HES"].iloc[0] assert bool(hes["resolved"]) is True assert hes["question"] == "Will HES go up?" assert hes["background"] == "Hess Corporation." assert hes["freeze_datetime_value"] == "N/A" - # Resolved tickers are forward-filled and uploaded. + # Nullified: never fetched (no 404), resolution file forward-filled from existing data. + assert "HES" not in seen assert "HES" in result.resolution_files + out = result.resolution_files["HES"] + assert pd.to_datetime(out["date"]).max().date() == date(2026, 3, 17) # yesterday + assert float(out["value"].iloc[-1]) == 149.0 # final close carried forward + + @patch.object(YfinanceSource, "_fetch_historical_prices") + def test_nullified_stock_without_existing_file_writes_nothing( + self, mock_fetch, yfinance_source, freeze_today + ): + """A nullified ticker with no existing resolution file is not fetched and writes no file.""" + freeze_today(date(2026, 3, 18)) + seen = [] + + def fake_fetch(symbol, period): + seen.append(symbol) + return pd.DataFrame({"date": pd.to_datetime(["2026-03-17"]), "value": [1.0]}) + + mock_fetch.side_effect = fake_fetch + dfq = make_question_df([{"id": "HES"}]) + dff = make_yfinance_fetch_df([{"id": "HES", "resolved": True}]) + + result = yfinance_source.update(dfq, dff) # no existing_resolution_files + + assert "HES" not in seen # never fetched + assert "HES" not in result.resolution_files # nothing to forward-fill class TestSourceFinalizeResolutionFile: From 40f974be991770d5a0c8faf2ed9fa54d6cb96cb4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 18 Jun 2026 16:27:57 +0300 Subject: [PATCH 3/4] fix: force renamed tickers files' to be identical by constructions; avoid fetching the old ticker name unnecessarily --- src/sources/yfinance.py | 82 ++++++++++++++++++++++++++------------ src/tests/test_yfinance.py | 67 +++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 26 deletions(-) diff --git a/src/sources/yfinance.py b/src/sources/yfinance.py index fdb35e2e..c4c76c28 100644 --- a/src/sources/yfinance.py +++ b/src/sources/yfinance.py @@ -53,11 +53,13 @@ def fetch( """Fetch S&P 500 stock data from Yahoo Finance. The ticker universe is the union of the current S&P 500 constituents and any tickers - already in the question bank, minus the curated nullified (known-delisted) tickers, which - are never fetched: they 404 on every run and the noise hides genuinely-new delistings. - Nullified tickers still in the pool are carried forward as resolved using their existing - question row. Tickers that are still in the pool, have dropped out of the S&P 500, and can - no longer be fetched (but are not yet curated as nullified) are likewise marked resolved. + already in the question bank, minus the tickers that are known to 404 on every run: + curated nullified (known-delisted) tickers and renamed originals (whose data is served + under their replacement symbol). Those are never fetched; the noise would only hide + genuinely-new delistings. Any of them still in the pool are carried forward as resolved + using their existing question row. Tickers that are still in the pool, have dropped out of + the S&P 500, and can no longer be fetched (but are not yet curated) are likewise marked + resolved. Args: dfq (DataFrame[QuestionFrame] | None): Existing question bank. @@ -66,23 +68,34 @@ def fetch( set_top_500 = set(top_500) set_current = set(dfq["id"].unique()) if dfq is not None and "id" in dfq.columns else set() - # Known-delisted (nullified) tickers 404 on every fetch and only add log noise that masks - # genuinely-new delistings. Never fetch them: drop them from the universe up front and - # carry their existing question rows forward as resolved (the same row the delisted - # heuristic below would have emitted, minus the wasted request and the 404). + # Tickers we never fetch because they 404 on every run and only add log noise that masks + # genuinely-new delistings: curated nullified (known-delisted) tickers, and renamed + # originals (their price data is served under the replacement symbol; update() rebuilds + # their resolution file from it). Drop both from the universe up front and carry their + # existing question rows forward as resolved. nullified_ids = self.get_nullified_ids() + renamed_original_ids = {entry["original_ticker"] for entry in self.ticker_renames} + skip_fetch_ids = nullified_ids | renamed_original_ids + all_tickers = list((set_top_500 | set_current) - skip_fetch_ids) + nullified_in_pool = sorted(set_current & nullified_ids) - all_tickers = list((set_top_500 | set_current) - nullified_ids) + renamed_in_pool = sorted(set_current & renamed_original_ids) + carry_forward_ids = sorted(set_current & skip_fetch_ids) logger.info( - "Stock tickers not in top 500 but in current stocks (excluding known-delisted): " - f"{set_current - set_top_500 - nullified_ids}" + "Stock tickers not in top 500 but in current stocks (excluding known-unfetchable): " + f"{set_current - set_top_500 - skip_fetch_ids}" ) if nullified_in_pool: logger.info( f"Skipping fetch for {len(nullified_in_pool)} known-delisted (nullified) tickers; " f"carrying them forward as resolved: {nullified_in_pool}" ) + if renamed_in_pool: + logger.info( + f"Skipping fetch for {len(renamed_in_pool)} renamed-original tickers (data comes " + f"via their replacement); carrying them forward as resolved: {renamed_in_pool}" + ) # Pin 'today' once for this run so all downstream date logic is consistent. self._today = dates.get_date_today() @@ -90,8 +103,8 @@ def fetch( rows = [] - # Carry forward known-delisted tickers as resolved without hitting the API. - for ticker_symbol in nullified_in_pool: + # Carry forward known-unfetchable tickers (nullified + renamed originals) without the API. + for ticker_symbol in carry_forward_ids: rows.append(self._carry_forward_resolved(ticker_symbol, dfq, current_time)) for ticker_symbol in all_tickers: @@ -242,9 +255,12 @@ def update( new_q_row = new_q_row.astype(constants.QUESTION_FILE_COLUMN_DTYPE) dfq = pd.concat([dfq, new_q_row], ignore_index=True) - # Renamed tickers: fetch under the replacement, write under the original ticker. + # Renamed tickers: write the original ticker's file as a copy of the replacement's + # already-built (or existing) series so the two files are identical by construction. resolution_files.update( - self._build_renamed_ticker_resolution_files(period, existing_resolution_files) + self._build_renamed_ticker_resolution_files( + period, existing_resolution_files, resolution_files + ) ) return UpdateResult( @@ -485,16 +501,24 @@ def _build_renamed_ticker_resolution_files( self, period: str, existing_resolution_files: dict[str, pd.DataFrame], + built_resolution_files: dict[str, pd.DataFrame], ) -> dict[str, pd.DataFrame]: - """Build resolution files for renamed tickers using their replacement symbols. + """Build resolution files for renamed tickers as a copy of their replacement's series. - For each entry in ``self.ticker_renames``, fetch price history under the replacement - ticker and write it to a resolution file keyed by the original ticker. + For each entry in ``self.ticker_renames``, the original ticker's resolution file is written + as a relabelled copy of the **replacement's** series, so the two files are identical by + construction (no second fetch that could diverge). The authoritative replacement series is + "what ``.jsonl`` will be after this run": the freshly-built frame if the + replacement was built in this run's main loop, else the existing on-disk file. Only when the + replacement isn't in this run's pool at all do we fetch it directly (there is then no + ``.jsonl`` to diverge from). Args: period (str): yfinance period string. existing_resolution_files (dict): Existing resolution data, keyed by question id; must - include the original tickers. + include the original tickers (and the replacements, when known). + built_resolution_files (dict): Resolution frames already built this run (the main + loop's output), keyed by question id. Returns: Mapping of original ticker -> resolution DataFrame, only for files that changed. @@ -506,22 +530,28 @@ def _build_renamed_ticker_resolution_files( existing_df = existing_resolution_files.get(original) - df_new = self._get_historical_prices(existing_df, replacement, period) - if df_new is None: + # Authoritative replacement series: freshly built this run, else the on-disk file, + # else fetch directly (replacement not in this run's pool, so nothing to diverge from). + repl_series = built_resolution_files.get(replacement) + if repl_series is None: + repl_series = existing_resolution_files.get(replacement) + if repl_series is None: + repl_series = self._get_historical_prices(existing_df, replacement, period) + + if repl_series is None or repl_series.empty: logger.warning( f"No data for replacement ticker {replacement} (original: {original})" ) continue - # df_new may be `existing_df` (fetch returned nothing); copy before relabelling so the - # caller's resolution file is never mutated in place. - df_new = df_new.copy() + # Copy before relabelling so neither the built map nor the caller's file is mutated. + df_new = repl_series.copy() df_new["id"] = original if existing_df is not None and not existing_df.empty and existing_df.equals(df_new): continue - logger.info(f"Built resolution file for {original} (via {replacement})") + logger.info(f"Built resolution file for {original} as a copy of {replacement}") resolution_files[original] = df_new return resolution_files diff --git a/src/tests/test_yfinance.py b/src/tests/test_yfinance.py index c3b04cd3..9bcbff99 100644 --- a/src/tests/test_yfinance.py +++ b/src/tests/test_yfinance.py @@ -425,6 +425,36 @@ def test_carry_forward_does_not_404( assert "WBA" not in ticker_calls, "nullified ticker must not be passed to yf.Ticker" +class TestSourceFetchSkipsRenamed: + """Renamed originals are never fetched; only their replacement is.""" + + @patch("sources.yfinance.yf.Ticker") + @patch.object(YfinanceSource, "_fetch_one_stock") + def test_renamed_original_skipped_and_carried_forward( + self, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today + ): + """The renamed original (e.g. FI) is not fetched and is carried forward as resolved; its + replacement (e.g. FISV) is still fetched normally.""" + freeze_today(date(2026, 3, 18)) + original = yfinance_source.ticker_renames[0]["original_ticker"] # FI + replacement = yfinance_source.ticker_renames[0]["replacement_ticker"] # FISV + hist = pd.DataFrame({"Close": [254.23], "Date": pd.to_datetime(["2026-03-17"])}) + mock_fetch_one.return_value = ("Some Co", hist) + mock_ticker_cls.return_value.info.get.return_value = "N/A" + + with patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL", replacement]): + dfq = make_question_df([{"id": "AAPL"}, {"id": original}, {"id": replacement}]) + dff = yfinance_source.fetch(dfq=dfq) + + fetched = [call.args[0] for call in mock_fetch_one.call_args_list] + assert original not in fetched, "renamed original must never be fetched" + assert replacement in fetched, "the live replacement must still be fetched" + + fi = dff[dff["id"] == original].iloc[0] + assert bool(fi["resolved"]) is True + assert fi["freeze_datetime_value"] == "N/A" + + class TestSourceBuildResolutionDf: """Tests for YfinanceSource._build_resolution_df.""" @@ -588,6 +618,43 @@ def fake_fetch(symbol, period): assert original not in seen # original symbol never fetched directly assert replacement in seen + @patch.object(YfinanceSource, "_fetch_historical_prices") + def test_renamed_original_is_exact_copy_of_replacement( + self, mock_fetch, yfinance_source, freeze_today + ): + """When the replacement is in the pool, the original's resolution file is a relabelled copy + of the replacement's series (identical date/value), and the replacement is fetched once.""" + freeze_today(date(2026, 3, 18)) + original = yfinance_source.ticker_renames[0]["original_ticker"] # FI + replacement = yfinance_source.ticker_renames[0]["replacement_ticker"] # FISV + + seen = [] + + def fake_fetch(symbol, period): + seen.append(symbol) + return pd.DataFrame( + {"date": pd.to_datetime(["2026-03-16", "2026-03-17"]), "value": [10.0, 11.0]} + ) + + mock_fetch.side_effect = fake_fetch + + dfq = make_question_df([{"id": original}, {"id": replacement}]) + dff = make_yfinance_fetch_df( + [{"id": replacement, "resolved": False}, {"id": original, "resolved": True}] + ) + + result = yfinance_source.update(dfq, dff) + + # Replacement fetched exactly once (its own build); original never fetched. + assert seen.count(replacement) == 1 + assert original not in seen + # Both files present and identical except for the id column. + fi = result.resolution_files[original].reset_index(drop=True) + fisv = result.resolution_files[replacement].reset_index(drop=True) + assert (fi["id"] == original).all() + assert (fisv["id"] == replacement).all() + assert fi[["date", "value"]].equals(fisv[["date", "value"]]) + @patch.object(YfinanceSource, "_fetch_historical_prices") def test_nullified_stock_preserves_question_fields_and_is_not_fetched( self, mock_fetch, yfinance_source, freeze_today From 6862d9b95157f7b8d56a7ed099cc9162f58220f2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 18 Jun 2026 17:05:55 +0300 Subject: [PATCH 4/4] feat: add slack msg for yfinance 404 fetches as renames are time-sensitive --- src/orchestration/func_yfinance_fetch/main.py | 23 ++++++++++++++++++- .../func_yfinance_fetch/requirements.txt | 2 ++ src/sources/yfinance.py | 15 ++++++++---- src/tests/test_yfinance.py | 21 +++++++++++++++++ 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/orchestration/func_yfinance_fetch/main.py b/src/orchestration/func_yfinance_fetch/main.py index 9e473a6c..647761e6 100644 --- a/src/orchestration/func_yfinance_fetch/main.py +++ b/src/orchestration/func_yfinance_fetch/main.py @@ -5,7 +5,7 @@ import logging from typing import Any -from helpers import data_utils, decorator +from helpers import data_utils, decorator, slack from orchestration import _source_io from sources.yfinance import YfinanceSource @@ -15,6 +15,23 @@ SOURCE = "yfinance" +def _alert_uncurated_delisted(tickers: list[str]) -> None: + """Slack alert for pooled tickers that 404'd but aren't curated. + + They are either newly delisted or newly renamed and need prompt triage into + nullified_questions or ticker_renames (a rename left uncurated resolves old questions to stale + prices). + """ + message = ( + f":warning: yfinance fetch: {len(tickers)} ticker(s) failed to fetch and are no longer in " + f"the S&P 500 (likely delisted or renamed): {', '.join(tickers)}. " + "If delisted, add to nullified_questions; if renamed, add to ticker_renames " + "(mapping to the replacement symbol)." + ) + logger.warning(message) + slack.send_message(message=message) + + @decorator.log_runtime def driver(_: Any) -> None: """Fetch Yahoo Finance stock data and upload to question bank.""" @@ -25,6 +42,10 @@ def driver(_: Any) -> None: dff = source.fetch(dfq=dfq) _source_io.write_fetch_output(SOURCE, dff) + + if source.uncurated_delisted_tickers: + _alert_uncurated_delisted(source.uncurated_delisted_tickers) + logger.info("Done.") diff --git a/src/orchestration/func_yfinance_fetch/requirements.txt b/src/orchestration/func_yfinance_fetch/requirements.txt index a52665cf..24369e12 100644 --- a/src/orchestration/func_yfinance_fetch/requirements.txt +++ b/src/orchestration/func_yfinance_fetch/requirements.txt @@ -1,4 +1,6 @@ google-cloud-storage +google-cloud-secret-manager +slack_sdk pandas>=2.2.2,<3.0 pandera requests diff --git a/src/sources/yfinance.py b/src/sources/yfinance.py index c4c76c28..7f60ff05 100644 --- a/src/sources/yfinance.py +++ b/src/sources/yfinance.py @@ -102,6 +102,9 @@ def fetch( current_time = dates.get_datetime_now() rows = [] + # Pooled tickers that 404 this run but aren't curated (neither nullified nor renamed). + # Recorded so the driver can surface them (e.g. via Slack) for triage into the right list. + self.uncurated_delisted_tickers: list[str] = [] # Carry forward known-unfetchable tickers (nullified + renamed originals) without the API. for ticker_symbol in carry_forward_ids: @@ -147,13 +150,15 @@ def fetch( and ticker_symbol in set_current and ticker_symbol not in set_top_500 ): - # Newly delisted: still in the question pool but no longer fetchable and out of - # the S&P 500, yet not in the curated nullified list. Carry the existing question - # row forward as resolved and warn so it can be added to nullified_questions. + # In the question pool, no longer fetchable, and out of the S&P 500, but not in any + # curated list: either newly delisted or newly renamed. Carry the existing row + # forward as resolved and record it so the driver can flag it for triage. rows.append(self._carry_forward_resolved(ticker_symbol, dfq, current_time)) + self.uncurated_delisted_tickers.append(ticker_symbol) logger.warning( - f"{ticker_symbol} detected as delisted (not in S&P 500 and fetch failed); " - "consider adding it to nullified_questions" + f"{ticker_symbol} failed to fetch and is no longer in the S&P 500 (likely " + "delisted or renamed). If delisted, add it to nullified_questions; if renamed, " + "add it to ticker_renames (mapping it to its replacement symbol)." ) return pd.DataFrame(rows) diff --git a/src/tests/test_yfinance.py b/src/tests/test_yfinance.py index 9bcbff99..4221d6b9 100644 --- a/src/tests/test_yfinance.py +++ b/src/tests/test_yfinance.py @@ -354,6 +354,27 @@ def test_not_in_sp500_but_fetch_succeeds_not_resolved( outco = dff[dff["id"] == "OUTCO"].iloc[0] assert bool(outco["resolved"]) is False + @patch("sources.yfinance.yf.Ticker") + @patch.object(YfinanceSource, "_fetch_one_stock") + @patch.object(YfinanceSource, "_get_sp500_tickers", return_value=["AAPL"]) + def test_records_uncurated_delisted_for_triage( + self, _mock_tickers, mock_fetch_one, mock_ticker_cls, yfinance_source, freeze_today + ): + """A pooled ticker that 404s but isn't curated is recorded for triage; nullified/renamed + tickers (carried forward without fetching) are not.""" + freeze_today(date(2026, 3, 18)) + hist = pd.DataFrame({"Close": [254.23], "Date": pd.to_datetime(["2026-03-17"])}) + mock_fetch_one.side_effect = lambda sym: ( + ("Apple Inc.", hist) if sym == "AAPL" else (None, None) + ) + mock_ticker_cls.return_value.info.get.return_value = "N/A" + + # ZZZZ: uncurated 404. WBA: nullified. FI: renamed original. Only ZZZZ is for triage. + dfq = make_question_df([{"id": "AAPL"}, {"id": "ZZZZ"}, {"id": "WBA"}, {"id": "FI"}]) + yfinance_source.fetch(dfq=dfq) + + assert yfinance_source.uncurated_delisted_tickers == ["ZZZZ"] + class TestSourceFetchSkipsNullified: """Nullified (known-delisted) tickers are never fetched, only carried forward."""