From d4304ff934c8a573e33ea8d3bab76bbedc725763 Mon Sep 17 00:00:00 2001 From: Manas Najmuddeen Date: Sun, 9 Aug 2026 09:48:32 +0000 Subject: [PATCH] fix: authenticate the x.com bootstrap fetch so transaction IDs work again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _ensure_client_transaction fetched https://x.com without cookies. Logged out, x.com now serves the new "x-web" shell (entry-client-logged-out-*.js), whose HTML carries no `ondemand.s` chunk map — so get_ondemand_file_url() matches nothing, ClientTransaction never initializes, and no request ever carries an x-client-transaction-id. Endpoints that require one (notably SearchTimeline, and the 1.1 auth-verification endpoints) answer 404. Sending the session cookies on that one fetch returns the logged-in responsive-web shell, which still ships the chunk map. Verified against live x.com: `twitter search` returns results, and the log line goes from "Failed to init ClientTransaction: 'NoneType' object has no attribute 'group'" to "ClientTransaction initialized for x-client-transaction-id". The ondemand.s file itself is fetched from the CDN and stays cookie-free. Bootstrap failure remains non-fatal. Co-Authored-By: Claude Opus 5 --- tests/test_client.py | 57 +++++++++++++++++++++++++++++++++++++++++++ twitter_cli/client.py | 11 ++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index c1393d3..a116588 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -332,6 +332,63 @@ def test_cookie_string_used_when_available( assert headers["sec-ch-ua-platform-version"] == '""' +# ── TwitterClient._ensure_client_transaction ───────────────────────────── + +class TestEnsureClientTransaction: + """The x.com bootstrap fetch must be authenticated. + + Logged out, x.com serves the new "x-web" shell whose HTML has no + `ondemand.s` chunk map, so no transaction ID can ever be generated and + every transaction-gated endpoint (SearchTimeline included) 404s. + """ + + @staticmethod + def _client(cookie_string=None): + client = TwitterClient.__new__(TwitterClient) + client._auth_token = "token" + client._ct0 = "ct0" + client._cookie_string = cookie_string + client._client_transaction = None + client._ct_init_attempted = False + return client + + @patch("twitter_cli.client._get_cffi_session") + @patch("twitter_cli.client._gen_ct_headers", return_value={}) + def _bootstrap_headers(self, cookie_string, mock_ct_headers, mock_session): + session = MagicMock() + # Abort right after the homepage fetch — we only assert on its headers. + session.get = MagicMock(side_effect=Exception("stop after homepage")) + mock_session.return_value = session + + client = self._client(cookie_string) + with patch.object(client, "_load_ct_cache", return_value=False): + client._ensure_client_transaction() + + assert session.get.call_args[0][0] == "https://x.com" + return session.get.call_args[1]["headers"] + + def test_homepage_fetch_sends_session_cookies(self): + headers = self._bootstrap_headers(None) + assert headers["Cookie"] == "auth_token=token; ct0=ct0" + + def test_homepage_fetch_prefers_full_cookie_string(self): + headers = self._bootstrap_headers("auth_token=x; ct0=y; other=z") + assert headers["Cookie"] == "auth_token=x; ct0=y; other=z" + + @patch("twitter_cli.client._get_cffi_session") + @patch("twitter_cli.client._gen_ct_headers", return_value={}) + def test_bootstrap_failure_is_not_fatal(self, mock_ct_headers, mock_session): + session = MagicMock() + session.get = MagicMock(side_effect=Exception("network down")) + mock_session.return_value = session + + client = self._client() + with patch.object(client, "_load_ct_cache", return_value=False): + client._ensure_client_transaction() + + assert client._client_transaction is None + + class TestPaginationBehavior: def test_fetch_timeline_can_include_promoted_content(self): client = TwitterClient.__new__(TwitterClient) diff --git a/twitter_cli/client.py b/twitter_cli/client.py index 0436c8e..39d2031 100644 --- a/twitter_cli/client.py +++ b/twitter_cli/client.py @@ -1103,8 +1103,17 @@ def _ensure_client_transaction(self): # a different TLS fingerprint on the same IP — a detection vector. cffi_session = _get_cffi_session() ct_headers = _gen_ct_headers() + # Send session cookies: the logged-OUT x.com shell is now the new + # "x-web" build, whose HTML carries no `ondemand.s` chunk map, so + # get_ondemand_file_url() finds nothing and transaction IDs never + # get generated. The logged-in shell still serves the + # responsive-web client-web bundle that does carry the map. + home_headers = dict(ct_headers) + home_headers["Cookie"] = self._cookie_string or "auth_token=%s; ct0=%s" % ( + self._auth_token, self._ct0, + ) home_page = cffi_session.get( - "https://x.com", headers=ct_headers, timeout=10, + "https://x.com", headers=home_headers, timeout=10, ) home_page_response = bs4.BeautifulSoup(home_page.content, "html.parser") ondemand_url = get_ondemand_file_url(response=home_page_response)