From 350c00b54e6eaa34c7331f37e42a4079a3471da9 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:31:53 +0200 Subject: [PATCH 1/4] Let servers delete cookies from the session jar Set-Cookie headers were parsed into a fresh jar and merged into the session jar, so a cookie the server expired was dropped from the fresh jar but stayed in the session jar. Extract into the session jar directly instead. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_cookies.py | 1 - tls_client/cookies.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_cookies.py b/tests/unit/test_cookies.py index 96965f0..ddd6f38 100644 --- a/tests/unit/test_cookies.py +++ b/tests/unit/test_cookies.py @@ -196,7 +196,6 @@ def test_cookie_for_a_foreign_domain_is_rejected(self): assert len(response_jar) == 0 assert len(session_jar) == 0 - @pytest.mark.xfail(reason="expired cookies are dropped from the response jar but stay in the session jar") def test_server_can_delete_a_session_cookie(self): session_jar = RequestsCookieJar() extract_cookies_to_jar("https://example.com/", {}, session_jar, {"Set-Cookie": ["sid=abc; Path=/"]}) diff --git a/tls_client/cookies.py b/tls_client/cookies.py index 6a39038..7eb7a52 100644 --- a/tls_client/cookies.py +++ b/tls_client/cookies.py @@ -444,6 +444,6 @@ def extract_cookies_to_jar( res = MockResponse(http_message) # extract_cookies only needs the duck-typed subset of HTTPResponse and Request the mocks implement response_cookie_jar.extract_cookies(res, req) # type: ignore[arg-type] - - merge_cookies(cookie_jar, response_cookie_jar) + # Extracted into the session jar too rather than merged, so a cookie the server expires is removed from it + cookie_jar.extract_cookies(res, req) # type: ignore[arg-type] return response_cookie_jar From c7b399272884dc9e8cd7222d13748719e38e4850 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:34:11 +0200 Subject: [PATCH 2/4] Make the session cookie jar the only cookie store The library kept a cookie jar per session of its own, so cookies removed from session.cookies, by clear() or del, were still sent. Requests now run with withoutCookieJar, and the cookies for the request URL are picked from the Python jar by domain, path, secure flag and expiry and sent as a Cookie header. With withoutCookieJar the library adds every requestCookies entry to the request without matching it to the URL, which is why they aren't sent that way anymore. A Cookie header passed with the request is still sent, with the jar's cookies after it. Cookie values are sent as stored; the library used to strip double quotes from them. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_httpbin.py | 23 +++++++++ tests/unit/test_async_sessions.py | 4 +- tests/unit/test_sessions.py | 82 ++++++++++++++++++++++++------- tls_client/cookies.py | 12 +++++ tls_client/sessions.py | 29 +++++------ 5 files changed, 116 insertions(+), 34 deletions(-) diff --git a/tests/integration/test_httpbin.py b/tests/integration/test_httpbin.py index aac5dae..c77fd50 100644 --- a/tests/integration/test_httpbin.py +++ b/tests/integration/test_httpbin.py @@ -141,6 +141,29 @@ def test_only_cookies_matching_the_host_are_sent(self, httpbin, session): assert response.json() == {"cookies": {"host_only": "1", "per_request": "3"}} + def test_cleared_cookies_are_no_longer_sent(self, httpbin, session): + session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat"}) + assert session.get(f"{httpbin.url}/cookies").json() == {"cookies": {"flavor": "oat"}} + + session.cookies.clear() + + assert session.get(f"{httpbin.url}/cookies").json() == {"cookies": {}} + + def test_removed_cookie_is_no_longer_sent(self, httpbin, session): + session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat", "size": "big"}) + + del session.cookies["flavor"] + + assert session.get(f"{httpbin.url}/cookies").json() == {"cookies": {"size": "big"}} + + def test_cookie_deleted_by_the_server_is_no_longer_sent(self, httpbin, session): + session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat", "size": "big"}) + + session.get(f"{httpbin.url}/cookies/delete", params={"flavor": ""}) + + assert "flavor" not in session.cookies + assert session.get(f"{httpbin.url}/cookies").json() == {"cookies": {"size": "big"}} + class TestFailures: def test_refused_connection_raises(self, session): diff --git a/tests/unit/test_async_sessions.py b/tests/unit/test_async_sessions.py index 195b329..8c0c4a6 100644 --- a/tests/unit/test_async_sessions.py +++ b/tests/unit/test_async_sessions.py @@ -13,7 +13,9 @@ def sent_cookies(payload: dict) -> dict[str, str]: - return {cookie["name"]: cookie["value"] for cookie in payload["requestCookies"]} + """Cookies of the Cookie header the request was sent with.""" + header = payload["headers"].get("Cookie", "") + return dict(pair.split("=", 1) for pair in header.split("; ") if pair) class CountingExecutor(ThreadPoolExecutor): diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py index bb94082..524fa43 100644 --- a/tests/unit/test_sessions.py +++ b/tests/unit/test_sessions.py @@ -16,7 +16,9 @@ def header_values(headers: Mapping[str, str], name: str) -> list[str]: def sent_cookies(payload: dict) -> dict[str, str]: - return {cookie["name"]: cookie["value"] for cookie in payload["requestCookies"]} + """Cookies of the Cookie header the request was sent with.""" + header = payload["headers"].get("Cookie", "") + return dict(pair.split("=", 1) for pair in header.split("; ") if pair) class TestUrl: @@ -163,28 +165,74 @@ def test_request_cookie_replaces_session_cookie_of_the_same_name(self, session, session.get(URL, cookies={"token": "new"}) - assert fake_lib.sent["requestCookies"] == [{"domain": "", "name": "token", "path": "/", "value": "new"}] + assert fake_lib.sent["headers"]["Cookie"] == "token=new" - def test_double_quotes_are_stripped_from_values(self, session, fake_lib): - session.get(URL, cookies={"quoted": '"abc"'}) + def test_cookies_are_sent_in_a_header_and_the_library_keeps_no_jar(self, session, fake_lib): + session.cookies.set("a", "1") - assert sent_cookies(fake_lib.sent) == {"quoted": "abc"} + session.get(URL) - def test_expiry_is_only_sent_for_persistent_cookies(self, session, fake_lib): - session.cookies.set("session_only", "1") - session.cookies.set("persistent", "2", domain="example.com", path="/api", expires=4102444800) + assert fake_lib.sent["headers"]["Cookie"] == "a=1" + assert fake_lib.sent["withoutCookieJar"] is True + assert "requestCookies" not in fake_lib.sent + def test_no_cookie_header_without_cookies(self, session, fake_lib): session.get(URL) - by_name = {cookie["name"]: cookie for cookie in fake_lib.sent["requestCookies"]} - assert by_name["session_only"] == {"domain": "", "name": "session_only", "path": "/", "value": "1"} - assert by_name["persistent"] == { - "domain": "example.com", - "name": "persistent", - "path": "/api", - "value": "2", - "expires": 4102444800, - } + assert header_values(fake_lib.sent["headers"], "Cookie") == [] + + def test_only_cookies_for_the_domain_and_path_are_sent(self, session, fake_lib): + session.cookies.set("any_host", "1") + session.cookies.set("host", "2", domain="example.com") + session.cookies.set("parent_domain", "3", domain=".example.com") + session.cookies.set("sub_path", "4", domain="example.com", path="/path/sub") + session.cookies.set("other_path", "5", domain="example.com", path="/other") + session.cookies.set("other_host", "6", domain="other.example") + session.cookies.set("subdomain", "7", domain="www.example.com") + + session.get(URL) + + assert sent_cookies(fake_lib.sent) == {"any_host": "1", "host": "2", "parent_domain": "3"} + + def test_secure_cookie_is_only_sent_over_https(self, session, fake_lib): + session.cookies.set("secure", "1", domain="example.com", secure=True) + + session.get("http://example.com/") + assert sent_cookies(fake_lib.sent) == {} + + session.get("https://example.com/") + assert sent_cookies(fake_lib.sent) == {"secure": "1"} + + def test_expired_cookie_is_not_sent(self, session, fake_lib): + session.cookies.set("expired", "1", domain="example.com", expires=1) + session.cookies.set("persistent", "2", domain="example.com", expires=4102444800) + + session.get(URL) + + assert sent_cookies(fake_lib.sent) == {"persistent": "2"} + + def test_cookies_cleared_from_the_jar_are_not_sent(self, session, fake_lib): + fake_lib.reply_with(headers={"Set-Cookie": ["a=1; Path=/"]}) + session.get(URL) + session.get(URL) + assert sent_cookies(fake_lib.sent) == {"a": "1"} + + session.cookies.clear() + session.get(URL) + + assert sent_cookies(fake_lib.sent) == {} + + def test_cookie_header_passed_in_is_sent_before_the_jar_cookies(self, session, fake_lib): + session.cookies.set("from_jar", "1") + + session.get(URL, headers={"cookie": "from_header=2"}) + + assert header_values(fake_lib.sent["headers"], "Cookie") == ["from_header=2; from_jar=1"] + + def test_quoted_value_is_sent_as_is(self, session, fake_lib): + session.get(URL, cookies={"quoted": '"abc"'}) + + assert sent_cookies(fake_lib.sent) == {"quoted": '"abc"'} def test_cookies_from_the_response_are_kept_for_the_next_request(self, session, fake_lib): fake_lib.reply_with(headers={"Set-Cookie": ["a=1; Path=/", "b=2; Path=/"]}) diff --git a/tls_client/cookies.py b/tls_client/cookies.py index 7eb7a52..d78d696 100644 --- a/tls_client/cookies.py +++ b/tls_client/cookies.py @@ -426,6 +426,18 @@ def merge_cookies(cookiejar: RequestsCookieJar, cookies: dict[str, str] | Cookie return cookiejar +def get_cookie_header(cookie_jar: CookieJar, request_url: str, request_headers: Mapping[str, str | None]) -> str | None: + """Cookie header for the cookies in the jar that may be sent to request_url, or None if there are none. + + The jar decides by domain, path, secure flag and expiry, like a browser. request_headers must not contain a Cookie + header, or the jar adds nothing. + """ + req = MockRequest(request_url, request_headers) + # add_cookie_header only needs the duck-typed subset of Request the mock implements + cookie_jar.add_cookie_header(req) # type: ignore[arg-type] + return req.get_new_headers().get("Cookie") + + def extract_cookies_to_jar( request_url: str, request_headers: Mapping[str, str | None], diff --git a/tls_client/sessions.py b/tls_client/sessions.py index ffa4a86..d7860a6 100644 --- a/tls_client/sessions.py +++ b/tls_client/sessions.py @@ -10,7 +10,7 @@ from .__version__ import __version__ from .cffi import destroySession, freeMemory, request -from .cookies import cookiejar_from_dict, extract_cookies_to_jar, merge_cookies +from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header, merge_cookies from .exceptions import TLSClientException from .multipart import Files, encode_multipart from .response import Response, build_response @@ -403,21 +403,16 @@ def execute_request( request_header_order = header_order # --- Cookies -------------------------------------------------------------------------------------------------- - # Merge with session cookies + # Request cookies are added to the session jar. The jar is the only cookie store: the library runs without one + # (withoutCookieJar), so a cookie removed from the jar is no longer sent. cookie_jar = merge_cookies(self.cookies, cookies or {}) - # turn cookie jar into dict - # in the cookie value the " gets removed, because the fhttp library in golang doesn't accept the character - # expires is omitted for session cookies, because the tls client parses null as 1970 and drops the cookie - request_cookies = [ - { - "domain": c.domain, - "name": c.name, - "path": c.path, - "value": (c.value or "").replace('"', ""), - **({"expires": c.expires} if c.expires is not None else {}), - } - for c in cookie_jar - ] + # The jar picks the cookies for the URL by domain, path, secure flag and expiry, and they're sent as a Cookie + # header. A Cookie header passed in is kept, with the jar's cookies after it. + cookie_header = "; ".join( + filter(None, [request_headers.pop("Cookie", None), get_cookie_header(cookie_jar, url, request_headers)]) + ) + if cookie_header: + request_headers["Cookie"] = cookie_header # --- Proxy ---------------------------------------------------------------------------------------------------- proxy = proxy or self.proxies @@ -446,6 +441,9 @@ def execute_request( request_payload: dict[str, Any] = { "sessionId": self._session_id, "followRedirects": allow_redirects, + # Cookies come from self.cookies only. Without this, the library also keeps the cookies of every response + # in a jar of its own and sends them on later requests of the session. + "withoutCookieJar": True, "forceHttp1": self.force_http1, "withDebug": self.debug, "catchPanics": self.catch_panics, @@ -462,7 +460,6 @@ def execute_request( "requestBody": base64.b64encode(request_body).decode() if isinstance(request_body, (bytes, bytearray)) else request_body, - "requestCookies": request_cookies, # Milliseconds, so fractions of a second work. The library rejects timeoutSeconds alongside it. "timeoutMilliseconds": round(request_timeout * 1000), } From f4f60e0ec9ec57fbd521eab9c34cd83136fb6254 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:40:08 +0200 Subject: [PATCH 3/4] Follow redirects in Python so cookies of every hop are kept With followRedirects the library followed redirects itself, and Python only saw the Set-Cookie headers of the last response, so cookies set on the way were lost. followRedirects is now always false, and with allow_redirects execute_request follows 301, 302, 303, 307 and 308 the way requests does: - every hop's cookies go into the session jar and the next hop is sent the jar's cookies for its URL - a relative Location is resolved against the current URL and the fragment carries over - 303 turns anything but HEAD into a GET, 301 and 302 turn a POST into one; the body and its Content-Type, Content-Length and Transfer-Encoding headers are dropped when the method changes, 307 and 308 resend them - Authorization and a Cookie header passed in aren't sent to another host, port or scheme - after 30 redirects TooManyRedirects, a TLSClientException, is raised The final response gets response.history with the earlier responses, and response.cookies holds the cookies of every hop, so AsyncSession, which merges response.cookies into the shared jar, keeps them too. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_httpbin.py | 76 +++++++++- tests/integration/test_local_server.py | 50 +++++- tests/unit/test_async_sessions.py | 36 ++++- tests/unit/test_sessions.py | 201 ++++++++++++++++++++++++- tls_client/async_sessions.py | 3 +- tls_client/exceptions.py | 4 + tls_client/response.py | 5 +- tls_client/sessions.py | 144 +++++++++++++----- 8 files changed, 471 insertions(+), 48 deletions(-) diff --git a/tests/integration/test_httpbin.py b/tests/integration/test_httpbin.py index c77fd50..4487d7e 100644 --- a/tests/integration/test_httpbin.py +++ b/tests/integration/test_httpbin.py @@ -1,3 +1,4 @@ +import asyncio import base64 import socket from collections.abc import Iterator @@ -5,8 +6,8 @@ import pytest from httpbin.core import resource as httpbin_resource -from tls_client import Session -from tls_client.exceptions import TLSClientException +from tls_client import AsyncSession, Session +from tls_client.exceptions import TLSClientException, TooManyRedirects pytestmark = pytest.mark.integration @@ -118,11 +119,61 @@ def test_not_followed_by_default(self, httpbin, session): assert response.status_code == 302 assert response.headers["Location"] == "/get" - def test_followed_when_allowed(self, httpbin, session): - response = session.get(f"{httpbin.url}/redirect/2", allow_redirects=True) + @pytest.mark.parametrize("path", ["/redirect/3", "/relative-redirect/3", "/absolute-redirect/3"]) + def test_followed_when_allowed(self, httpbin, session, path): + response = session.get(f"{httpbin.url}{path}", allow_redirects=True) assert response.status_code == 200 assert response.url == f"{httpbin.url}/get" + assert [r.status_code for r in response.history] == [302, 302, 302] + assert response.history[0].url == f"{httpbin.url}{path}" + + def test_redirect_to_a_relative_url(self, httpbin, session): + response = session.get(f"{httpbin.url}/redirect-to", params={"url": "/get?a=1"}, allow_redirects=True) + + assert response.url == f"{httpbin.url}/get?a=1" + assert response.json()["args"] == {"a": "1"} + + @pytest.mark.parametrize( + ("status", "method"), [(301, "GET"), (302, "GET"), (303, "GET"), (307, "POST"), (308, "POST")] + ) + def test_post_is_resent_as_the_status_requires(self, httpbin, session, status, method): + response = session.post( + f"{httpbin.url}/redirect-to", + params={"url": "/anything", "status_code": status}, + data={"a": "1"}, + allow_redirects=True, + ) + + echoed = response.json() + assert echoed["method"] == method + if method == "POST": + assert echoed["form"] == {"a": "1"} + assert echoed["headers"]["Content-Type"] == "application/x-www-form-urlencoded" + else: + assert echoed["form"] == {} + assert "Content-Type" not in echoed["headers"] + + def test_too_many_redirects_raise(self, httpbin, session): + with pytest.raises(TooManyRedirects): + session.get(f"{httpbin.url}/redirect/31", allow_redirects=True) + + def test_credentials_and_cookies_stay_with_their_host(self, httpbin, session): + session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat"}) + other_host = httpbin.url.replace("127.0.0.1", "localhost") + + response = session.get( + f"{httpbin.url}/redirect-to", + params={"url": f"{other_host}/headers"}, + headers={"Authorization": "Bearer secret", "X-Custom": "1"}, + allow_redirects=True, + ) + + received = response.json()["headers"] + assert received["Host"] == other_host.removeprefix("http://") + assert received["X-Custom"] == "1" + assert "Authorization" not in received + assert "Cookie" not in received class TestCookies: @@ -141,6 +192,23 @@ def test_only_cookies_matching_the_host_are_sent(self, httpbin, session): assert response.json() == {"cookies": {"host_only": "1", "per_request": "3"}} + def test_cookie_set_by_a_redirect_is_stored_and_sent_to_its_target(self, httpbin, session): + # /cookies/set answers with the cookie and a redirect to /cookies, which echoes the cookies it receives. + response = session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat"}, allow_redirects=True) + + assert response.history[0].status_code == 302 + assert response.json() == {"cookies": {"flavor": "oat"}} + assert response.cookies["flavor"] == "oat" + assert session.cookies["flavor"] == "oat" + + def test_cookie_set_by_a_redirect_reaches_the_async_session_jar(self, httpbin): + async def main(): + async with AsyncSession() as session: + await session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat"}, allow_redirects=True) + return session.cookies.get_dict() + + assert asyncio.run(main()) == {"flavor": "oat"} + def test_cleared_cookies_are_no_longer_sent(self, httpbin, session): session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat"}) assert session.get(f"{httpbin.url}/cookies").json() == {"cookies": {"flavor": "oat"}} diff --git a/tests/integration/test_local_server.py b/tests/integration/test_local_server.py index 2222f44..38ab008 100644 --- a/tests/integration/test_local_server.py +++ b/tests/integration/test_local_server.py @@ -1,6 +1,7 @@ -"""Real requests through the native library against a stdlib HTTP server, for bodies and headers httpbin can't -produce byte for byte.""" +"""Real requests through the native library against a stdlib HTTP server, for bodies, headers and redirects httpbin +can't produce byte for byte.""" +import asyncio import base64 import json import os @@ -11,7 +12,7 @@ import pytest -from tls_client import Session +from tls_client import AsyncSession, Session from tls_client.exceptions import TLSClientException pytestmark = pytest.mark.integration @@ -45,6 +46,12 @@ def do_GET(self) -> None: elif self.path == "/slow": time.sleep(SLOW_SECONDS) self.reply("text/plain", b"done") + elif self.path.startswith("/chain/"): + # /chain/ sets the cookie hop and redirects to /chain/, /chain/0 to /headers. + hops = int(self.path.removeprefix("/chain/")) + self.redirect(f"/chain/{hops - 1}" if hops > 1 else "/headers", f"hop{hops}=set; Path=/") + elif self.path == "/expire-hop1": + self.redirect("/headers", "hop1=; Max-Age=0; Path=/") else: content_type, body = BODIES[self.path] self.reply(content_type, body, {"x-MiXeD-case": "yes"}) @@ -57,6 +64,13 @@ def echo_headers(self) -> None: received = {name.lower(): value for name, value in self.headers.items()} self.reply("application/json", json.dumps(received).encode()) + def redirect(self, location: str, set_cookie: str) -> None: + self.send_response(302) + self.send_header("Location", location) + self.send_header("Set-Cookie", set_cookie) + self.send_header("Content-Length", "0") + self.end_headers() + def reply(self, content_type: str, body: bytes, extra_headers: dict[str, str] | None = None) -> None: self.send_response(200) self.send_header("Content-Type", content_type) @@ -163,3 +177,33 @@ def test_fractional_session_timeout_is_applied(self, server_url, session): with pytest.raises(TLSClientException, match="Timeout exceeded"): session.get(f"{server_url}/slow") + + +class TestRedirectCookies: + def test_cookie_set_on_every_hop_is_stored_and_sent_on_the_next_one(self, server_url, session): + response = session.get(f"{server_url}/chain/3", allow_redirects=True) + + assert [r.status_code for r in response.history] == [302, 302, 302] + assert response.url == f"{server_url}/headers" + assert response.json()["cookie"] == "hop3=set; hop2=set; hop1=set" + assert session.cookies.get_dict() == {"hop3": "set", "hop2": "set", "hop1": "set"} + assert response.cookies.get_dict() == {"hop3": "set", "hop2": "set", "hop1": "set"} + + def test_cookie_expired_on_a_hop_is_not_sent_on_the_next_one(self, server_url, session): + session.get(f"{server_url}/chain/2", allow_redirects=True) + + response = session.get(f"{server_url}/expire-hop1", allow_redirects=True) + + assert response.json()["cookie"] == "hop2=set" + assert session.cookies.get_dict() == {"hop2": "set"} + + def test_cookies_set_on_every_hop_end_up_in_the_async_session_jar(self, server_url): + async def main(): + async with AsyncSession() as session: + await session.get(f"{server_url}/chain/2", allow_redirects=True) + return session.cookies.get_dict(), (await session.get(f"{server_url}/headers")).json() + + jar, received = asyncio.run(main()) + + assert jar == {"hop2": "set", "hop1": "set"} + assert received["cookie"] == "hop2=set; hop1=set" diff --git a/tests/unit/test_async_sessions.py b/tests/unit/test_async_sessions.py index 8c0c4a6..fc27c12 100644 --- a/tests/unit/test_async_sessions.py +++ b/tests/unit/test_async_sessions.py @@ -102,14 +102,15 @@ async def main(): assert fake_lib.sent["requestBody"] == "raw" def test_keyword_arguments_are_forwarded(self, fake_lib): + fake_lib.reply_with(status=302, headers={"Location": ["/next"]}) + async def main(): async with AsyncSession() as session: await session.execute_request("GET", URL, params={"q": "1"}, allow_redirects=True) asyncio.run(main()) - assert fake_lib.sent["requestUrl"] == f"{URL}?q=1" - assert fake_lib.sent["followRedirects"] is True + assert [payload["requestUrl"] for payload in fake_lib.payloads] == [f"{URL}?q=1", "https://example.com/next"] def test_requests_use_the_native_session_of_the_wrapped_session(self, fake_lib): async def main(): @@ -163,6 +164,37 @@ async def main(): assert dict(session.cookies) == {"a": "1", "b": "2"} assert sent_cookies(fake_lib.sent) == {"a": "1", "b": "2"} + def test_cookies_set_on_every_redirect_hop_end_up_in_the_jar(self, fake_lib): + fake_lib.reply_with(status=302, headers={"Location": ["/second"], "Set-Cookie": ["a=1; Path=/"]}) + fake_lib.reply_with(status=307, headers={"Location": ["/third"], "Set-Cookie": ["b=2; Path=/"]}) + fake_lib.reply_with(headers={"Set-Cookie": ["c=3; Path=/"]}) + + async def main(): + async with AsyncSession() as session: + response = await session.get(URL, allow_redirects=True) + await session.get(URL) + return session, response + + session, response = asyncio.run(main()) + + assert len(response.history) == 2 + assert dict(session.cookies) == {"a": "1", "b": "2", "c": "3"} + assert sent_cookies(fake_lib.payloads[2]) == {"a": "1", "b": "2"} + assert sent_cookies(fake_lib.sent) == {"a": "1", "b": "2", "c": "3"} + + def test_cleared_cookies_are_not_sent(self, fake_lib): + fake_lib.reply_with(headers={"Set-Cookie": ["a=1; Path=/"]}) + + async def main(): + async with AsyncSession() as session: + await session.get(URL) + session.cookies.clear() + await session.get(URL) + + asyncio.run(main()) + + assert sent_cookies(fake_lib.sent) == {} + @pytest.mark.parametrize("cookies", [{"from_request": "2"}, cookiejar_from_dict({"from_request": "2"})]) def test_request_cookies_are_kept_in_the_jar_like_session_does(self, fake_lib, cookies): async def main(): diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py index 524fa43..db90e52 100644 --- a/tests/unit/test_sessions.py +++ b/tests/unit/test_sessions.py @@ -5,7 +5,7 @@ import pytest from tls_client import Session -from tls_client.exceptions import TLSClientException +from tls_client.exceptions import TLSClientException, TooManyRedirects URL = "https://example.com/path" @@ -245,6 +245,204 @@ def test_cookies_from_the_response_are_kept_for_the_next_request(self, session, assert sent_cookies(fake_lib.sent) == {"a": "1", "b": "2"} +def redirect(status: int = 302, location: str = "/next", **headers: list[str]) -> dict: + """Reply fields for a redirect to location.""" + return {"status": status, "headers": {"Location": [location], **headers}} + + +class TestRedirects: + def test_not_followed_by_default(self, session, fake_lib): + fake_lib.reply_with(**redirect()) + + response = session.get(URL) + + assert response.status_code == 302 + assert response.history == [] + assert len(fake_lib.payloads) == 1 + assert fake_lib.sent["followRedirects"] is False + + def test_chain_is_followed_in_python_and_every_hop_is_kept_in_history(self, session, fake_lib): + fake_lib.reply_with(**redirect(301, "/second")) + fake_lib.reply_with(**redirect(302, "https://example.com/third")) + + response = session.get(URL, allow_redirects=True) + + assert response.status_code == 200 + assert response.url == "https://example.com/third" + assert [r.status_code for r in response.history] == [301, 302] + assert [r.url for r in response.history] == [URL, "https://example.com/second"] + assert [p["requestUrl"] for p in fake_lib.payloads] == [ + URL, + "https://example.com/second", + "https://example.com/third", + ] + # The library must not follow them itself, or the cookies of the hops are lost. + assert all(p["followRedirects"] is False for p in fake_lib.payloads) + assert all(p["withoutCookieJar"] is True for p in fake_lib.payloads) + + def test_cookies_set_on_every_hop_are_stored_and_sent_on_the_next_one(self, session, fake_lib): + fake_lib.reply_with(**redirect(location="/second", **{"Set-Cookie": ["a=1; Path=/"]})) + fake_lib.reply_with(**redirect(location="/third", **{"Set-Cookie": ["b=2; Path=/"]})) + fake_lib.reply_with(headers={"Set-Cookie": ["c=3; Path=/"]}) + + response = session.get(URL, allow_redirects=True) + + assert [sent_cookies(p) for p in fake_lib.payloads] == [{}, {"a": "1"}, {"a": "1", "b": "2"}] + assert dict(session.cookies) == {"a": "1", "b": "2", "c": "3"} + assert dict(response.cookies) == {"a": "1", "b": "2", "c": "3"} + assert [dict(r.cookies) for r in response.history] == [{"a": "1"}, {"b": "2"}] + + def test_cookie_changed_on_a_hop_is_sent_with_its_new_value(self, session, fake_lib): + session.cookies.set("token", "old", domain="example.com") + fake_lib.reply_with(**redirect(**{"Set-Cookie": ["token=new; Path=/"]})) + + session.get(URL, allow_redirects=True) + + assert sent_cookies(fake_lib.payloads[0]) == {"token": "old"} + assert sent_cookies(fake_lib.sent) == {"token": "new"} + + def test_request_cookies_are_sent_on_every_hop(self, session, fake_lib): + fake_lib.reply_with(**redirect()) + + session.get(URL, cookies={"per_request": "1"}, allow_redirects=True) + + assert [sent_cookies(p) for p in fake_lib.payloads] == [{"per_request": "1"}, {"per_request": "1"}] + + def test_cookies_of_one_host_are_not_sent_to_another(self, session, fake_lib): + fake_lib.reply_with(**redirect(location="https://other.example/", **{"Set-Cookie": ["a=1; Path=/"]})) + + session.get(URL, allow_redirects=True) + + assert sent_cookies(fake_lib.sent) == {} + assert session.cookies.list_domains() == ["example.com"] + + @pytest.mark.parametrize( + ("url", "location", "expected"), + [ + ("https://example.com/a/b", "c", "https://example.com/a/c"), + ("https://example.com/a/b", "../c?x=1", "https://example.com/c?x=1"), + ("https://example.com/a/b", "//other.example/c", "https://other.example/c"), + ("https://example.com/a#top", "/b", "https://example.com/b#top"), + ("https://example.com/a#top", "/b#bottom", "https://example.com/b#bottom"), + ], + ) + def test_location_is_resolved_against_the_current_url(self, session, fake_lib, url, location, expected): + fake_lib.reply_with(**redirect(location=location)) + + response = session.get(url, allow_redirects=True) + + assert fake_lib.sent["requestUrl"] == expected + assert response.url == expected + + @pytest.mark.parametrize( + ("status", "method", "expected_method"), + [ + (301, "POST", "GET"), + (302, "POST", "GET"), + (303, "POST", "GET"), + (303, "PUT", "GET"), + (303, "HEAD", "HEAD"), + (301, "PUT", "PUT"), + (302, "DELETE", "DELETE"), + (307, "POST", "POST"), + (308, "PUT", "PUT"), + ], + ) + def test_body_is_only_dropped_when_the_method_changes(self, session, fake_lib, status, method, expected_method): + fake_lib.reply_with(**redirect(status)) + + session.execute_request(method, URL, json={"a": 1}, headers={"Content-Length": "8"}, allow_redirects=True) + + first, second = fake_lib.payloads + assert second["requestMethod"] == expected_method + if expected_method == method: + assert second["requestBody"] == first["requestBody"] == '{"a": 1}' + assert second["headers"]["Content-Type"] == "application/json" + assert second["headers"]["Content-Length"] == "8" + else: + assert second["requestBody"] is None + assert "Content-Type" not in second["headers"] + assert "Content-Length" not in second["headers"] + + def test_binary_body_is_sent_again_on_a_307(self, session, fake_lib): + fake_lib.reply_with(**redirect(307)) + + session.post(URL, data=b"\x00\xff", allow_redirects=True) + + assert fake_lib.sent["isByteRequest"] is True + assert base64.b64decode(fake_lib.sent["requestBody"]) == b"\x00\xff" + + @pytest.mark.parametrize( + ("url", "location", "stripped"), + [ + ("https://example.com/", "/next", False), + ("https://example.com/", "https://example.com:443/next", False), + ("http://example.com/", "https://example.com/next", False), + ("https://example.com/", "https://other.example/", True), + ("https://example.com/", "https://www.example.com/", True), + ("https://example.com/", "http://example.com/", True), + ("https://example.com/", "https://example.com:8443/", True), + ], + ) + def test_credentials_are_not_sent_to_another_origin(self, session, fake_lib, url, location, stripped): + fake_lib.reply_with(**redirect(location=location)) + + session.get(url, headers={"Authorization": "Bearer secret", "Cookie": "passed=1"}, allow_redirects=True) + + sent = fake_lib.sent["headers"] + if stripped: + assert "Authorization" not in sent + assert "Cookie" not in sent + else: + assert sent["Authorization"] == "Bearer secret" + assert sent["Cookie"] == "passed=1" + + def test_other_headers_are_sent_on_every_hop(self, session, fake_lib): + fake_lib.reply_with(**redirect(location="https://other.example/")) + + session.get(URL, headers={"X-Custom": "1"}, allow_redirects=True) + + assert fake_lib.sent["headers"]["X-Custom"] == "1" + assert fake_lib.sent["headers"]["User-Agent"] == session.headers["User-Agent"] + + @pytest.mark.parametrize("reply", [{"status": 302, "headers": {}}, redirect(300), redirect(304)]) + def test_response_that_is_no_redirect_is_returned(self, session, fake_lib, reply): + fake_lib.reply_with(**reply) + + response = session.get(URL, allow_redirects=True) + + assert response.status_code == reply["status"] + assert len(fake_lib.payloads) == 1 + + def test_up_to_30_redirects_are_followed(self, session, fake_lib): + for _ in range(30): + fake_lib.reply_with(**redirect()) + + response = session.get(URL, allow_redirects=True) + + assert response.status_code == 200 + assert len(response.history) == 30 + + def test_more_than_30_redirects_raise(self, session, fake_lib): + for _ in range(31): + fake_lib.reply_with(**redirect()) + + with pytest.raises(TooManyRedirects, match="30 redirects"): + session.get(URL, allow_redirects=True) + + assert len(fake_lib.payloads) == 31 + assert issubclass(TooManyRedirects, TLSClientException) + + def test_library_error_on_a_hop_is_raised(self, session, fake_lib): + fake_lib.reply_with(**redirect()) + fake_lib.reply_with(status=0, body="failed to do request: connection refused") + + with pytest.raises(TLSClientException, match="connection refused"): + session.get(URL, allow_redirects=True) + + assert fake_lib.freed_ids == ["response-1", "response-2"] + + class TestTransportOptions: @pytest.mark.parametrize( ("proxies", "expected"), @@ -297,7 +495,6 @@ def test_timeout_and_timeout_seconds_together_are_rejected(self, session, fake_l ({"force_http1": True}, {}, "forceHttp1"), ({"catch_panics": True}, {}, "catchPanics"), ({"debug": True}, {}, "withDebug"), - ({}, {"allow_redirects": True}, "followRedirects"), ({}, {"insecure_skip_verify": True}, "insecureSkipVerify"), ], ) diff --git a/tls_client/async_sessions.py b/tls_client/async_sessions.py index a8d5a11..4fca998 100644 --- a/tls_client/async_sessions.py +++ b/tls_client/async_sessions.py @@ -99,7 +99,8 @@ async def execute_request(self, method: str, url: str, **kwargs: Any) -> Respons response = await self._run(request_session.execute_request, method=method, url=url, **kwargs) # Back on the event loop thread, so no other request touches the jar while the new cookies go in. Like Session, - # keep the cookies sent with the request, then the ones the server set. + # keep the cookies sent with the request, then the ones the server set (response.cookies holds those of every + # redirect followed too). cookies = kwargs.get("cookies") if cookies: self._session.cookies.update(cookies) diff --git a/tls_client/exceptions.py b/tls_client/exceptions.py index f5283b2..4d20298 100644 --- a/tls_client/exceptions.py +++ b/tls_client/exceptions.py @@ -2,5 +2,9 @@ class TLSClientException(IOError): """General error with the TLS client""" +class TooManyRedirects(TLSClientException): + """A request with allow_redirects was redirected more than 30 times""" + + class UnsupportedPlatformError(TLSClientException): """No native library is available for this OS, CPU architecture or C library""" diff --git a/tls_client/response.py b/tls_client/response.py index c721ca6..2f5b59c 100644 --- a/tls_client/response.py +++ b/tls_client/response.py @@ -20,9 +20,12 @@ def __init__(self) -> None: # Case-insensitive dictionary of Response Headers. Headers sent more than once map to a list of values. self.headers: CaseInsensitiveDict[str | list[str]] = CaseInsensitiveDict() - # A CookieJar of Cookies the server sent back. + # A CookieJar of Cookies the server sent back, on every redirect that led to this response too. self.cookies = cookiejar_from_dict({}) + # Responses of the redirects followed to get this one, oldest first. + self.history: list[Response] = [] + # Encoding used to decode .text: the charset of the Content-Type header, else a BOM or the charset the document # declares in a tag. Set it to decode the body with another encoding; None means UTF-8. self.encoding: str | None = None diff --git a/tls_client/sessions.py b/tls_client/sessions.py index d7860a6..089925d 100644 --- a/tls_client/sessions.py +++ b/tls_client/sessions.py @@ -11,7 +11,7 @@ from .__version__ import __version__ from .cffi import destroySession, freeMemory, request from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header, merge_cookies -from .exceptions import TLSClientException +from .exceptions import TLSClientException, TooManyRedirects from .multipart import Files, encode_multipart from .response import Response, build_response from .settings import CLIENT_IDENTIFIERS, ClientIdentifiers @@ -24,6 +24,35 @@ # Either a proxy URL, or a dict whose "http" entry is used. _Proxy: TypeAlias = dict[str, str] | str +# Statuses followed with allow_redirects, if the response has a Location header +_REDIRECT_STATUSES = frozenset({301, 302, 303, 307, 308}) +# Redirects followed for one request before TooManyRedirects is raised, the same limit as in requests +MAX_REDIRECTS = 30 +_DEFAULT_PORTS = {"http": 80, "https": 443} + + +def _redirect_method(status_code: int, method: str) -> str: + """Method of the request that follows a redirect, as browsers pick it: 303 turns anything but HEAD into a GET, and + 301 and 302 turn a POST into one. 307 and 308 keep the method.""" + if status_code == 303 and method.upper() != "HEAD": + return "GET" + if status_code in (301, 302) and method.upper() == "POST": + return "GET" + return method + + +def _leaves_origin(url: str, next_url: str) -> bool: + """Whether a redirect from url to next_url changes host, port or scheme. An upgrade from http to https on the + default ports doesn't count, like in requests.""" + old, new = urllib.parse.urlsplit(url), urllib.parse.urlsplit(next_url) + if old.hostname != new.hostname: + return True + old_port = old.port or _DEFAULT_PORTS.get(old.scheme) + new_port = new.port or _DEFAULT_PORTS.get(new.scheme) + if old.scheme == "http" and old_port == 80 and new.scheme == "https" and new_port == 443: + return False + return old.scheme != new.scheme or old_port != new_port + class Session: def __init__( @@ -406,13 +435,9 @@ def execute_request( # Request cookies are added to the session jar. The jar is the only cookie store: the library runs without one # (withoutCookieJar), so a cookie removed from the jar is no longer sent. cookie_jar = merge_cookies(self.cookies, cookies or {}) - # The jar picks the cookies for the URL by domain, path, secure flag and expiry, and they're sent as a Cookie - # header. A Cookie header passed in is kept, with the jar's cookies after it. - cookie_header = "; ".join( - filter(None, [request_headers.pop("Cookie", None), get_cookie_header(cookie_jar, url, request_headers)]) - ) - if cookie_header: - request_headers["Cookie"] = cookie_header + # The jar's cookies are added to each request, redirects included, as a Cookie header. A Cookie header passed in + # is sent before them. + passed_cookie_header = request_headers.pop("Cookie", None) # --- Proxy ---------------------------------------------------------------------------------------------------- proxy = proxy or self.proxies @@ -437,29 +462,23 @@ def execute_request( certificate_pinning = self.certificate_pinning # --- Request -------------------------------------------------------------------------------------------------- - is_byte_request = isinstance(request_body, (bytes, bytearray)) + # Settings shared by every request of a redirect chain. The URL, method, headers and body are set per request. request_payload: dict[str, Any] = { "sessionId": self._session_id, - "followRedirects": allow_redirects, + # Redirects are followed below, so that the cookies set on every hop end up in self.cookies. + "followRedirects": False, # Cookies come from self.cookies only. Without this, the library also keeps the cookies of every response # in a jar of its own and sends them on later requests of the session. "withoutCookieJar": True, "forceHttp1": self.force_http1, "withDebug": self.debug, "catchPanics": self.catch_panics, - "headers": dict(request_headers), "headerOrder": request_header_order, "insecureSkipVerify": insecure_skip_verify, - "isByteRequest": is_byte_request, # Return the body as base64 so binary bodies survive; build_response decodes it. "isByteResponse": True, "additionalDecode": self.additional_decode, "proxyUrl": proxy_url, - "requestUrl": url, - "requestMethod": method, - "requestBody": base64.b64encode(request_body).decode() - if isinstance(request_body, (bytes, bytearray)) - else request_body, # Milliseconds, so fractions of a second work. The library rejects timeoutSeconds alongside it. "timeoutMilliseconds": round(request_timeout * 1000), } @@ -486,24 +505,79 @@ def execute_request( request_payload["tlsClientIdentifier"] = self.client_identifier request_payload["withRandomTLSExtensionOrder"] = self.random_tls_extension_order - # dereference the pointer to the response, decode it and parse it (tls client returns json). None of the steps - # is bound to a name, so each copy of a large reply is freed as soon as the next one exists. - response_object = loads(ctypes.string_at(request(dumps(request_payload).encode("utf-8"))).decode("utf-8")) - # free the memory - freeMemory(response_object["id"].encode("utf-8")) - # --- Response ------------------------------------------------------------------------------------------------- - # Error handling - if response_object["status"] == 0: - raise TLSClientException(response_object["body"]) - # Set response cookies - response_cookie_jar = extract_cookies_to_jar( - request_url=url, - request_headers=request_headers, - cookie_jar=cookie_jar, - response_headers=response_object["headers"], - ) - # build response class - return build_response(response_object, response_cookie_jar) + # Sends the request, then with allow_redirects each request a redirect asks for, like requests does. + history: list[Response] = [] + # Cookies set on any hop of a redirect chain, for the final response + chain_cookies = cookiejar_from_dict({}) + while True: + # The jar picks the cookies for the URL by domain, path, secure flag and expiry. + sent_headers = CaseInsensitiveDict(request_headers) + cookie_header = "; ".join( + filter(None, [passed_cookie_header, get_cookie_header(cookie_jar, url, request_headers)]) + ) + if cookie_header: + sent_headers["Cookie"] = cookie_header + request_payload["headers"] = dict(sent_headers) + request_payload["requestUrl"] = url + request_payload["requestMethod"] = method + request_payload["isByteRequest"] = isinstance(request_body, (bytes, bytearray)) + request_payload["requestBody"] = ( + base64.b64encode(request_body).decode() + if isinstance(request_body, (bytes, bytearray)) + else request_body + ) + + # dereference the pointer to the response, decode it and parse it (tls client returns json). None of the + # steps is bound to a name, so each copy of a large reply is freed as soon as the next one exists. + response_object = loads(ctypes.string_at(request(dumps(request_payload).encode("utf-8"))).decode("utf-8")) + # free the memory + freeMemory(response_object["id"].encode("utf-8")) + # --- Response --------------------------------------------------------------------------------------------- + # Error handling + if response_object["status"] == 0: + raise TLSClientException(response_object["body"]) + # Set response cookies + response_cookie_jar = extract_cookies_to_jar( + request_url=url, + request_headers=request_headers, + cookie_jar=cookie_jar, + response_headers=response_object["headers"], + ) + chain_cookies.update(response_cookie_jar) + # build response class + response = build_response(response_object, response_cookie_jar) + + # --- Redirects -------------------------------------------------------------------------------------------- + location = response.headers.get("Location") + if not allow_redirects or response.status_code not in _REDIRECT_STATUSES or location is None: + break + if len(history) == MAX_REDIRECTS: + raise TooManyRedirects(f"Exceeded {MAX_REDIRECTS} redirects, the last from {url}") + history.append(response) + + # A relative Location is resolved against the current URL. Like in requests, the fragment carries over + # unless the Location has one of its own. + next_url = urllib.parse.urljoin(url, location if isinstance(location, str) else location[0]) + fragment = urllib.parse.urlsplit(url).fragment + if fragment and not urllib.parse.urlsplit(next_url).fragment: + next_url = f"{next_url}#{fragment}" + # Credentials passed for one host aren't sent to another + if _leaves_origin(url, next_url): + request_headers.pop("Authorization", None) + passed_cookie_header = None + url = next_url + + next_method = _redirect_method(response.status_code, method) + if next_method != method: + # The request turned into a GET, which has no body + method = next_method + request_body = None + for header in ("Content-Type", "Content-Length", "Transfer-Encoding"): + request_headers.pop(header, None) + + response.history = history + response.cookies = chain_cookies + return response def get(self, url: str, **kwargs: Any) -> Response: """Sends a GET request""" From 406de788125455e6202211d9eab080ba68e04bfe Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:48:48 +0200 Subject: [PATCH 4/4] Don't depend on the order of same-path cookies in the Cookie header Python 3.10 orders them differently from later versions. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_local_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_local_server.py b/tests/integration/test_local_server.py index 38ab008..ca23c69 100644 --- a/tests/integration/test_local_server.py +++ b/tests/integration/test_local_server.py @@ -185,7 +185,8 @@ def test_cookie_set_on_every_hop_is_stored_and_sent_on_the_next_one(self, server assert [r.status_code for r in response.history] == [302, 302, 302] assert response.url == f"{server_url}/headers" - assert response.json()["cookie"] == "hop3=set; hop2=set; hop1=set" + # Cookies of the same path have no defined order in the header; it differs between Python versions. + assert sorted(response.json()["cookie"].split("; ")) == ["hop1=set", "hop2=set", "hop3=set"] assert session.cookies.get_dict() == {"hop3": "set", "hop2": "set", "hop1": "set"} assert response.cookies.get_dict() == {"hop3": "set", "hop2": "set", "hop1": "set"} @@ -206,4 +207,4 @@ async def main(): jar, received = asyncio.run(main()) assert jar == {"hop2": "set", "hop1": "set"} - assert received["cookie"] == "hop2=set; hop1=set" + assert sorted(received["cookie"].split("; ")) == ["hop1=set", "hop2=set"]