diff --git a/tests/integration/test_httpbin.py b/tests/integration/test_httpbin.py index aac5dae..4487d7e 100644 --- a/tests/integration/test_httpbin.py +++ b/tests/integration/test_httpbin.py @@ -1,3 +1,4 @@ +import asyncio import base64 import socket from collections.abc import Iterator @@ -5,8 +6,8 @@ import pytest from httpbin.core import resource as httpbin_resource -from tls_client import Session -from tls_client.exceptions import TLSClientException +from tls_client import AsyncSession, Session +from tls_client.exceptions import TLSClientException, TooManyRedirects pytestmark = pytest.mark.integration @@ -118,11 +119,61 @@ def test_not_followed_by_default(self, httpbin, session): assert response.status_code == 302 assert response.headers["Location"] == "/get" - def test_followed_when_allowed(self, httpbin, session): - response = session.get(f"{httpbin.url}/redirect/2", allow_redirects=True) + @pytest.mark.parametrize("path", ["/redirect/3", "/relative-redirect/3", "/absolute-redirect/3"]) + def test_followed_when_allowed(self, httpbin, session, path): + response = session.get(f"{httpbin.url}{path}", allow_redirects=True) assert response.status_code == 200 assert response.url == f"{httpbin.url}/get" + assert [r.status_code for r in response.history] == [302, 302, 302] + assert response.history[0].url == f"{httpbin.url}{path}" + + def test_redirect_to_a_relative_url(self, httpbin, session): + response = session.get(f"{httpbin.url}/redirect-to", params={"url": "/get?a=1"}, allow_redirects=True) + + assert response.url == f"{httpbin.url}/get?a=1" + assert response.json()["args"] == {"a": "1"} + + @pytest.mark.parametrize( + ("status", "method"), [(301, "GET"), (302, "GET"), (303, "GET"), (307, "POST"), (308, "POST")] + ) + def test_post_is_resent_as_the_status_requires(self, httpbin, session, status, method): + response = session.post( + f"{httpbin.url}/redirect-to", + params={"url": "/anything", "status_code": status}, + data={"a": "1"}, + allow_redirects=True, + ) + + echoed = response.json() + assert echoed["method"] == method + if method == "POST": + assert echoed["form"] == {"a": "1"} + assert echoed["headers"]["Content-Type"] == "application/x-www-form-urlencoded" + else: + assert echoed["form"] == {} + assert "Content-Type" not in echoed["headers"] + + def test_too_many_redirects_raise(self, httpbin, session): + with pytest.raises(TooManyRedirects): + session.get(f"{httpbin.url}/redirect/31", allow_redirects=True) + + def test_credentials_and_cookies_stay_with_their_host(self, httpbin, session): + session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat"}) + other_host = httpbin.url.replace("127.0.0.1", "localhost") + + response = session.get( + f"{httpbin.url}/redirect-to", + params={"url": f"{other_host}/headers"}, + headers={"Authorization": "Bearer secret", "X-Custom": "1"}, + allow_redirects=True, + ) + + received = response.json()["headers"] + assert received["Host"] == other_host.removeprefix("http://") + assert received["X-Custom"] == "1" + assert "Authorization" not in received + assert "Cookie" not in received class TestCookies: @@ -141,6 +192,46 @@ def test_only_cookies_matching_the_host_are_sent(self, httpbin, session): assert response.json() == {"cookies": {"host_only": "1", "per_request": "3"}} + def test_cookie_set_by_a_redirect_is_stored_and_sent_to_its_target(self, httpbin, session): + # /cookies/set answers with the cookie and a redirect to /cookies, which echoes the cookies it receives. + response = session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat"}, allow_redirects=True) + + assert response.history[0].status_code == 302 + assert response.json() == {"cookies": {"flavor": "oat"}} + assert response.cookies["flavor"] == "oat" + assert session.cookies["flavor"] == "oat" + + def test_cookie_set_by_a_redirect_reaches_the_async_session_jar(self, httpbin): + async def main(): + async with AsyncSession() as session: + await session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat"}, allow_redirects=True) + return session.cookies.get_dict() + + assert asyncio.run(main()) == {"flavor": "oat"} + + def test_cleared_cookies_are_no_longer_sent(self, httpbin, session): + session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat"}) + assert session.get(f"{httpbin.url}/cookies").json() == {"cookies": {"flavor": "oat"}} + + session.cookies.clear() + + assert session.get(f"{httpbin.url}/cookies").json() == {"cookies": {}} + + def test_removed_cookie_is_no_longer_sent(self, httpbin, session): + session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat", "size": "big"}) + + del session.cookies["flavor"] + + assert session.get(f"{httpbin.url}/cookies").json() == {"cookies": {"size": "big"}} + + def test_cookie_deleted_by_the_server_is_no_longer_sent(self, httpbin, session): + session.get(f"{httpbin.url}/cookies/set", params={"flavor": "oat", "size": "big"}) + + session.get(f"{httpbin.url}/cookies/delete", params={"flavor": ""}) + + assert "flavor" not in session.cookies + assert session.get(f"{httpbin.url}/cookies").json() == {"cookies": {"size": "big"}} + class TestFailures: def test_refused_connection_raises(self, session): diff --git a/tests/integration/test_local_server.py b/tests/integration/test_local_server.py index 2222f44..ca23c69 100644 --- a/tests/integration/test_local_server.py +++ b/tests/integration/test_local_server.py @@ -1,6 +1,7 @@ -"""Real requests through the native library against a stdlib HTTP server, for bodies and headers httpbin can't -produce byte for byte.""" +"""Real requests through the native library against a stdlib HTTP server, for bodies, headers and redirects httpbin +can't produce byte for byte.""" +import asyncio import base64 import json import os @@ -11,7 +12,7 @@ import pytest -from tls_client import Session +from tls_client import AsyncSession, Session from tls_client.exceptions import TLSClientException pytestmark = pytest.mark.integration @@ -45,6 +46,12 @@ def do_GET(self) -> None: elif self.path == "/slow": time.sleep(SLOW_SECONDS) self.reply("text/plain", b"done") + elif self.path.startswith("/chain/"): + # /chain/ sets the cookie hop and redirects to /chain/, /chain/0 to /headers. + hops = int(self.path.removeprefix("/chain/")) + self.redirect(f"/chain/{hops - 1}" if hops > 1 else "/headers", f"hop{hops}=set; Path=/") + elif self.path == "/expire-hop1": + self.redirect("/headers", "hop1=; Max-Age=0; Path=/") else: content_type, body = BODIES[self.path] self.reply(content_type, body, {"x-MiXeD-case": "yes"}) @@ -57,6 +64,13 @@ def echo_headers(self) -> None: received = {name.lower(): value for name, value in self.headers.items()} self.reply("application/json", json.dumps(received).encode()) + def redirect(self, location: str, set_cookie: str) -> None: + self.send_response(302) + self.send_header("Location", location) + self.send_header("Set-Cookie", set_cookie) + self.send_header("Content-Length", "0") + self.end_headers() + def reply(self, content_type: str, body: bytes, extra_headers: dict[str, str] | None = None) -> None: self.send_response(200) self.send_header("Content-Type", content_type) @@ -163,3 +177,34 @@ def test_fractional_session_timeout_is_applied(self, server_url, session): with pytest.raises(TLSClientException, match="Timeout exceeded"): session.get(f"{server_url}/slow") + + +class TestRedirectCookies: + def test_cookie_set_on_every_hop_is_stored_and_sent_on_the_next_one(self, server_url, session): + response = session.get(f"{server_url}/chain/3", allow_redirects=True) + + assert [r.status_code for r in response.history] == [302, 302, 302] + assert response.url == f"{server_url}/headers" + # Cookies of the same path have no defined order in the header; it differs between Python versions. + assert sorted(response.json()["cookie"].split("; ")) == ["hop1=set", "hop2=set", "hop3=set"] + assert session.cookies.get_dict() == {"hop3": "set", "hop2": "set", "hop1": "set"} + assert response.cookies.get_dict() == {"hop3": "set", "hop2": "set", "hop1": "set"} + + def test_cookie_expired_on_a_hop_is_not_sent_on_the_next_one(self, server_url, session): + session.get(f"{server_url}/chain/2", allow_redirects=True) + + response = session.get(f"{server_url}/expire-hop1", allow_redirects=True) + + assert response.json()["cookie"] == "hop2=set" + assert session.cookies.get_dict() == {"hop2": "set"} + + def test_cookies_set_on_every_hop_end_up_in_the_async_session_jar(self, server_url): + async def main(): + async with AsyncSession() as session: + await session.get(f"{server_url}/chain/2", allow_redirects=True) + return session.cookies.get_dict(), (await session.get(f"{server_url}/headers")).json() + + jar, received = asyncio.run(main()) + + assert jar == {"hop2": "set", "hop1": "set"} + assert sorted(received["cookie"].split("; ")) == ["hop1=set", "hop2=set"] diff --git a/tests/unit/test_async_sessions.py b/tests/unit/test_async_sessions.py index 195b329..fc27c12 100644 --- a/tests/unit/test_async_sessions.py +++ b/tests/unit/test_async_sessions.py @@ -13,7 +13,9 @@ def sent_cookies(payload: dict) -> dict[str, str]: - return {cookie["name"]: cookie["value"] for cookie in payload["requestCookies"]} + """Cookies of the Cookie header the request was sent with.""" + header = payload["headers"].get("Cookie", "") + return dict(pair.split("=", 1) for pair in header.split("; ") if pair) class CountingExecutor(ThreadPoolExecutor): @@ -100,14 +102,15 @@ async def main(): assert fake_lib.sent["requestBody"] == "raw" def test_keyword_arguments_are_forwarded(self, fake_lib): + fake_lib.reply_with(status=302, headers={"Location": ["/next"]}) + async def main(): async with AsyncSession() as session: await session.execute_request("GET", URL, params={"q": "1"}, allow_redirects=True) asyncio.run(main()) - assert fake_lib.sent["requestUrl"] == f"{URL}?q=1" - assert fake_lib.sent["followRedirects"] is True + assert [payload["requestUrl"] for payload in fake_lib.payloads] == [f"{URL}?q=1", "https://example.com/next"] def test_requests_use_the_native_session_of_the_wrapped_session(self, fake_lib): async def main(): @@ -161,6 +164,37 @@ async def main(): assert dict(session.cookies) == {"a": "1", "b": "2"} assert sent_cookies(fake_lib.sent) == {"a": "1", "b": "2"} + def test_cookies_set_on_every_redirect_hop_end_up_in_the_jar(self, fake_lib): + fake_lib.reply_with(status=302, headers={"Location": ["/second"], "Set-Cookie": ["a=1; Path=/"]}) + fake_lib.reply_with(status=307, headers={"Location": ["/third"], "Set-Cookie": ["b=2; Path=/"]}) + fake_lib.reply_with(headers={"Set-Cookie": ["c=3; Path=/"]}) + + async def main(): + async with AsyncSession() as session: + response = await session.get(URL, allow_redirects=True) + await session.get(URL) + return session, response + + session, response = asyncio.run(main()) + + assert len(response.history) == 2 + assert dict(session.cookies) == {"a": "1", "b": "2", "c": "3"} + assert sent_cookies(fake_lib.payloads[2]) == {"a": "1", "b": "2"} + assert sent_cookies(fake_lib.sent) == {"a": "1", "b": "2", "c": "3"} + + def test_cleared_cookies_are_not_sent(self, fake_lib): + fake_lib.reply_with(headers={"Set-Cookie": ["a=1; Path=/"]}) + + async def main(): + async with AsyncSession() as session: + await session.get(URL) + session.cookies.clear() + await session.get(URL) + + asyncio.run(main()) + + assert sent_cookies(fake_lib.sent) == {} + @pytest.mark.parametrize("cookies", [{"from_request": "2"}, cookiejar_from_dict({"from_request": "2"})]) def test_request_cookies_are_kept_in_the_jar_like_session_does(self, fake_lib, cookies): async def main(): diff --git a/tests/unit/test_cookies.py b/tests/unit/test_cookies.py index 96965f0..ddd6f38 100644 --- a/tests/unit/test_cookies.py +++ b/tests/unit/test_cookies.py @@ -196,7 +196,6 @@ def test_cookie_for_a_foreign_domain_is_rejected(self): assert len(response_jar) == 0 assert len(session_jar) == 0 - @pytest.mark.xfail(reason="expired cookies are dropped from the response jar but stay in the session jar") def test_server_can_delete_a_session_cookie(self): session_jar = RequestsCookieJar() extract_cookies_to_jar("https://example.com/", {}, session_jar, {"Set-Cookie": ["sid=abc; Path=/"]}) diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py index bb94082..db90e52 100644 --- a/tests/unit/test_sessions.py +++ b/tests/unit/test_sessions.py @@ -5,7 +5,7 @@ import pytest from tls_client import Session -from tls_client.exceptions import TLSClientException +from tls_client.exceptions import TLSClientException, TooManyRedirects URL = "https://example.com/path" @@ -16,7 +16,9 @@ def header_values(headers: Mapping[str, str], name: str) -> list[str]: def sent_cookies(payload: dict) -> dict[str, str]: - return {cookie["name"]: cookie["value"] for cookie in payload["requestCookies"]} + """Cookies of the Cookie header the request was sent with.""" + header = payload["headers"].get("Cookie", "") + return dict(pair.split("=", 1) for pair in header.split("; ") if pair) class TestUrl: @@ -163,28 +165,74 @@ def test_request_cookie_replaces_session_cookie_of_the_same_name(self, session, session.get(URL, cookies={"token": "new"}) - assert fake_lib.sent["requestCookies"] == [{"domain": "", "name": "token", "path": "/", "value": "new"}] + assert fake_lib.sent["headers"]["Cookie"] == "token=new" - def test_double_quotes_are_stripped_from_values(self, session, fake_lib): - session.get(URL, cookies={"quoted": '"abc"'}) + def test_cookies_are_sent_in_a_header_and_the_library_keeps_no_jar(self, session, fake_lib): + session.cookies.set("a", "1") - assert sent_cookies(fake_lib.sent) == {"quoted": "abc"} + session.get(URL) - def test_expiry_is_only_sent_for_persistent_cookies(self, session, fake_lib): - session.cookies.set("session_only", "1") - session.cookies.set("persistent", "2", domain="example.com", path="/api", expires=4102444800) + assert fake_lib.sent["headers"]["Cookie"] == "a=1" + assert fake_lib.sent["withoutCookieJar"] is True + assert "requestCookies" not in fake_lib.sent + def test_no_cookie_header_without_cookies(self, session, fake_lib): session.get(URL) - by_name = {cookie["name"]: cookie for cookie in fake_lib.sent["requestCookies"]} - assert by_name["session_only"] == {"domain": "", "name": "session_only", "path": "/", "value": "1"} - assert by_name["persistent"] == { - "domain": "example.com", - "name": "persistent", - "path": "/api", - "value": "2", - "expires": 4102444800, - } + assert header_values(fake_lib.sent["headers"], "Cookie") == [] + + def test_only_cookies_for_the_domain_and_path_are_sent(self, session, fake_lib): + session.cookies.set("any_host", "1") + session.cookies.set("host", "2", domain="example.com") + session.cookies.set("parent_domain", "3", domain=".example.com") + session.cookies.set("sub_path", "4", domain="example.com", path="/path/sub") + session.cookies.set("other_path", "5", domain="example.com", path="/other") + session.cookies.set("other_host", "6", domain="other.example") + session.cookies.set("subdomain", "7", domain="www.example.com") + + session.get(URL) + + assert sent_cookies(fake_lib.sent) == {"any_host": "1", "host": "2", "parent_domain": "3"} + + def test_secure_cookie_is_only_sent_over_https(self, session, fake_lib): + session.cookies.set("secure", "1", domain="example.com", secure=True) + + session.get("http://example.com/") + assert sent_cookies(fake_lib.sent) == {} + + session.get("https://example.com/") + assert sent_cookies(fake_lib.sent) == {"secure": "1"} + + def test_expired_cookie_is_not_sent(self, session, fake_lib): + session.cookies.set("expired", "1", domain="example.com", expires=1) + session.cookies.set("persistent", "2", domain="example.com", expires=4102444800) + + session.get(URL) + + assert sent_cookies(fake_lib.sent) == {"persistent": "2"} + + def test_cookies_cleared_from_the_jar_are_not_sent(self, session, fake_lib): + fake_lib.reply_with(headers={"Set-Cookie": ["a=1; Path=/"]}) + session.get(URL) + session.get(URL) + assert sent_cookies(fake_lib.sent) == {"a": "1"} + + session.cookies.clear() + session.get(URL) + + assert sent_cookies(fake_lib.sent) == {} + + def test_cookie_header_passed_in_is_sent_before_the_jar_cookies(self, session, fake_lib): + session.cookies.set("from_jar", "1") + + session.get(URL, headers={"cookie": "from_header=2"}) + + assert header_values(fake_lib.sent["headers"], "Cookie") == ["from_header=2; from_jar=1"] + + def test_quoted_value_is_sent_as_is(self, session, fake_lib): + session.get(URL, cookies={"quoted": '"abc"'}) + + assert sent_cookies(fake_lib.sent) == {"quoted": '"abc"'} def test_cookies_from_the_response_are_kept_for_the_next_request(self, session, fake_lib): fake_lib.reply_with(headers={"Set-Cookie": ["a=1; Path=/", "b=2; Path=/"]}) @@ -197,6 +245,204 @@ def test_cookies_from_the_response_are_kept_for_the_next_request(self, session, assert sent_cookies(fake_lib.sent) == {"a": "1", "b": "2"} +def redirect(status: int = 302, location: str = "/next", **headers: list[str]) -> dict: + """Reply fields for a redirect to location.""" + return {"status": status, "headers": {"Location": [location], **headers}} + + +class TestRedirects: + def test_not_followed_by_default(self, session, fake_lib): + fake_lib.reply_with(**redirect()) + + response = session.get(URL) + + assert response.status_code == 302 + assert response.history == [] + assert len(fake_lib.payloads) == 1 + assert fake_lib.sent["followRedirects"] is False + + def test_chain_is_followed_in_python_and_every_hop_is_kept_in_history(self, session, fake_lib): + fake_lib.reply_with(**redirect(301, "/second")) + fake_lib.reply_with(**redirect(302, "https://example.com/third")) + + response = session.get(URL, allow_redirects=True) + + assert response.status_code == 200 + assert response.url == "https://example.com/third" + assert [r.status_code for r in response.history] == [301, 302] + assert [r.url for r in response.history] == [URL, "https://example.com/second"] + assert [p["requestUrl"] for p in fake_lib.payloads] == [ + URL, + "https://example.com/second", + "https://example.com/third", + ] + # The library must not follow them itself, or the cookies of the hops are lost. + assert all(p["followRedirects"] is False for p in fake_lib.payloads) + assert all(p["withoutCookieJar"] is True for p in fake_lib.payloads) + + def test_cookies_set_on_every_hop_are_stored_and_sent_on_the_next_one(self, session, fake_lib): + fake_lib.reply_with(**redirect(location="/second", **{"Set-Cookie": ["a=1; Path=/"]})) + fake_lib.reply_with(**redirect(location="/third", **{"Set-Cookie": ["b=2; Path=/"]})) + fake_lib.reply_with(headers={"Set-Cookie": ["c=3; Path=/"]}) + + response = session.get(URL, allow_redirects=True) + + assert [sent_cookies(p) for p in fake_lib.payloads] == [{}, {"a": "1"}, {"a": "1", "b": "2"}] + assert dict(session.cookies) == {"a": "1", "b": "2", "c": "3"} + assert dict(response.cookies) == {"a": "1", "b": "2", "c": "3"} + assert [dict(r.cookies) for r in response.history] == [{"a": "1"}, {"b": "2"}] + + def test_cookie_changed_on_a_hop_is_sent_with_its_new_value(self, session, fake_lib): + session.cookies.set("token", "old", domain="example.com") + fake_lib.reply_with(**redirect(**{"Set-Cookie": ["token=new; Path=/"]})) + + session.get(URL, allow_redirects=True) + + assert sent_cookies(fake_lib.payloads[0]) == {"token": "old"} + assert sent_cookies(fake_lib.sent) == {"token": "new"} + + def test_request_cookies_are_sent_on_every_hop(self, session, fake_lib): + fake_lib.reply_with(**redirect()) + + session.get(URL, cookies={"per_request": "1"}, allow_redirects=True) + + assert [sent_cookies(p) for p in fake_lib.payloads] == [{"per_request": "1"}, {"per_request": "1"}] + + def test_cookies_of_one_host_are_not_sent_to_another(self, session, fake_lib): + fake_lib.reply_with(**redirect(location="https://other.example/", **{"Set-Cookie": ["a=1; Path=/"]})) + + session.get(URL, allow_redirects=True) + + assert sent_cookies(fake_lib.sent) == {} + assert session.cookies.list_domains() == ["example.com"] + + @pytest.mark.parametrize( + ("url", "location", "expected"), + [ + ("https://example.com/a/b", "c", "https://example.com/a/c"), + ("https://example.com/a/b", "../c?x=1", "https://example.com/c?x=1"), + ("https://example.com/a/b", "//other.example/c", "https://other.example/c"), + ("https://example.com/a#top", "/b", "https://example.com/b#top"), + ("https://example.com/a#top", "/b#bottom", "https://example.com/b#bottom"), + ], + ) + def test_location_is_resolved_against_the_current_url(self, session, fake_lib, url, location, expected): + fake_lib.reply_with(**redirect(location=location)) + + response = session.get(url, allow_redirects=True) + + assert fake_lib.sent["requestUrl"] == expected + assert response.url == expected + + @pytest.mark.parametrize( + ("status", "method", "expected_method"), + [ + (301, "POST", "GET"), + (302, "POST", "GET"), + (303, "POST", "GET"), + (303, "PUT", "GET"), + (303, "HEAD", "HEAD"), + (301, "PUT", "PUT"), + (302, "DELETE", "DELETE"), + (307, "POST", "POST"), + (308, "PUT", "PUT"), + ], + ) + def test_body_is_only_dropped_when_the_method_changes(self, session, fake_lib, status, method, expected_method): + fake_lib.reply_with(**redirect(status)) + + session.execute_request(method, URL, json={"a": 1}, headers={"Content-Length": "8"}, allow_redirects=True) + + first, second = fake_lib.payloads + assert second["requestMethod"] == expected_method + if expected_method == method: + assert second["requestBody"] == first["requestBody"] == '{"a": 1}' + assert second["headers"]["Content-Type"] == "application/json" + assert second["headers"]["Content-Length"] == "8" + else: + assert second["requestBody"] is None + assert "Content-Type" not in second["headers"] + assert "Content-Length" not in second["headers"] + + def test_binary_body_is_sent_again_on_a_307(self, session, fake_lib): + fake_lib.reply_with(**redirect(307)) + + session.post(URL, data=b"\x00\xff", allow_redirects=True) + + assert fake_lib.sent["isByteRequest"] is True + assert base64.b64decode(fake_lib.sent["requestBody"]) == b"\x00\xff" + + @pytest.mark.parametrize( + ("url", "location", "stripped"), + [ + ("https://example.com/", "/next", False), + ("https://example.com/", "https://example.com:443/next", False), + ("http://example.com/", "https://example.com/next", False), + ("https://example.com/", "https://other.example/", True), + ("https://example.com/", "https://www.example.com/", True), + ("https://example.com/", "http://example.com/", True), + ("https://example.com/", "https://example.com:8443/", True), + ], + ) + def test_credentials_are_not_sent_to_another_origin(self, session, fake_lib, url, location, stripped): + fake_lib.reply_with(**redirect(location=location)) + + session.get(url, headers={"Authorization": "Bearer secret", "Cookie": "passed=1"}, allow_redirects=True) + + sent = fake_lib.sent["headers"] + if stripped: + assert "Authorization" not in sent + assert "Cookie" not in sent + else: + assert sent["Authorization"] == "Bearer secret" + assert sent["Cookie"] == "passed=1" + + def test_other_headers_are_sent_on_every_hop(self, session, fake_lib): + fake_lib.reply_with(**redirect(location="https://other.example/")) + + session.get(URL, headers={"X-Custom": "1"}, allow_redirects=True) + + assert fake_lib.sent["headers"]["X-Custom"] == "1" + assert fake_lib.sent["headers"]["User-Agent"] == session.headers["User-Agent"] + + @pytest.mark.parametrize("reply", [{"status": 302, "headers": {}}, redirect(300), redirect(304)]) + def test_response_that_is_no_redirect_is_returned(self, session, fake_lib, reply): + fake_lib.reply_with(**reply) + + response = session.get(URL, allow_redirects=True) + + assert response.status_code == reply["status"] + assert len(fake_lib.payloads) == 1 + + def test_up_to_30_redirects_are_followed(self, session, fake_lib): + for _ in range(30): + fake_lib.reply_with(**redirect()) + + response = session.get(URL, allow_redirects=True) + + assert response.status_code == 200 + assert len(response.history) == 30 + + def test_more_than_30_redirects_raise(self, session, fake_lib): + for _ in range(31): + fake_lib.reply_with(**redirect()) + + with pytest.raises(TooManyRedirects, match="30 redirects"): + session.get(URL, allow_redirects=True) + + assert len(fake_lib.payloads) == 31 + assert issubclass(TooManyRedirects, TLSClientException) + + def test_library_error_on_a_hop_is_raised(self, session, fake_lib): + fake_lib.reply_with(**redirect()) + fake_lib.reply_with(status=0, body="failed to do request: connection refused") + + with pytest.raises(TLSClientException, match="connection refused"): + session.get(URL, allow_redirects=True) + + assert fake_lib.freed_ids == ["response-1", "response-2"] + + class TestTransportOptions: @pytest.mark.parametrize( ("proxies", "expected"), @@ -249,7 +495,6 @@ def test_timeout_and_timeout_seconds_together_are_rejected(self, session, fake_l ({"force_http1": True}, {}, "forceHttp1"), ({"catch_panics": True}, {}, "catchPanics"), ({"debug": True}, {}, "withDebug"), - ({}, {"allow_redirects": True}, "followRedirects"), ({}, {"insecure_skip_verify": True}, "insecureSkipVerify"), ], ) diff --git a/tls_client/async_sessions.py b/tls_client/async_sessions.py index a8d5a11..4fca998 100644 --- a/tls_client/async_sessions.py +++ b/tls_client/async_sessions.py @@ -99,7 +99,8 @@ async def execute_request(self, method: str, url: str, **kwargs: Any) -> Respons response = await self._run(request_session.execute_request, method=method, url=url, **kwargs) # Back on the event loop thread, so no other request touches the jar while the new cookies go in. Like Session, - # keep the cookies sent with the request, then the ones the server set. + # keep the cookies sent with the request, then the ones the server set (response.cookies holds those of every + # redirect followed too). cookies = kwargs.get("cookies") if cookies: self._session.cookies.update(cookies) diff --git a/tls_client/cookies.py b/tls_client/cookies.py index 6a39038..d78d696 100644 --- a/tls_client/cookies.py +++ b/tls_client/cookies.py @@ -426,6 +426,18 @@ def merge_cookies(cookiejar: RequestsCookieJar, cookies: dict[str, str] | Cookie return cookiejar +def get_cookie_header(cookie_jar: CookieJar, request_url: str, request_headers: Mapping[str, str | None]) -> str | None: + """Cookie header for the cookies in the jar that may be sent to request_url, or None if there are none. + + The jar decides by domain, path, secure flag and expiry, like a browser. request_headers must not contain a Cookie + header, or the jar adds nothing. + """ + req = MockRequest(request_url, request_headers) + # add_cookie_header only needs the duck-typed subset of Request the mock implements + cookie_jar.add_cookie_header(req) # type: ignore[arg-type] + return req.get_new_headers().get("Cookie") + + def extract_cookies_to_jar( request_url: str, request_headers: Mapping[str, str | None], @@ -444,6 +456,6 @@ def extract_cookies_to_jar( res = MockResponse(http_message) # extract_cookies only needs the duck-typed subset of HTTPResponse and Request the mocks implement response_cookie_jar.extract_cookies(res, req) # type: ignore[arg-type] - - merge_cookies(cookie_jar, response_cookie_jar) + # Extracted into the session jar too rather than merged, so a cookie the server expires is removed from it + cookie_jar.extract_cookies(res, req) # type: ignore[arg-type] return response_cookie_jar diff --git a/tls_client/exceptions.py b/tls_client/exceptions.py index f5283b2..4d20298 100644 --- a/tls_client/exceptions.py +++ b/tls_client/exceptions.py @@ -2,5 +2,9 @@ class TLSClientException(IOError): """General error with the TLS client""" +class TooManyRedirects(TLSClientException): + """A request with allow_redirects was redirected more than 30 times""" + + class UnsupportedPlatformError(TLSClientException): """No native library is available for this OS, CPU architecture or C library""" diff --git a/tls_client/response.py b/tls_client/response.py index c721ca6..2f5b59c 100644 --- a/tls_client/response.py +++ b/tls_client/response.py @@ -20,9 +20,12 @@ def __init__(self) -> None: # Case-insensitive dictionary of Response Headers. Headers sent more than once map to a list of values. self.headers: CaseInsensitiveDict[str | list[str]] = CaseInsensitiveDict() - # A CookieJar of Cookies the server sent back. + # A CookieJar of Cookies the server sent back, on every redirect that led to this response too. self.cookies = cookiejar_from_dict({}) + # Responses of the redirects followed to get this one, oldest first. + self.history: list[Response] = [] + # Encoding used to decode .text: the charset of the Content-Type header, else a BOM or the charset the document # declares in a tag. Set it to decode the body with another encoding; None means UTF-8. self.encoding: str | None = None diff --git a/tls_client/sessions.py b/tls_client/sessions.py index ffa4a86..089925d 100644 --- a/tls_client/sessions.py +++ b/tls_client/sessions.py @@ -10,8 +10,8 @@ from .__version__ import __version__ from .cffi import destroySession, freeMemory, request -from .cookies import cookiejar_from_dict, extract_cookies_to_jar, merge_cookies -from .exceptions import TLSClientException +from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header, merge_cookies +from .exceptions import TLSClientException, TooManyRedirects from .multipart import Files, encode_multipart from .response import Response, build_response from .settings import CLIENT_IDENTIFIERS, ClientIdentifiers @@ -24,6 +24,35 @@ # Either a proxy URL, or a dict whose "http" entry is used. _Proxy: TypeAlias = dict[str, str] | str +# Statuses followed with allow_redirects, if the response has a Location header +_REDIRECT_STATUSES = frozenset({301, 302, 303, 307, 308}) +# Redirects followed for one request before TooManyRedirects is raised, the same limit as in requests +MAX_REDIRECTS = 30 +_DEFAULT_PORTS = {"http": 80, "https": 443} + + +def _redirect_method(status_code: int, method: str) -> str: + """Method of the request that follows a redirect, as browsers pick it: 303 turns anything but HEAD into a GET, and + 301 and 302 turn a POST into one. 307 and 308 keep the method.""" + if status_code == 303 and method.upper() != "HEAD": + return "GET" + if status_code in (301, 302) and method.upper() == "POST": + return "GET" + return method + + +def _leaves_origin(url: str, next_url: str) -> bool: + """Whether a redirect from url to next_url changes host, port or scheme. An upgrade from http to https on the + default ports doesn't count, like in requests.""" + old, new = urllib.parse.urlsplit(url), urllib.parse.urlsplit(next_url) + if old.hostname != new.hostname: + return True + old_port = old.port or _DEFAULT_PORTS.get(old.scheme) + new_port = new.port or _DEFAULT_PORTS.get(new.scheme) + if old.scheme == "http" and old_port == 80 and new.scheme == "https" and new_port == 443: + return False + return old.scheme != new.scheme or old_port != new_port + class Session: def __init__( @@ -403,21 +432,12 @@ def execute_request( request_header_order = header_order # --- Cookies -------------------------------------------------------------------------------------------------- - # Merge with session cookies + # Request cookies are added to the session jar. The jar is the only cookie store: the library runs without one + # (withoutCookieJar), so a cookie removed from the jar is no longer sent. cookie_jar = merge_cookies(self.cookies, cookies or {}) - # turn cookie jar into dict - # in the cookie value the " gets removed, because the fhttp library in golang doesn't accept the character - # expires is omitted for session cookies, because the tls client parses null as 1970 and drops the cookie - request_cookies = [ - { - "domain": c.domain, - "name": c.name, - "path": c.path, - "value": (c.value or "").replace('"', ""), - **({"expires": c.expires} if c.expires is not None else {}), - } - for c in cookie_jar - ] + # The jar's cookies are added to each request, redirects included, as a Cookie header. A Cookie header passed in + # is sent before them. + passed_cookie_header = request_headers.pop("Cookie", None) # --- Proxy ---------------------------------------------------------------------------------------------------- proxy = proxy or self.proxies @@ -442,27 +462,23 @@ def execute_request( certificate_pinning = self.certificate_pinning # --- Request -------------------------------------------------------------------------------------------------- - is_byte_request = isinstance(request_body, (bytes, bytearray)) + # Settings shared by every request of a redirect chain. The URL, method, headers and body are set per request. request_payload: dict[str, Any] = { "sessionId": self._session_id, - "followRedirects": allow_redirects, + # Redirects are followed below, so that the cookies set on every hop end up in self.cookies. + "followRedirects": False, + # Cookies come from self.cookies only. Without this, the library also keeps the cookies of every response + # in a jar of its own and sends them on later requests of the session. + "withoutCookieJar": True, "forceHttp1": self.force_http1, "withDebug": self.debug, "catchPanics": self.catch_panics, - "headers": dict(request_headers), "headerOrder": request_header_order, "insecureSkipVerify": insecure_skip_verify, - "isByteRequest": is_byte_request, # Return the body as base64 so binary bodies survive; build_response decodes it. "isByteResponse": True, "additionalDecode": self.additional_decode, "proxyUrl": proxy_url, - "requestUrl": url, - "requestMethod": method, - "requestBody": base64.b64encode(request_body).decode() - if isinstance(request_body, (bytes, bytearray)) - else request_body, - "requestCookies": request_cookies, # Milliseconds, so fractions of a second work. The library rejects timeoutSeconds alongside it. "timeoutMilliseconds": round(request_timeout * 1000), } @@ -489,24 +505,79 @@ def execute_request( request_payload["tlsClientIdentifier"] = self.client_identifier request_payload["withRandomTLSExtensionOrder"] = self.random_tls_extension_order - # dereference the pointer to the response, decode it and parse it (tls client returns json). None of the steps - # is bound to a name, so each copy of a large reply is freed as soon as the next one exists. - response_object = loads(ctypes.string_at(request(dumps(request_payload).encode("utf-8"))).decode("utf-8")) - # free the memory - freeMemory(response_object["id"].encode("utf-8")) - # --- Response ------------------------------------------------------------------------------------------------- - # Error handling - if response_object["status"] == 0: - raise TLSClientException(response_object["body"]) - # Set response cookies - response_cookie_jar = extract_cookies_to_jar( - request_url=url, - request_headers=request_headers, - cookie_jar=cookie_jar, - response_headers=response_object["headers"], - ) - # build response class - return build_response(response_object, response_cookie_jar) + # Sends the request, then with allow_redirects each request a redirect asks for, like requests does. + history: list[Response] = [] + # Cookies set on any hop of a redirect chain, for the final response + chain_cookies = cookiejar_from_dict({}) + while True: + # The jar picks the cookies for the URL by domain, path, secure flag and expiry. + sent_headers = CaseInsensitiveDict(request_headers) + cookie_header = "; ".join( + filter(None, [passed_cookie_header, get_cookie_header(cookie_jar, url, request_headers)]) + ) + if cookie_header: + sent_headers["Cookie"] = cookie_header + request_payload["headers"] = dict(sent_headers) + request_payload["requestUrl"] = url + request_payload["requestMethod"] = method + request_payload["isByteRequest"] = isinstance(request_body, (bytes, bytearray)) + request_payload["requestBody"] = ( + base64.b64encode(request_body).decode() + if isinstance(request_body, (bytes, bytearray)) + else request_body + ) + + # dereference the pointer to the response, decode it and parse it (tls client returns json). None of the + # steps is bound to a name, so each copy of a large reply is freed as soon as the next one exists. + response_object = loads(ctypes.string_at(request(dumps(request_payload).encode("utf-8"))).decode("utf-8")) + # free the memory + freeMemory(response_object["id"].encode("utf-8")) + # --- Response --------------------------------------------------------------------------------------------- + # Error handling + if response_object["status"] == 0: + raise TLSClientException(response_object["body"]) + # Set response cookies + response_cookie_jar = extract_cookies_to_jar( + request_url=url, + request_headers=request_headers, + cookie_jar=cookie_jar, + response_headers=response_object["headers"], + ) + chain_cookies.update(response_cookie_jar) + # build response class + response = build_response(response_object, response_cookie_jar) + + # --- Redirects -------------------------------------------------------------------------------------------- + location = response.headers.get("Location") + if not allow_redirects or response.status_code not in _REDIRECT_STATUSES or location is None: + break + if len(history) == MAX_REDIRECTS: + raise TooManyRedirects(f"Exceeded {MAX_REDIRECTS} redirects, the last from {url}") + history.append(response) + + # A relative Location is resolved against the current URL. Like in requests, the fragment carries over + # unless the Location has one of its own. + next_url = urllib.parse.urljoin(url, location if isinstance(location, str) else location[0]) + fragment = urllib.parse.urlsplit(url).fragment + if fragment and not urllib.parse.urlsplit(next_url).fragment: + next_url = f"{next_url}#{fragment}" + # Credentials passed for one host aren't sent to another + if _leaves_origin(url, next_url): + request_headers.pop("Authorization", None) + passed_cookie_header = None + url = next_url + + next_method = _redirect_method(response.status_code, method) + if next_method != method: + # The request turned into a GET, which has no body + method = next_method + request_body = None + for header in ("Content-Type", "Content-Length", "Transfer-Encoding"): + request_headers.pop(header, None) + + response.history = history + response.cookies = chain_cookies + return response def get(self, url: str, **kwargs: Any) -> Response: """Sends a GET request"""