Skip to content

WIP: refactor: retry logging #78

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
vgavro wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

WIP: refactor: retry logging #78

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 8 additions & 11 deletions httpx_retries/retry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import random
import sys
import time
from collections.abc import Iterable, Mapping
from collections.abc import Iterable
from email.utils import parsedate_to_datetime
from enum import Enum
from http import HTTPStatus
Expand Down Expand Up @@ -226,12 +226,12 @@ def backoff_strategy(self) -> float:

return min(backoff, self.max_backoff_wait)

def _calculate_sleep(self, headers: httpx.Headers | Mapping[str, str]) -> float:
def calculate_sleep(self, response: httpx.Response | Exception) -> float:
"""Calculate the sleep duration based on headers and backoff strategy."""
sleep_time = 0.0
# Check Retry-After header first if enabled
if self.respect_retry_after_header:
retry_after = headers.get("Retry-After", "").strip()
if isinstance(response, httpx.Response) and self.respect_retry_after_header:
retry_after = response.headers.get("Retry-After", "").strip()
if retry_after:
try:
retry_after_sleep = min(self.parse_retry_after(retry_after), self.max_backoff_wait)
Expand All @@ -251,35 +251,32 @@ def _calculate_sleep(self, headers: httpx.Headers | Mapping[str, str]) -> float:

return sleep_time

def sleep(self, response: httpx.Response | Exception) -> None:
def sleep(self, response: httpx.Response | Exception | float) -> None:
"""
Sleep between retry attempts using the calculated duration.

This method will respect a server’s `Retry-After` response header and sleep the duration
of the time requested. If that is not present, it will use an exponential backoff. By default,
the backoff factor is 0 and this method will return immediately.
"""
time_to_sleep = self._calculate_sleep(response.headers if isinstance(response, httpx.Response) else {})
logger.debug("sleep seconds=%s", time_to_sleep)
time_to_sleep = response if isinstance(response, (int, float)) else self.calculate_sleep(response)
time.sleep(time_to_sleep)
self.elapsed_sleep += time_to_sleep

async def asleep(self, response: httpx.Response | Exception) -> None:
async def asleep(self, response: httpx.Response | Exception | float) -> None:
"""
Sleep between retry attempts asynchronously using the calculated duration.

This method will respect a server’s `Retry-After` response header and sleep the duration
of the time requested. If that is not present, it will use an exponential backoff. By default,
the backoff factor is 0 and this method will return immediately.
"""
time_to_sleep = self._calculate_sleep(response.headers if isinstance(response, httpx.Response) else {})
logger.debug("asleep seconds=%s", time_to_sleep)
time_to_sleep = response if isinstance(response, (int, float)) else self.calculate_sleep(response)
await asyncio.sleep(time_to_sleep)
self.elapsed_sleep += time_to_sleep

def increment(self) -> "Retry":
"""Return a new Retry instance with the attempt count incremented."""
logger.debug("increment retry=%s new_attempts_made=%s", self, self.attempts_made + 1)
return self.__class__(
total=self.total,
max_backoff_wait=self.max_backoff_wait,
Expand Down
46 changes: 30 additions & 16 deletions httpx_retries/transport.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,16 +85,12 @@ def handle_request(self, request: httpx.Request) -> httpx.Response:
if self._sync_transport is None:
raise RuntimeError("Synchronous request received but no sync transport available")

logger.debug("handle_request started request=%s", request)

if self.retry.is_retryable_method(request.method):
send_method = partial(self._sync_transport.handle_request)
response = self._retry_operation(request, send_method)
else:
response = self._sync_transport.handle_request(request)

logger.debug("handle_request finished request=%s response=%s", request, response)

return response

async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
Expand All @@ -109,16 +105,12 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
if self._async_transport is None:
raise RuntimeError("Async request received but no async transport available")

logger.debug("handle_async_request started request=%s", request)

if self.retry.is_retryable_method(request.method):
send_method = partial(self._async_transport.handle_async_request)
response = await self._retry_operation_async(request, send_method)
else:
response = await self._async_transport.handle_async_request(request)

logger.debug("handle_async_request finished request=%s response=%s", request, response)

return response

def _retry_operation(
Expand All @@ -134,9 +126,7 @@ def _retry_operation(
if isinstance(response, httpx.Response):
response.close()

logger.debug("_retry_operation retrying request=%s response=%s retry=%s", request, response, retry)
retry = retry.increment()
retry.sleep(response)
retry = self._retry_increment(request, response, retry)
try:
response = send_method(request)
except Exception as e:
Expand All @@ -149,6 +139,19 @@ def _retry_operation(
if retry.is_exhausted() or not retry.is_retryable_status_code(response.status_code):
return response

def _retry_increment(self, request: httpx.Request, response: httpx.Response | Exception, retry: Retry) -> Retry:
time_to_sleep = retry.calculate_sleep(response)
logger.debug(
"retry request=%s response=%s retry=%s sleep=%s",
request,
response,
retry,
time_to_sleep,
)
retry = retry.increment()
retry.sleep(time_to_sleep)
return retry

async def _retry_operation_async(
self,
request: httpx.Request,
Expand All @@ -162,11 +165,7 @@ async def _retry_operation_async(
if isinstance(response, httpx.Response):
await response.aclose()

logger.debug(
"_retry_operation_async retrying request=%s response=%s retry=%s", request, response, retry
)
retry = retry.increment()
await retry.asleep(response)
retry = await self._retry_increment_async(request, response, retry)
try:
response = await send_method(request)
except Exception as e:
Expand All @@ -178,3 +177,18 @@ async def _retry_operation_async(

if retry.is_exhausted() or not retry.is_retryable_status_code(response.status_code):
return response

async def _retry_increment_async(
self, request: httpx.Request, response: httpx.Response | Exception, retry: Retry
) -> Retry:
time_to_sleep = retry.calculate_sleep(response)
logger.debug(
"retry request=%s response=%s retry=%s sleep=%s",
request,
response,
retry,
time_to_sleep,
)
retry = retry.increment()
await retry.asleep(time_to_sleep)
return retry
Loading