Merge pull request #412 from reef-technologies/5xx-token-refresh

Avoid b2http retries for upload requests

Author: olzhasar-reef

b2sdk/_internal/b2http.py

@@ -36,12 +36,12 @@
     B2ConnectionError,
     B2Error,
     B2RequestTimeout,
-    B2RequestTimeoutDuringUpload,
     BadDateFormat,
     BrokenPipe,
     ClockSkew,
     ConnectionReset,
     PotentialS3EndpointPassedAsRealm,
+    ServiceError,
     UnknownError,
     UnknownHost,
     interpret_b2_error,
@@ -281,7 +281,7 @@ def do_request():
             self._run_post_request_hooks(method, url, request_headers, response)
             return response
 
-        return self._translate_and_retry(do_request, try_count, params)
+        return self._translate_and_retry(do_request, try_count, method, request_headers, params)
 
     def request_content_return_json(
         self,
@@ -355,14 +355,9 @@ def post_content_return_json(
         :param data: a file-like object to send
         :return: a dict that is the decoded JSON
         """
-        try:
-            return self.request_content_return_json(
-                'POST', url, headers, data, try_count, post_params, _timeout=_timeout
-            )
-        except B2RequestTimeout:
-            # this forces a token refresh, which is necessary if request is still alive
-            # on the server but has terminated for some reason on the client. See #79
-            raise B2RequestTimeoutDuringUpload()
+        return self.request_content_return_json(
+            'POST', url, headers, data, try_count, post_params, _timeout=_timeout
+        )
 
     def post_json_return_json(self, url, headers, params, try_count: int = TRY_COUNT_OTHER):
         """
@@ -579,7 +574,12 @@ def _translate_errors(cls, fcn, post_params=None):
 
     @classmethod
     def _translate_and_retry(
-        cls, fcn: Callable, try_count: int, post_params: dict[str, Any] | None = None
+        cls,
+        fcn: Callable,
+        try_count: int,
+        method: str,
+        headers: dict[str, Any],
+        post_params: dict[str, Any] | None = None,
     ):
         """
         Try calling fcn try_count times, retrying only if
@@ -598,6 +598,15 @@ def _translate_and_retry(
             except B2Error as e:
                 if not e.should_retry_http():
                     raise
+                if (
+                    method == 'POST'
+                    and 'X-Bz-Content-Sha1' in headers
+                    and isinstance(e, (ServiceError, B2RequestTimeout))
+                ):
+                    # This is an upload operation, so we avoid http-level retries
+                    # here to force an upload token refresh
+                    raise
+
                 logger.debug(str(e), exc_info=True)
                 if e.retry_after_seconds is not None:
                     sleep_duration = e.retry_after_seconds

changelog.d/+b2http-upload-no-retries.fixed.md

@@ -0,0 +1 @@
+Avoid http-level retries during upload requests.

test/unit/b2http/test_b2http.py

@@ -387,6 +387,37 @@ def test_too_many_requests_retry_header_combination_two(
         b2_http.request(responses.GET, self.URL, {}, try_count=4)
         assert mock_time.mock_calls == [call(1.0), call(5), call(2.25)]
 
+    @responses.activate
+    def test_service_error_during_upload_no_retries(self, b2_http: B2Http, mock_time: MagicMock):
+        _mock_error_response(self.URL, method=responses.POST, status=503)
+        responses.post(self.URL)
+
+        headers = {'X-Bz-Content-Sha1': '1234'}
+
+        with pytest.raises(ServiceError):
+            b2_http.request(responses.POST, self.URL, headers)
+
+        mock_time.assert_not_called()
+
+    @responses.activate
+    def test_request_timeout_during_upload_no_retries(self, b2_http: B2Http, mock_time: MagicMock):
+        responses.post(
+            self.URL,
+            body=requests.ConnectionError(
+                requests.packages.urllib3.exceptions.ProtocolError(
+                    'dummy', TimeoutError('The write operation timed out')
+                )
+            ),
+        )
+        responses.post(self.URL)
+
+        headers = {'X-Bz-Content-Sha1': '1234'}
+
+        with pytest.raises(B2RequestTimeout):
+            b2_http.request(responses.POST, self.URL, headers)
+
+        mock_time.assert_not_called()
+
 
 class TestB2Http:
     URL = 'http://example.com'