From 250f86b479d4fe7a7c17ce082bd412686903286f Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 21 May 2026 17:22:36 +0000 Subject: [PATCH 1/5] Retry integration tests on transient service errors --- b2sdk/_internal/exception.py | 11 +++++++- .../testing/helpers/bucket_manager.py | 11 ++++++-- changelog.d/1232.changed.md | 1 + test/integration/conftest.py | 28 +++++++++++++++++++ test/integration/test_upload.py | 24 +++++++++++++++- test/unit/test_exception.py | 3 ++ 6 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 changelog.d/1232.changed.md diff --git a/b2sdk/_internal/exception.py b/b2sdk/_internal/exception.py index 11a574753..2cf50dd74 100644 --- a/b2sdk/_internal/exception.py +++ b/b2sdk/_internal/exception.py @@ -433,6 +433,15 @@ class ServiceError(TransientErrorMixin, B2Error): Used for HTTP status codes 500 through 599. """ + def __init__(self, status, code, message): + super().__init__() + self._status = status + self._code = code + self._message = message + + def __str__(self): + return f'{self._status} {self._code} {self._message}' + class CapExceeded(B2Error): def __str__(self): @@ -744,5 +753,5 @@ def interpret_b2_error( elif status == 429: return TooManyRequests(retry_after_seconds=response_headers.get('retry-after')) elif 500 <= status < 600: - return ServiceError('%d %s %s' % (status, code, message)) + return ServiceError(status, code, message) return UnknownError('%d %s %s' % (status, code, message)) diff --git a/b2sdk/_internal/testing/helpers/bucket_manager.py b/b2sdk/_internal/testing/helpers/bucket_manager.py index bc76ef3b7..9c4230bb8 100644 --- a/b2sdk/_internal/testing/helpers/bucket_manager.py +++ b/b2sdk/_internal/testing/helpers/bucket_manager.py @@ -25,6 +25,7 @@ BucketIdNotFound, DuplicateBucketName, FileNotPresent, + ServiceError, TooManyRequests, ) from b2sdk._internal.file_lock import NO_RETENTION_FILE_SETTING, LegalHold, RetentionMode @@ -43,6 +44,12 @@ logger = logging.getLogger(__name__) +def _retry_bucket_test_operation(exc: BaseException) -> bool: + return isinstance(exc, TooManyRequests) or ( + isinstance(exc, ServiceError) and exc._status == 503 + ) + + class BucketManager: def __init__( self, @@ -83,7 +90,7 @@ def new_bucket_info(self) -> dict: } @tenacity.retry( - retry=tenacity.retry_if_exception_type(TooManyRequests), + retry=tenacity.retry_if_exception(_retry_bucket_test_operation), wait=tenacity.wait_exponential(), stop=tenacity.stop_after_attempt(8), ) @@ -152,7 +159,7 @@ def clean_buckets(self, quick=False): print(bucket) @tenacity.retry( - retry=tenacity.retry_if_exception_type(TooManyRequests), + retry=tenacity.retry_if_exception(_retry_bucket_test_operation), wait=tenacity.wait_exponential(), stop=tenacity.stop_after_attempt(8), ) diff --git a/changelog.d/1232.changed.md b/changelog.d/1232.changed.md new file mode 100644 index 000000000..ec9087412 --- /dev/null +++ b/changelog.d/1232.changed.md @@ -0,0 +1 @@ +Retry integration tests automatically when they fail because of a transient HTTP 503 from B2. diff --git a/test/integration/conftest.py b/test/integration/conftest.py index d700e2fd3..3d683a3a8 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -7,9 +7,37 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations + import pytest +from b2sdk._internal.exception import ServiceError + +RETRYABLE_SERVICE_ERROR_STATUSES = {500, 503} +INTEGRATION_TEST_RETRY_COUNT = 2 + @pytest.fixture(scope='session', autouse=True) def auto_change_account_info_dir(change_account_info_dir): pass + + +@pytest.hookimpl(tryfirst=True) +def pytest_pyfunc_call(pyfuncitem): + testfunction = pyfuncitem.obj + funcargs = pyfuncitem.funcargs + testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} + + for attempt in range(INTEGRATION_TEST_RETRY_COUNT + 1): + try: + testfunction(**testargs) + return True + except ServiceError as exc: + if exc._status not in RETRYABLE_SERVICE_ERROR_STATUSES: + raise + if attempt >= INTEGRATION_TEST_RETRY_COUNT: + raise + print( + f'Retrying {pyfuncitem.nodeid} after transient service error {exc._status}:' + f' attempt {attempt + 1} of {INTEGRATION_TEST_RETRY_COUNT}' + ) diff --git a/test/integration/test_upload.py b/test/integration/test_upload.py index 8dbebcae2..b38ce9d19 100644 --- a/test/integration/test_upload.py +++ b/test/integration/test_upload.py @@ -10,8 +10,9 @@ from __future__ import annotations import io +import logging -from b2sdk._internal.b2http import B2Http +from b2sdk._internal.b2http import B2Http, HttpCallback from b2sdk._internal.encryption.setting import EncryptionKey, EncryptionSetting from b2sdk._internal.encryption.types import EncryptionAlgorithm, EncryptionMode from b2sdk.v2 import B2RawHTTPApi @@ -19,6 +20,15 @@ from .test_raw_api import authorize_raw_api +logger = logging.getLogger(__name__) + + +class FailSomeUploads(HttpCallback): + def pre_request(self, method, url, headers): + if method == 'POST' and 'b2_upload_file' in url: + headers['X-Bz-Test-Mode'] = 'fail_some_uploads' + logger.info('Added X-Bz-Test-Mode=fail_some_uploads header to %s', url) + class TestUnboundStreamUpload(IntegrationTestBase): def assert_data_uploaded_via_stream(self, data: bytes, part_size: int | None = None): @@ -46,6 +56,18 @@ def test_streamed_large_buffer_small_part_size(self): class TestUploadLargeFile(IntegrationTestBase): + def test_upload_bytes_with_intermittent_failures(self): + bucket = self.create_bucket() + b2_http = self.b2_api.session.raw_api.b2_http + callback = FailSomeUploads() + b2_http.add_callback(callback) + try: + for i in range(10): + payload = f'payload-{i}'.encode() + bucket.upload_bytes(payload, f'fail-some-uploads-{i}') + finally: + b2_http.callbacks.remove(callback) + def test_ssec_key_id(self): sse_c = EncryptionSetting( mode=EncryptionMode.SSE_C, diff --git a/test/unit/test_exception.py b/test/unit/test_exception.py index 258e301a4..5e44af7cc 100644 --- a/test/unit/test_exception.py +++ b/test/unit/test_exception.py @@ -159,6 +159,9 @@ def test_bad_bucket_id(self): def test_service_error(self): error = interpret_b2_error(500, 'code', 'message', {}) assert isinstance(error, ServiceError) + assert error._status == 500 + assert error._code == 'code' + assert error._message == 'message' assert '500 code message' == str(error) def test_unknown_error(self): From 9db5f076d0c4b342dc6dd10cd27c7232cb437826 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 21 May 2026 17:28:46 +0000 Subject: [PATCH 2/5] Exercise raw upload retries in integration CI --- test/integration/test_upload.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/test/integration/test_upload.py b/test/integration/test_upload.py index b38ce9d19..224497bab 100644 --- a/test/integration/test_upload.py +++ b/test/integration/test_upload.py @@ -11,10 +11,14 @@ import io import logging +import secrets + +import pytest from b2sdk._internal.b2http import B2Http, HttpCallback from b2sdk._internal.encryption.setting import EncryptionKey, EncryptionSetting from b2sdk._internal.encryption.types import EncryptionAlgorithm, EncryptionMode +from b2sdk._internal.utils import hex_sha1_of_stream from b2sdk.v2 import B2RawHTTPApi from b2sdk.v3.testing import IntegrationTestBase @@ -56,15 +60,32 @@ def test_streamed_large_buffer_small_part_size(self): class TestUploadLargeFile(IntegrationTestBase): - def test_upload_bytes_with_intermittent_failures(self): + @pytest.mark.parametrize('upload_number', range(10)) + def test_raw_upload_with_intermittent_failures(self, upload_number): bucket = self.create_bucket() - b2_http = self.b2_api.session.raw_api.b2_http + raw_api = self.b2_api.session.raw_api + b2_http = raw_api.b2_http + account_info = self.b2_api.account_info callback = FailSomeUploads() b2_http.add_callback(callback) try: - for i in range(10): - payload = f'payload-{i}'.encode() - bucket.upload_bytes(payload, f'fail-some-uploads-{i}') + payload = f'payload-{upload_number}'.encode() + file_name = f'fail-some-uploads-{upload_number}-{secrets.token_hex(4)}' + upload_url = raw_api.get_upload_url( + account_info.get_api_url(), + account_info.get_account_auth_token(), + bucket.id_, + ) + raw_api.upload_file( + upload_url['uploadUrl'], + upload_url['authorizationToken'], + file_name, + len(payload), + 'text/plain', + hex_sha1_of_stream(io.BytesIO(payload), len(payload)), + {}, + io.BytesIO(payload), + ) finally: b2_http.callbacks.remove(callback) From 84bcfe0b4abf33a87d707e9af0e856424ff5c8e3 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 21 May 2026 17:33:26 +0000 Subject: [PATCH 3/5] Increase integration retry budget for transient service errors --- test/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 3d683a3a8..5acbf6fa6 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -14,7 +14,7 @@ from b2sdk._internal.exception import ServiceError RETRYABLE_SERVICE_ERROR_STATUSES = {500, 503} -INTEGRATION_TEST_RETRY_COUNT = 2 +INTEGRATION_TEST_RETRY_COUNT = 4 @pytest.fixture(scope='session', autouse=True) From 2ec8bc22d8a50dbe71f0a7e18ddc268508b3c7e5 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 21 May 2026 17:39:28 +0000 Subject: [PATCH 4/5] Reduce fail_some_uploads integration load --- test/integration/test_upload.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/integration/test_upload.py b/test/integration/test_upload.py index 224497bab..15e8e01a7 100644 --- a/test/integration/test_upload.py +++ b/test/integration/test_upload.py @@ -13,8 +13,6 @@ import logging import secrets -import pytest - from b2sdk._internal.b2http import B2Http, HttpCallback from b2sdk._internal.encryption.setting import EncryptionKey, EncryptionSetting from b2sdk._internal.encryption.types import EncryptionAlgorithm, EncryptionMode @@ -60,8 +58,7 @@ def test_streamed_large_buffer_small_part_size(self): class TestUploadLargeFile(IntegrationTestBase): - @pytest.mark.parametrize('upload_number', range(10)) - def test_raw_upload_with_intermittent_failures(self, upload_number): + def test_raw_upload_with_intermittent_failures(self): bucket = self.create_bucket() raw_api = self.b2_api.session.raw_api b2_http = raw_api.b2_http @@ -69,8 +66,8 @@ def test_raw_upload_with_intermittent_failures(self, upload_number): callback = FailSomeUploads() b2_http.add_callback(callback) try: - payload = f'payload-{upload_number}'.encode() - file_name = f'fail-some-uploads-{upload_number}-{secrets.token_hex(4)}' + payload = b'payload' + file_name = f'fail-some-uploads-{secrets.token_hex(4)}' upload_url = raw_api.get_upload_url( account_info.get_api_url(), account_info.get_account_auth_token(), From 8a1001a74bd3fff021532ce0f8063b3345919de8 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 21 May 2026 19:24:59 +0000 Subject: [PATCH 5/5] Retry integration tests on too many requests --- test/integration/conftest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 5acbf6fa6..1544f62d3 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -11,7 +11,7 @@ import pytest -from b2sdk._internal.exception import ServiceError +from b2sdk._internal.exception import ServiceError, TooManyRequests RETRYABLE_SERVICE_ERROR_STATUSES = {500, 503} INTEGRATION_TEST_RETRY_COUNT = 4 @@ -41,3 +41,10 @@ def pytest_pyfunc_call(pyfuncitem): f'Retrying {pyfuncitem.nodeid} after transient service error {exc._status}:' f' attempt {attempt + 1} of {INTEGRATION_TEST_RETRY_COUNT}' ) + except TooManyRequests: + if attempt >= INTEGRATION_TEST_RETRY_COUNT: + raise + print( + f'Retrying {pyfuncitem.nodeid} after transient too many requests:' + f' attempt {attempt + 1} of {INTEGRATION_TEST_RETRY_COUNT}' + )