From bcb2b4722f0f07106f052a0279c6c8f8363e852d Mon Sep 17 00:00:00 2001 From: Kyle Knapp Date: Tue, 2 Sep 2025 12:24:16 -0700 Subject: [PATCH 1/2] Respect protocol tuple when stripping full URIs This change makes AzureBlobFileSystem consistent with other fsspec implementations that use the protocol class tuple when stripping the protocol from fully qualified URIs. It also unblocks consumers to be able to update this protocol tuple if out of the box adlfs does not respect a particular protocol (e.g., deprecated wasb://). --- CHANGELOG.md | 7 +++++++ adlfs/spec.py | 8 ++++++-- adlfs/tests/test_spec.py | 39 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3cf86b4..e917e855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ Unreleased ---------- +* Respect `AzureBlobFileSystem.protocol` tuple when removing protocols from fully-qualified + paths provided to `AzureBlobFileSystem` methods. + + +2025.8.0 +-------- + - Added "adlfs" to library's default user agent - Fix issue where ``AzureBlobFile`` did not respect ``location_mode`` parameter from parent ``AzureBlobFileSystem`` when using SAS credentials and connecting to diff --git a/adlfs/spec.py b/adlfs/spec.py index bfc13bda..38477df2 100644 --- a/adlfs/spec.py +++ b/adlfs/spec.py @@ -261,7 +261,7 @@ class AzureBlobFileSystem(AsyncFileSystem): ... 'account_name': ACCOUNT_NAME, 'account_key': ACCOUNT_KEY,}) """ - protocol = "abfs" + protocol = ("abfs", "az", "abfss") def __init__( self, @@ -398,7 +398,11 @@ def _strip_protocol(cls, path: str): STORE_SUFFIX = ".dfs.core.windows.net" logger.debug(f"_strip_protocol for {path}") - if not path.startswith(("abfs://", "az://", "abfss://")): + if isinstance(cls.protocol, str): + protocol_startswith = (f"{cls.protocol}://",) + else: + protocol_startswith = tuple([f"{proto}://" for proto in cls.protocol]) + if not path.startswith(protocol_startswith): path = path.lstrip("/") path = "abfs://" + path ops = infer_storage_options(path) diff --git a/adlfs/tests/test_spec.py b/adlfs/tests/test_spec.py index 0639c2d0..bb5d8f78 100644 --- a/adlfs/tests/test_spec.py +++ b/adlfs/tests/test_spec.py @@ -1776,7 +1776,7 @@ def test_find_with_prefix(storage): ] -@pytest.mark.parametrize("proto", [None, "abfs://", "az://"]) +@pytest.mark.parametrize("proto", [None, "abfs://", "az://", "abfss://"]) @pytest.mark.parametrize("path", ["container/file", "container/file?versionid=1234"]) def test_strip_protocol(proto, path): assert ( @@ -1784,7 +1784,7 @@ def test_strip_protocol(proto, path): ) -@pytest.mark.parametrize("proto", ["", "abfs://", "az://"]) +@pytest.mark.parametrize("proto", ["", "abfs://", "az://", "abfss://"]) @pytest.mark.parametrize("key", ["file", "dir/file"]) @pytest.mark.parametrize("version_aware", [True, False]) @pytest.mark.parametrize("version_id", [None, "1970-01-01T00:00:00.0000000Z"]) @@ -1805,6 +1805,41 @@ def test_split_path(storage, proto, key, version_aware, version_id): ) +@pytest.mark.parametrize("unsupported_proto", ["unsupported", "wasb", "wasbs"]) +def test_can_update_default_supported_protocols(storage, mocker, unsupported_proto): + mocker.patch.object( + AzureBlobFileSystem, + "protocol", + AzureBlobFileSystem.protocol + (unsupported_proto,), + ) + fs = AzureBlobFileSystem( + account_name=storage.account_name, + connection_string=CONN_STR, + skip_instance_cache=True, + ) + assert unsupported_proto in fs.protocol + assert ( + fs._strip_protocol(f"{unsupported_proto}://container/file") == "container/file" + ) + assert fs.split_path(f"{unsupported_proto}://container/file") == ( + "container", + "file", + None, + ) + + +def test_can_restrict_protocol_to_single_string(storage, mocker): + mocker.patch.object(AzureBlobFileSystem, "protocol", "abfs") + fs = AzureBlobFileSystem( + account_name=storage.account_name, + connection_string=CONN_STR, + skip_instance_cache=True, + ) + assert fs.protocol == "abfs" + assert fs._strip_protocol("abfs://container/file") == "container/file" + assert fs.split_path("abfs://container/file") == ("container", "file", None) + + async def test_details_versioned(storage): from azure.storage.blob import BlobProperties From 77670e5c55572cff0dd9e6306184256dfc82bd23 Mon Sep 17 00:00:00 2001 From: Kyle Knapp Date: Wed, 3 Sep 2025 14:04:01 -0700 Subject: [PATCH 2/2] Update based on feedback --- adlfs/spec.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/adlfs/spec.py b/adlfs/spec.py index 38477df2..5ff509e1 100644 --- a/adlfs/spec.py +++ b/adlfs/spec.py @@ -399,9 +399,13 @@ def _strip_protocol(cls, path: str): STORE_SUFFIX = ".dfs.core.windows.net" logger.debug(f"_strip_protocol for {path}") if isinstance(cls.protocol, str): + # The protocol can be either a string or a tuple of strings. + # While the default protocol is a tuple, a user can override the protocol to be a string. + # So, this handles the case where the user has overridden the protocol, restricting + # supported protocols to a single string. protocol_startswith = (f"{cls.protocol}://",) else: - protocol_startswith = tuple([f"{proto}://" for proto in cls.protocol]) + protocol_startswith = tuple(f"{proto}://" for proto in cls.protocol) if not path.startswith(protocol_startswith): path = path.lstrip("/") path = "abfs://" + path