Skip to content

Commit 12b4ccb

Browse files
committed
fix(io): pass GCS token expiry to gcsfs
1 parent 27a5162 commit 12b4ccb

2 files changed

Lines changed: 33 additions & 1 deletion

File tree

‎pyiceberg/io/fsspec.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
GCS_SERVICE_HOST,
6767
GCS_SESSION_KWARGS,
6868
GCS_TOKEN,
69+
GCS_TOKEN_EXPIRES_AT_MS,
6970
GCS_VERSION_AWARE,
7071
HF_ENDPOINT,
7172
HF_TOKEN,
@@ -95,6 +96,7 @@
9596
)
9697
from pyiceberg.typedef import Properties
9798
from pyiceberg.types import strtobool
99+
from pyiceberg.utils.datetime import millis_to_datetime
98100
from pyiceberg.utils.properties import get_first_property_value, get_header_properties, property_as_bool
99101

100102
logger = logging.getLogger(__name__)
@@ -246,11 +248,16 @@ def _s3(properties: Properties) -> AbstractFileSystem:
246248
def _gs(properties: Properties) -> AbstractFileSystem:
247249
# https://gcsfs.readthedocs.io/en/latest/api.html#gcsfs.core.GCSFileSystem
248250
from gcsfs import GCSFileSystem
251+
from google.oauth2.credentials import Credentials
252+
253+
token: str | Credentials | None = properties.get(GCS_TOKEN)
254+
if token and (expires_at := properties.get(GCS_TOKEN_EXPIRES_AT_MS)):
255+
token = Credentials(token, expiry=millis_to_datetime(int(expires_at)))
249256

250257
return GCSFileSystem(
251258
project=properties.get(GCS_PROJECT_ID),
252259
access=properties.get(GCS_ACCESS, "full_control"),
253-
token=properties.get(GCS_TOKEN),
260+
token=token,
254261
consistency=properties.get(GCS_CONSISTENCY, "none"),
255262
cache_timeout=properties.get(GCS_CACHE_TIMEOUT),
256263
requester_pays=property_as_bool(properties, GCS_REQUESTER_PAYS, False),

‎tests/io/test_fsspec.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import tempfile
2121
import threading
2222
import uuid
23+
from datetime import datetime
2324
from typing import Any
2425
from unittest import mock
2526

@@ -30,6 +31,7 @@
3031
from botocore.credentials import Credentials
3132
from fsspec.implementations.local import LocalFileSystem
3233
from fsspec.spec import AbstractFileSystem
34+
from google.oauth2.credentials import Credentials as GoogleCredentials
3335
from requests_mock import Mocker
3436

3537
from pyiceberg.catalog.rest.auth import AUTH_MANAGER
@@ -760,6 +762,29 @@ def test_adls_account_name_not_overridden_when_in_properties() -> None:
760762
)
761763

762764

765+
def test_fsspec_gcs_token_with_expiry() -> None:
766+
"""Test that a GCS token with an expiry is passed to gcsfs as google-auth credentials."""
767+
properties: Properties = {"gcs.oauth2.token": "token", "gcs.oauth2.token-expires-at": "1700000000000"}
768+
769+
with mock.patch("gcsfs.GCSFileSystem") as mock_gcsfs:
770+
FsspecFileIO(properties=properties).new_input(location="gs://bucket/path/file.parquet")
771+
772+
token = mock_gcsfs.call_args.kwargs["token"]
773+
assert isinstance(token, GoogleCredentials)
774+
assert token.token == "token"
775+
assert token.expiry == datetime(2023, 11, 14, 22, 13, 20)
776+
777+
778+
def test_fsspec_gcs_token_without_expiry() -> None:
779+
"""Test that a GCS token without an expiry is passed to gcsfs unchanged."""
780+
properties: Properties = {"gcs.oauth2.token": "token"}
781+
782+
with mock.patch("gcsfs.GCSFileSystem") as mock_gcsfs:
783+
FsspecFileIO(properties=properties).new_input(location="gs://bucket/path/file.parquet")
784+
785+
assert mock_gcsfs.call_args.kwargs["token"] == "token"
786+
787+
763788
@pytest.mark.gcs
764789
def test_fsspec_new_input_file_gcs(fsspec_fileio_gcs: FsspecFileIO) -> None:
765790
"""Test creating a new input file from a fsspec file-io"""

0 commit comments

Comments
 (0)