Skip to content

Commit a13531a

Browse files
committed
Merge remote-tracking branch 'origin/main' into fix/collect-nan-value-counts
# Conflicts: # tests/catalog/test_catalog_behaviors.py
2 parents 780e128 + c4cd513 commit a13531a

60 files changed

Lines changed: 2798 additions & 402 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.codespellrc‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎.github/workflows/codeql.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,11 @@ jobs:
5151
persist-credentials: false
5252

5353
- name: Initialize CodeQL
54-
uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
54+
uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
5555
with:
5656
languages: actions
5757

5858
- name: Perform CodeQL Analysis
59-
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
59+
uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
6060
with:
6161
category: "/language:actions"

‎.github/workflows/license_check.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ jobs:
3636
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
3737
with:
3838
persist-credentials: false
39-
- uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
39+
- uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
4040
with:
4141
distribution: temurin
4242
java-version: 21

‎.github/workflows/nightly-pypi-build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ jobs:
7979

8080
steps:
8181
- name: Download all the artifacts
82-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
82+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
8383
with:
8484
merge-multiple: true
8585
path: dist/

‎.github/workflows/pypi-build-artifacts.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ jobs:
6868
if: matrix.os == 'ubuntu-24.04'
6969

7070
- name: Build wheels
71-
uses: pypa/cibuildwheel@e090b81e30c4d855ea63bf4b6e59204c09a101ae # v4.2.1
71+
uses: pypa/cibuildwheel@b9e566a2b284570af229f6c2a33673410d0389a3 # v4.3.4
7272
with:
7373
output-dir: wheelhouse
7474
config-file: "pyproject.toml"

‎.github/workflows/python-integration.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ jobs:
168168
- name: Install dependencies
169169
run: uv sync --group dev
170170
- name: Download all coverage artifacts
171-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
171+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
172172
with:
173173
pattern: coverage-*
174174
merge-multiple: true

‎.github/workflows/svn-build-artifacts.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ jobs:
6262
if: matrix.os == 'ubuntu-24.04'
6363

6464
- name: Build wheels
65-
uses: pypa/cibuildwheel@e090b81e30c4d855ea63bf4b6e59204c09a101ae # v4.2.1
65+
uses: pypa/cibuildwheel@b9e566a2b284570af229f6c2a33673410d0389a3 # v4.3.4
6666
with:
6767
output-dir: wheelhouse
6868
config-file: "pyproject.toml"

‎mkdocs/docs/configuration.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ For the FileIO there are several configuration options available:
180180
| s3.anonymous | True | Configure whether to use anonymous connection. If False (default), uses key/secret if configured or boto's credential resolver. |
181181
| s3.server-side-encryption | aws:kms | Configure server-side encryption (e.g. `AES256` or `aws:kms`). Only supported by `FsspecFileIO`. |
182182
| s3.sse-kms-key-id | alias/my-key | Configure the SSE-KMS key id (or ARN) for multipart uploads. Only supported by `FsspecFileIO`. |
183+
| s3.ssl.ca-cert | /etc/ssl/certs/ca.pem | Path to a CA certificate bundle used to verify TLS connections to the S3 endpoint. Only supported by `PyArrowFileIO`, and requires pyarrow 21.0.0 or later. |
183184

184185
<!-- markdown-link-check-enable-->
185186

@@ -233,7 +234,7 @@ For the FileIO there are several configuration options available:
233234
| gcs.requester-pays | False | Configure whether to use requester-pays requests |
234235
| gcs.session-kwargs | {} | Configure a dict of parameters to pass on to aiohttp.ClientSession; can contain, for example, proxy settings. |
235236
| gcs.service.host | <http://0.0.0.0:4443> | Configure an alternative endpoint for the GCS FileIO to access (format protocol://host:port) If not given, defaults to the value of environment variable "STORAGE_EMULATOR_HOST"; if that is not set either, will use the standard Google endpoint. |
236-
| gcs.default-location | US | Configure the default location where buckets are created, like 'US' or 'EUROPE-WEST3'. |
237+
| gcs.default-bucket-location | US | Configure the default location where buckets are created, like 'US' or 'EUROPE-WEST3'. |
237238
| gcs.version-aware | False | Configure whether to support object versioning on the GCS bucket. |
238239

239240
<!-- markdown-link-check-enable-->
@@ -448,6 +449,9 @@ as a single request timeout (floored to whole seconds). Retries are applied to i
448449
only (`GET`, `HEAD`, `OPTIONS`, `PUT`, `DELETE`) and to the transient HTTP status codes `429`, `500`,
449450
`502`, `503`, `504`. Other failures are not retried.
450451

452+
When the server advertises `idempotency-key-lifetime` in its config response, mutating requests carry an
453+
`Idempotency-Key` header and `POST` is retried as well, reusing the same key for each retry.
454+
451455
#### Headers in REST Catalog
452456

453457
To configure custom headers in REST Catalog, include them in the catalog properties with `header.<Header-Name>`. This

‎pyiceberg/avro/file.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,11 +88,10 @@ def compression_codec(self) -> type[Codec] | None:
8888
"""Get the file's compression codec algorithm from the file's metadata.
8989
9090
In the case of a null codec, we return a None indicating that we
91-
don't need to compress/decompress.
91+
don't need to compress/decompress. A file without the avro.codec
92+
entry is uncompressed: the Avro specification makes "null" the default.
9293
"""
93-
from pyiceberg.table import TableProperties
94-
95-
codec_name = self.meta.get(AVRO_CODEC_KEY, TableProperties.WRITE_AVRO_COMPRESSION_DEFAULT)
94+
codec_name = self.meta.get(AVRO_CODEC_KEY, "null")
9695
if codec_name not in KNOWN_CODECS:
9796
raise ValueError(f"Unsupported codec: {codec_name}")
9897

‎pyiceberg/avro/resolver.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,10 @@ def schema_partner(self, partner: IcebergType | None) -> IcebergType | None:
548548
raise ResolveError(f"File/read schema are not aligned for schema, got {partner}")
549549

550550
def field_partner(self, partner: IcebergType | None, field_id: int, field_name: str) -> IcebergType | None:
551+
# A file-only field has no partner; its nested fields are read and dropped
552+
if partner is None:
553+
return None
554+
551555
if isinstance(partner, StructType):
552556
field = partner.field(field_id)
553557
else:
@@ -556,18 +560,27 @@ def field_partner(self, partner: IcebergType | None, field_id: int, field_name:
556560
return field.field_type if field else None
557561

558562
def list_element_partner(self, partner_list: IcebergType | None) -> IcebergType | None:
563+
if partner_list is None:
564+
return None
565+
559566
if isinstance(partner_list, ListType):
560567
return partner_list.element_type
561568

562569
raise ResolveError(f"File/read schema are not aligned for list, got {partner_list}")
563570

564571
def map_key_partner(self, partner_map: IcebergType | None) -> IcebergType | None:
572+
if partner_map is None:
573+
return None
574+
565575
if isinstance(partner_map, MapType):
566576
return partner_map.key_type
567577

568578
raise ResolveError(f"File/read schema are not aligned for map, got {partner_map}")
569579

570580
def map_value_partner(self, partner_map: IcebergType | None) -> IcebergType | None:
581+
if partner_map is None:
582+
return None
583+
571584
if isinstance(partner_map, MapType):
572585
return partner_map.value_type
573586

0 commit comments

Comments
 (0)