From 0174353d12172a03995a4420da46a5c9d5caeb83 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sun, 18 Oct 2026 11:58:22 +0900 Subject: [PATCH 1/4] State the current reasons for the Polars chunk reader factories The cross-thread close() that the split was made for is unsupported now. Describe what creating the readers during execute() does instead: the requests that prepare the reads raise their errors there, and the Parquet schema is read once. Closes #1249 Co-Authored-By: Claude Opus 5.5 --- pyathena/polars/result_set.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyathena/polars/result_set.py b/pyathena/polars/result_set.py index bd1e10505..2c066a98e 100644 --- a/pyathena/polars/result_set.py +++ b/pyathena/polars/result_set.py @@ -456,9 +456,7 @@ def _get_parquet_storage_options(connection: Connection[Any]) -> dict[str, Any]: fsspec. The credentials are frozen when the options are created. Args: - connection: The connection whose session and region the reads use. The - chunk readers give the connection that they took when they were - created, as ``close()`` releases the result set's connection. + connection: The connection whose session and region the reads use. Returns: Dictionary with AWS credentials and region for S3 access. @@ -606,9 +604,11 @@ def _create_dataframe_iterator(self) -> PolarsDataFrameIterator: For unload queries, it replaces the metadata with the schema of the Parquet files, as ``_as_polars()`` does, before the chunks are read. - The chunk readers take the state that they need, such as the output location, - the column names, and the connection, when they are created. The credentials - are frozen at the first read. + The result set creates the iterator during ``execute()``. The requests that + prepare the reads are made then, so ``execute()`` raises their errors: the size + check of a CSV result file, and the data manifest and the Parquet schema of an + UNLOAD result. The schema is read once, for the metadata and for the chunk + reader. The chunks themselves are read lazily. Returns: PolarsDataFrameIterator that reads each chunk lazily. From cf186d92c6998a137106dc8db6dd61c6c414f43b Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sun, 18 Oct 2026 12:07:41 +0900 Subject: [PATCH 2/4] Name the async future and the Polars scan in the chunk reader docstring AsyncPolarsCursor raises the preparation errors from its future, not from execute(), and the pyarrow chunk reader does not use the Parquet schema. Co-Authored-By: Claude Opus 5.5 --- pyathena/polars/result_set.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyathena/polars/result_set.py b/pyathena/polars/result_set.py index 2c066a98e..4d375dcc8 100644 --- a/pyathena/polars/result_set.py +++ b/pyathena/polars/result_set.py @@ -604,11 +604,12 @@ def _create_dataframe_iterator(self) -> PolarsDataFrameIterator: For unload queries, it replaces the metadata with the schema of the Parquet files, as ``_as_polars()`` does, before the chunks are read. - The result set creates the iterator during ``execute()``. The requests that - prepare the reads are made then, so ``execute()`` raises their errors: the size - check of a CSV result file, and the data manifest and the Parquet schema of an - UNLOAD result. The schema is read once, for the metadata and for the chunk - reader. The chunks themselves are read lazily. + The result set calls this method when it is created, in ``execute()`` or in the + future that ``AsyncPolarsCursor.execute()`` returns. The requests that prepare + the reads are made then, before the first fetch, and raise their errors there: + the size check of a CSV result file, and the data manifest and the Parquet + schema of an UNLOAD result. The schema sets the metadata, and the Polars scan + of the chunks also uses it. The chunks themselves are read lazily. Returns: PolarsDataFrameIterator that reads each chunk lazily. From 088b79e085fb7926c6c3a56936e15c935da6e93a Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sun, 18 Oct 2026 12:11:25 +0900 Subject: [PATCH 3/4] Qualify the schema use of the Polars chunk scan With a schema given to execute(), the scan reads with that schema and does not use the one read from the files. Co-Authored-By: Claude Opus 5.5 --- pyathena/polars/result_set.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyathena/polars/result_set.py b/pyathena/polars/result_set.py index 4d375dcc8..9bd3b6e83 100644 --- a/pyathena/polars/result_set.py +++ b/pyathena/polars/result_set.py @@ -609,7 +609,8 @@ def _create_dataframe_iterator(self) -> PolarsDataFrameIterator: the reads are made then, before the first fetch, and raise their errors there: the size check of a CSV result file, and the data manifest and the Parquet schema of an UNLOAD result. The schema sets the metadata, and the Polars scan - of the chunks also uses it. The chunks themselves are read lazily. + of the chunks uses it to read timestamps at microseconds unless ``execute()`` + was given a ``schema``. The chunks themselves are read lazily. Returns: PolarsDataFrameIterator that reads each chunk lazily. From df1d35bd4811193ef29565f4c1a81e06559491d0 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sun, 18 Oct 2026 12:14:18 +0900 Subject: [PATCH 4/4] Refer to _iter_parquet_chunks() for the use of the Parquet schema How the chunk scan uses the schema depends on the Polars version and on the arguments given to execute(), which _iter_parquet_chunks() describes. Co-Authored-By: Claude Opus 5.5 --- pyathena/polars/result_set.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyathena/polars/result_set.py b/pyathena/polars/result_set.py index 9bd3b6e83..3eab3059b 100644 --- a/pyathena/polars/result_set.py +++ b/pyathena/polars/result_set.py @@ -608,9 +608,8 @@ def _create_dataframe_iterator(self) -> PolarsDataFrameIterator: future that ``AsyncPolarsCursor.execute()`` returns. The requests that prepare the reads are made then, before the first fetch, and raise their errors there: the size check of a CSV result file, and the data manifest and the Parquet - schema of an UNLOAD result. The schema sets the metadata, and the Polars scan - of the chunks uses it to read timestamps at microseconds unless ``execute()`` - was given a ``schema``. The chunks themselves are read lazily. + schema of an UNLOAD result. The schema sets the metadata and is passed to + ``_iter_parquet_chunks()``. The chunks themselves are read lazily. Returns: PolarsDataFrameIterator that reads each chunk lazily.