From 5d0dc765fd6982fc96a07d8552a7d030f2b245c7 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sun, 20 Sep 2026 22:31:25 +0900 Subject: [PATCH 1/8] Backport #798 (async adapter cursor class only): Let the async adapter accept a cursor class by name The shared dialect now asks for the API cursor class positionally, which the async adapted connection did not accept, and it could not drive a synchronous Cursor anyway. Accept the argument and map it to the async counterpart. (cherry picked from commit c61c3611b841c26b7434adac04bdfba503f24095) 3.x resolution: by maintainer decision, only this commit of #798 is ported, as the prerequisite of #1185, whose do_ping() asks the async adapted connection for an API cursor. The rest of #798 (reflection through an API cursor and the information_schema fallback) builds on metadata work that 3.x does not have. - pyathena/aio/sqlalchemy/base.py: AsyncAdapt_pyathena_connection.cursor() takes the optional cursor class. It takes no **kwargs, because the keyword passthrough came with #777, which 3.x does not have; the cursor_kwargs and retry_config properties of the conflict hunk come from the same excluded work. - pyathena/sqlalchemy/base.py: unchanged. This commit only retyped AthenaDialect._internal_cursor(), which 3.x does not have. Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 --- pyathena/aio/sqlalchemy/base.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pyathena/aio/sqlalchemy/base.py b/pyathena/aio/sqlalchemy/base.py index 9f692e405..b1bb6f7c0 100644 --- a/pyathena/aio/sqlalchemy/base.py +++ b/pyathena/aio/sqlalchemy/base.py @@ -10,6 +10,8 @@ import pyathena from pyathena.aio.connection import AioConnection +from pyathena.aio.cursor import AioCursor +from pyathena.cursor import Cursor from pyathena.error import ( DatabaseError, DataError, @@ -29,6 +31,9 @@ from sqlalchemy import URL +_ASYNC_CURSOR_CLASSES: dict[Any, Any] = {Cursor: AioCursor} + + class AsyncAdapt_pyathena_cursor: """Wraps any async PyAthena cursor with a sync DBAPI interface. @@ -139,8 +144,10 @@ def catalog_name(self) -> str | None: def schema_name(self) -> str | None: return self._connection.schema_name # type: ignore[no-any-return] - def cursor(self) -> AsyncAdapt_pyathena_cursor: - raw_cursor = self._connection.cursor() + def cursor(self, cursor: Any = None) -> AsyncAdapt_pyathena_cursor: + # The shared dialect names a cursor class in its synchronous form; this + # connection can only drive the async counterpart. + raw_cursor = self._connection.cursor(_ASYNC_CURSOR_CLASSES.get(cursor, cursor)) return AsyncAdapt_pyathena_cursor(raw_cursor) def close(self) -> None: From 92ca99f808e30786dd80df26fe0bab3f199a9787 Mon Sep 17 00:00:00 2001 From: Greg Campion Date: Wed, 14 Oct 2026 14:17:24 -0700 Subject: [PATCH 2/8] Backport #1185: Run the SQLAlchemy ping with a plain cursor This fixes the issue of running a ping when unload=true is on which results in a MISSING_COLUMN_NAME error. Closes #1189 (cherry picked from commit 119cc70e2dceb868327682c7b87187a806fc488b) 3.x resolution: 3.x has neither Connection._internal_cursor() (#1102, a 4.0.0 breaking change) nor pyathena.util.override, so: - pyathena/sqlalchemy/base.py: do_ping() opens the API cursor with dbapi_connection.cursor(Cursor), which the async adapted connection maps to AioCursor since the #798 commit ported before this one. 3.x Connection.cursor() passes every cursor_kwargs entry to the cursor, and Cursor ignores DataFrame-cursor options such as unload and chunksize, so the ping runs SELECT 1 without UNLOAD; it fetches no rows, so the connection-level converter does not matter. The method has no @override decorator, and do_ping() sits after _get_table(), since 3.x has no _internal_cursor() to follow. Cursor is imported, as #798 did on master. - tests/pyathena/sqlalchemy/test_base.py: the unit test uses the 3.x adapter names AsyncAdapt_pyathena_connection and AsyncAdapt_pyathena_cursor and imports MagicMock and the connection and cursor classes it needs; the other master tests in the conflict hunk are not part of this change. Co-Authored-By: Claude Opus 5.5 --- pyathena/sqlalchemy/base.py | 10 +++ tests/pyathena/aio/sqlalchemy/test_base.py | 11 ++++ tests/pyathena/sqlalchemy/test_base.py | 71 ++++++++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/pyathena/sqlalchemy/base.py b/pyathena/sqlalchemy/base.py index 5ceb02336..1568e68b4 100644 --- a/pyathena/sqlalchemy/base.py +++ b/pyathena/sqlalchemy/base.py @@ -23,6 +23,7 @@ ) import pyathena +from pyathena.cursor import Cursor from pyathena.sqlalchemy.compiler import ( AthenaDDLCompiler, AthenaStatementCompiler, @@ -52,6 +53,7 @@ PoolProxiedConnection, ) from sqlalchemy.engine.interfaces import ( + DBAPIConnection, ReflectedForeignKeyConstraint, ReflectedIndex, ReflectedPrimaryKeyConstraint, @@ -284,6 +286,14 @@ def _get_table(self, connection, table_name: str, schema: str | None = None, **k raise exc.NoSuchTableError(table_name) from e raise + def do_ping(self, dbapi_connection: DBAPIConnection) -> bool: + # When unload is enabled in the default cursor, SELECT 1 is wrapped in UNLOAD + # which Athena rejects with MISSING_COLUMN_NAME because the column is unnamed. + # Using the plain cursor here performs the SELECT without the UNLOAD. + with dbapi_connection.cursor(Cursor) as cursor: # type: ignore[attr-defined] + cursor.execute(self._dialect_specific_select_one) + return True + @reflection.cache def _get_tables(self, connection, schema: str | None = None, **kw): raw_connection = self._raw_connection(connection) diff --git a/tests/pyathena/aio/sqlalchemy/test_base.py b/tests/pyathena/aio/sqlalchemy/test_base.py index 8f35cdea1..95f3e7441 100644 --- a/tests/pyathena/aio/sqlalchemy/test_base.py +++ b/tests/pyathena/aio/sqlalchemy/test_base.py @@ -7,6 +7,17 @@ class TestAsyncSQLAlchemyAthena: + @pytest.mark.parametrize( + "async_engine", + [{"driver": "aiopandas", "unload": True}], + indirect=True, + ) + async def test_ping_with_unload(self, async_engine): + _, conn = async_engine + assert await conn.run_sync( + lambda sync_conn: sync_conn.dialect.do_ping(sync_conn.connection.dbapi_connection) + ) + @pytest.mark.parametrize( "async_engine", [ diff --git a/tests/pyathena/sqlalchemy/test_base.py b/tests/pyathena/sqlalchemy/test_base.py index 8aec1fa4c..bc16e2244 100644 --- a/tests/pyathena/sqlalchemy/test_base.py +++ b/tests/pyathena/sqlalchemy/test_base.py @@ -6,6 +6,7 @@ from datetime import date, datetime from decimal import Decimal from pathlib import Path +from unittest.mock import MagicMock from urllib.parse import quote_plus import numpy as np @@ -20,6 +21,17 @@ from sqlalchemy.sql.selectable import TextualSelect from sqlalchemy.util import PluginLoader +from pyathena.aio.connection import AioConnection +from pyathena.aio.cursor import AioCursor +from pyathena.aio.pandas.cursor import AioPandasCursor +from pyathena.aio.sqlalchemy.base import ( + AsyncAdapt_pyathena_connection, + AsyncAdapt_pyathena_cursor, + AthenaAioDialect, +) +from pyathena.connection import Connection +from pyathena.cursor import Cursor +from pyathena.pandas.cursor import PandasCursor from pyathena.sqlalchemy.base import AthenaDialect from pyathena.sqlalchemy.rest import AthenaRestDialect from pyathena.sqlalchemy.types import ( @@ -82,6 +94,48 @@ def test_compliance_suite_registry_matches_entry_points(self, monkeypatch): assert entry_points assert {name: load() for name, load in loader.impls.items()} == entry_points + @pytest.mark.parametrize( + ("dialect_class", "connection_class", "cursor_class", "expected"), + [ + (AthenaDialect, Connection, PandasCursor, Cursor), + (AthenaAioDialect, AioConnection, AioPandasCursor, AioCursor), + ], + ) + def test_do_ping_uses_api_cursor_without_unload( + self, monkeypatch, dialect_class, connection_class, cursor_class, expected + ): + conn = connection_class( + region_name="us-west-2", + s3_staging_dir="s3://bucket/path/", + aws_access_key_id="access_key", + aws_secret_access_key="secret_key", + cursor_class=cursor_class, + cursor_kwargs={"unload": True}, + ) + executed = [] + closed = [] + monkeypatch.setattr(expected, "close", lambda self: closed.append(self)) + if connection_class is AioConnection: + dbapi_connection = AsyncAdapt_pyathena_connection(MagicMock(), conn) + monkeypatch.setattr( + AsyncAdapt_pyathena_cursor, + "execute", + lambda self, operation: executed.append((self._cursor, operation)), + ) + else: + dbapi_connection = conn + monkeypatch.setattr( + Cursor, "execute", lambda self, operation: executed.append((self, operation)) + ) + try: + assert dialect_class().do_ping(dbapi_connection) is True + [(cursor, operation)] = executed + assert type(cursor) is expected + assert operation == "SELECT 1" + assert closed == [cursor] + finally: + conn.close() + @pytest.mark.parametrize( ("map_type", "struct_type"), [ @@ -161,6 +215,23 @@ def test_native_array_results_across_cursors(self, engine): finally: array_engine.dispose() + @pytest.mark.parametrize( + "engine", + [{"driver": "pandas", "unload": True}], + indirect=True, + ) + def test_ping_with_unload(self, engine): + engine, conn = engine + assert engine.dialect.do_ping(conn.connection.dbapi_connection) is True + ping_engine = sqlalchemy.create_engine(engine.url, pool_pre_ping=True) + try: + # The second checkout of the pooled connection pings it. + for _ in range(2): + with ping_engine.connect() as ping_conn: + assert ping_conn.execute(text("SELECT 1 AS one")).scalar() == 1 + finally: + ping_engine.dispose() + @pytest.mark.parametrize( "engine", [ From 053120cd03b645ece549a2916a45fd627f5bb1db Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Wed, 14 Oct 2026 21:56:44 +0900 Subject: [PATCH 3/8] Backport #1169: Read CSV results through fsspec for Polars 2.0 Polars 2.0 reads a path in read_csv() with its native object store and rejects the fsspec storage options that PyAthena passed, so every CSV result read without chunksize failed. The result set now opens the file with fsspec.open() and the same options, and passes its contents to Polars, which keeps PyAthena's S3 filesystem and its read settings with Polars 1.x and 2.0. Polars 2.0 also names headerless columns from column_0 and requires new_columns to name every column, so the tests that pass those options through no longer depend on the Polars version. The dev lock tests Polars 2 on Python 3.14, which pull requests run, and keeps the newest Polars 1.x release on older versions. Closes #1168 (cherry picked from commit 0c6a0b2de52fe1c3d8d03b79f01b2b3bf52c1ab9) 3.x resolution: 3.x does not have the master changes that this commit builds on (execute()-level storage_options replacement, the timestamp re-read and the GetQueryResults CSV path of _read_csv(), and the TestAthenaPolarsResultSet unit tests), so only this commit's own delta is applied: - pyathena/polars/result_set.py: _read_csv() opens output_location with fsspec.open(..., **self._csv_storage_options) and passes the bytes to pl.read_csv() with 3.x's existing arguments, instead of passing the path and storage_options to Polars. - pyproject.toml: the dev group keeps 3.x's "polars>=1.0.0" floor and adds the Polars 2 marker pair for Python 3.14 and older versions. - uv.lock: relocked; only polars and polars-runtime-32 change. Python 3.14 resolves Polars 2.0.0, and the older versions keep 3.x's locked 1.36.1 rather than master's newest 1.x release. - tests/pyathena/polars/test_cursor.py, test_result_set.py: unchanged. The tests this commit adjusts (new_columns with schema_overrides, headerless column names, and the storage_options replacement cases) do not exist on 3.x; the existing PolarsCursor integration tests cover _read_csv() with both Polars majors. Co-Authored-By: Claude Opus 5.5 --- pyathena/polars/result_set.py | 15 ++- pyproject.toml | 4 + uv.lock | 176 ++++++++++++++++++++++------------ 3 files changed, 127 insertions(+), 68 deletions(-) diff --git a/pyathena/polars/result_set.py b/pyathena/polars/result_set.py index 39c3563d2..608f58c79 100644 --- a/pyathena/polars/result_set.py +++ b/pyathena/polars/result_set.py @@ -10,6 +10,8 @@ cast, ) +import fsspec + from pyathena import OperationalError from pyathena.converter import Converter from pyathena.error import ProgrammingError @@ -268,10 +270,10 @@ def __init__( @property def _csv_storage_options(self) -> dict[str, Any]: - """Get storage options for Polars CSV reading via fsspec. + """Get storage options for reading a CSV result file via fsspec. - Polars read_csv uses fsspec for cloud storage access, which works - with PyAthena's registered S3FileSystem. + ``_read_csv()`` opens the file with ``fsspec.open()``, which uses + PyAthena's registered S3FileSystem, and passes its contents to Polars. Returns: Dictionary with fsspec-compatible options for S3 access. @@ -463,12 +465,15 @@ def _read_csv(self) -> pl.DataFrame: separator, has_header, new_columns = self._get_csv_params() try: + # Polars 2.0 reads a path with its native object store, which does not take + # fsspec options, so the result file is read through fsspec here. + with fsspec.open(self.output_location, "rb", **self._csv_storage_options) as file: + source = file.read() df = pl.read_csv( - self.output_location, + source, separator=separator, has_header=has_header, schema_overrides=self.dtypes, - storage_options=self._csv_storage_options, **self._kwargs, ) if new_columns: diff --git a/pyproject.toml b/pyproject.toml index acb5e53b3..f886db18c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,10 @@ dev = [ "pyarrow>=10.0.0; python_version<'3.14'", "pyarrow>=22.0.0; python_version>='3.14'", "polars>=1.0.0", + # Pull requests test the newest Python with Polars 2, and the older Pythons keep + # the newest Polars 1.x release. + "polars>=2.0.0; python_version >= '3.14'", + "polars<2.0.0; python_version < '3.14'", "Jinja2>=3.1.0", "mypy>=0.900", "pytest>=3.5", diff --git a/uv.lock b/uv.lock index c8ba38276..32512e81a 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14'", @@ -401,7 +401,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "mdurl", marker = "python_full_version < '3.11'" }, + { name = "mdurl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } wheels = [ @@ -419,7 +419,7 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "mdurl", marker = "python_full_version >= '3.11'" }, + { name = "mdurl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } wheels = [ @@ -557,12 +557,12 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "docutils", marker = "python_full_version < '3.11'" }, - { name = "jinja2", marker = "python_full_version < '3.11'" }, - { name = "markdown-it-py", version = "3.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "mdit-py-plugins", marker = "python_full_version < '3.11'" }, - { name = "pyyaml", marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "docutils" }, + { name = "jinja2" }, + { name = "markdown-it-py", version = "3.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "mdit-py-plugins" }, + { name = "pyyaml" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/a5/9626ba4f73555b3735ad86247a8077d4603aa8628537687c839ab08bfe44/myst_parser-4.0.1.tar.gz", hash = "sha256:5cfea715e4f3574138aecbf7d54132296bfd72bb614d31168f48c477a830a7c4", size = 93985, upload-time = "2025-02-12T10:53:03.833Z" } wheels = [ @@ -580,12 +580,12 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "docutils", marker = "python_full_version >= '3.11'" }, - { name = "jinja2", marker = "python_full_version >= '3.11'" }, - { name = "markdown-it-py", version = "4.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "mdit-py-plugins", marker = "python_full_version >= '3.11'" }, - { name = "pyyaml", marker = "python_full_version >= '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "docutils" }, + { name = "jinja2" }, + { name = "markdown-it-py", version = "4.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "mdit-py-plugins" }, + { name = "pyyaml" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/fa/7b45eef11b7971f0beb29d27b7bfe0d747d063aa29e170d9edd004733c8a/myst_parser-5.0.0.tar.gz", hash = "sha256:f6f231452c56e8baa662cc352c548158f6a16fcbd6e3800fc594978002b94f3a", size = 98535, upload-time = "2026-01-15T09:08:18.036Z" } wheels = [ @@ -739,10 +739,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "python-dateutil", marker = "python_full_version < '3.13'" }, - { name = "pytz", marker = "python_full_version < '3.13'" }, - { name = "tzdata", marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" } wheels = [ @@ -791,10 +791,10 @@ resolution-markers = [ "python_full_version == '3.13.*'", ] dependencies = [ - { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.13'" }, - { name = "pytz", marker = "python_full_version >= '3.13'" }, - { name = "tzdata", marker = "python_full_version >= '3.13'" }, + { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -860,18 +860,45 @@ wheels = [ name = "polars" version = "1.36.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] dependencies = [ - { name = "polars-runtime-32" }, + { name = "polars-runtime-32", version = "1.36.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/9f/dc/56f2a90c79a2cb13f9e956eab6385effe54216ae7a2068b3a6406bae4345/polars-1.36.1.tar.gz", hash = "sha256:12c7616a2305559144711ab73eaa18814f7aa898c522e7645014b68f1432d54c", size = 711993, upload-time = "2025-12-10T01:14:53.033Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f6/c6/36a1b874036b49893ecae0ac44a2f63d1a76e6212631a5b2f50a86e0e8af/polars-1.36.1-py3-none-any.whl", hash = "sha256:853c1bbb237add6a5f6d133c15094a9b727d66dd6a4eb91dbb07cdb056b2b8ef", size = 802429, upload-time = "2025-12-10T01:13:53.838Z" }, ] +[[package]] +name = "polars" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14'", +] +dependencies = [ + { name = "polars-runtime-32", version = "2.0.0", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/e9/001f371ec6a1bb54893f599ceebd56e6144fed4091f09f09fec0021a9276/polars-2.0.0.tar.gz", hash = "sha256:62da109e27a19a9d36657ee25dc035c9d3f87e7bd610526fe467dc37ea7dc115", size = 778215, upload-time = "2026-10-06T11:51:29.679Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/09/cc33bbd5463749c116b62c204d88bed6c02a6cb901eac7adab0d38651b07/polars-2.0.0-py3-none-any.whl", hash = "sha256:35d62f3541b7a6d4c360a2e2f07fccc0c2bcbd33b0ea51c83a25417a47a3f3ad", size = 876611, upload-time = "2026-10-06T11:44:04.327Z" }, +] + [[package]] name = "polars-runtime-32" version = "1.36.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] sdist = { url = "https://files.pythonhosted.org/packages/31/df/597c0ef5eb8d761a16d72327846599b57c5d40d7f9e74306fc154aba8c37/polars_runtime_32-1.36.1.tar.gz", hash = "sha256:201c2cfd80ceb5d5cd7b63085b5fd08d6ae6554f922bcb941035e39638528a09", size = 2788751, upload-time = "2025-12-10T01:14:54.172Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e1/ea/871129a2d296966c0925b078a9a93c6c5e7facb1c5eebfcd3d5811aeddc1/polars_runtime_32-1.36.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:327b621ca82594f277751f7e23d4b939ebd1be18d54b4cdf7a2f8406cecc18b2", size = 43494311, upload-time = "2025-12-10T01:13:56.096Z" }, @@ -882,6 +909,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d8/a12e6aa14f63784cead437083319ec7cece0d5bb9a5bfe7678cc6578b52a/polars_runtime_32-1.36.1-cp39-abi3-win_arm64.whl", hash = "sha256:809e73857be71250141225ddd5d2b30c97e6340aeaa0d445f930e01bef6888dc", size = 39798896, upload-time = "2025-12-10T01:14:11.568Z" }, ] +[[package]] +name = "polars-runtime-32" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14'", +] +sdist = { url = "https://files.pythonhosted.org/packages/34/ad/dbb6f6d7070867951532bcfe5e6a648d8777b416b18cddabc07030404e8c/polars_runtime_32-2.0.0.tar.gz", hash = "sha256:b5f9afcc742b4a67eabd2c680ff0f12eb02ede9b4bf807bffabd6dbb9a58d5c7", size = 3591339, upload-time = "2026-10-06T11:51:31.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/88/d35dec6c8928dfbaa1cccf9b626a1067da906e792c92d9f994ca825ab2b5/polars_runtime_32-2.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ffb7ac6cf4e8c4a652df1951e3c3840c7c23a033603d5a9efd422fa8dd699d82", size = 52494314, upload-time = "2026-10-06T11:44:07.768Z" }, + { url = "https://files.pythonhosted.org/packages/5f/fd/2237bf53ffaff47cdf1edc6c10587a7a6444d4951150eeb08d84f3493ff8/polars_runtime_32-2.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7012d8a0201bd95638545ce8f256c0efe2c5cab0f806eb043021dddde5a9498b", size = 47930083, upload-time = "2026-10-06T11:44:11.592Z" }, + { url = "https://files.pythonhosted.org/packages/0d/0d/85e3ed90417996fc09770be91b39979074fe2978fc15b431bf8a9459760d/polars_runtime_32-2.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b85bb42e6009acc9629afcc70a83473fd468694d6a30ffb0ab376c8dd1a0a17", size = 50417889, upload-time = "2026-10-06T11:50:20.774Z" }, + { url = "https://files.pythonhosted.org/packages/83/88/e9fecfd49159da92f54ff2445883577a0f1bc195da53ecc9535c458d55dd/polars_runtime_32-2.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d6ac584ea2b38913784db943879412380d92e28ab9cb88e20a77ba71ba3f911", size = 54475036, upload-time = "2026-10-06T11:50:24.411Z" }, + { url = "https://files.pythonhosted.org/packages/48/ad/b2abf732697b21467aaaeaac0f3bf7eee0d89c59ce8125f1ed41b28a2d97/polars_runtime_32-2.0.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a6bf5e260e0a6f00d0f9181438fe9e45776df8c66cee9cba16e3675cc3888488", size = 50579474, upload-time = "2026-10-06T11:50:28.377Z" }, + { url = "https://files.pythonhosted.org/packages/7f/05/304deee59a95865e1b5e9ec7b066069b49093b81b768f473d9d3b165c686/polars_runtime_32-2.0.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55c26eef325b6840584d91aac232e9cf3ac19e1b904594b9b54131be1edeab4d", size = 54413293, upload-time = "2026-10-06T11:50:31.828Z" }, + { url = "https://files.pythonhosted.org/packages/61/59/8c9fd7199f7c4eb1b64e640306a946a2e4a46337b3bbb33b840972c7d84b/polars_runtime_32-2.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:7da1caf3c7b4f397fb213c984013a0c755557619a2d511899a1ff74392484078", size = 54229989, upload-time = "2026-10-06T11:50:35.206Z" }, + { url = "https://files.pythonhosted.org/packages/e2/93/43608026f38aa6ed4d22da8597706a61682ee403caef0021ce8e6dc73227/polars_runtime_32-2.0.0-cp310-abi3-win_arm64.whl", hash = "sha256:c30ba698c8904048df4a9bc3d6c5033cc2d0a7cbb0e13f4fd2de5a1947b61994", size = 48730655, upload-time = "2026-10-06T11:50:38.756Z" }, +] + [[package]] name = "pyarrow" version = "23.0.1" @@ -962,7 +1008,8 @@ pandas = [ { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, ] polars = [ - { name = "polars" }, + { name = "polars", version = "1.36.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, + { name = "polars", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, ] sqlalchemy = [ { name = "sqlalchemy" }, @@ -979,7 +1026,8 @@ dev = [ { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, { name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, - { name = "polars" }, + { name = "polars", version = "1.36.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, + { name = "polars", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, { name = "pyarrow" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -1025,6 +1073,8 @@ dev = [ { name = "pandas", marker = "python_full_version < '3.13'", specifier = ">=1.3.0" }, { name = "pandas", marker = "python_full_version >= '3.13'", specifier = ">=2.3.0" }, { name = "polars", specifier = ">=1.0.0" }, + { name = "polars", marker = "python_full_version < '3.14'", specifier = "<2.0.0" }, + { name = "polars", marker = "python_full_version >= '3.14'", specifier = ">=2.0.0" }, { name = "pyarrow", marker = "python_full_version < '3.14'", specifier = ">=10.0.0" }, { name = "pyarrow", marker = "python_full_version >= '3.14'", specifier = ">=22.0.0" }, { name = "pytest", specifier = ">=3.5" }, @@ -1245,7 +1295,7 @@ name = "roman-numerals-py" version = "4.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "roman-numerals", marker = "python_full_version >= '3.11'" }, + { name = "roman-numerals" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", hash = "sha256:f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9", size = 4274, upload-time = "2025-12-17T18:25:41.153Z" } wheels = [ @@ -1308,23 +1358,23 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "alabaster", marker = "python_full_version < '3.11'" }, - { name = "babel", marker = "python_full_version < '3.11'" }, - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version < '3.11'" }, - { name = "imagesize", marker = "python_full_version < '3.11'" }, - { name = "jinja2", marker = "python_full_version < '3.11'" }, - { name = "packaging", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "requests", marker = "python_full_version < '3.11'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.11'" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/be/50e50cb4f2eff47df05673d361095cafd95521d2a22521b920c67a372dcb/sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe", size = 8067911, upload-time = "2024-07-20T14:46:56.059Z" } wheels = [ @@ -1342,23 +1392,23 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "alabaster", marker = "python_full_version >= '3.11'" }, - { name = "babel", marker = "python_full_version >= '3.11'" }, - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version >= '3.11'" }, - { name = "imagesize", marker = "python_full_version >= '3.11'" }, - { name = "jinja2", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "requests", marker = "python_full_version >= '3.11'" }, - { name = "roman-numerals-py", marker = "python_full_version >= '3.11'" }, - { name = "snowballstemmer", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.11'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals-py" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/ad/4360e50ed56cb483667b8e6dadf2d3fda62359593faabbe749a27c4eaca6/sphinx-8.2.3.tar.gz", hash = "sha256:398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348", size = 8321876, upload-time = "2025-03-02T22:31:59.658Z" } wheels = [ @@ -1386,7 +1436,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/2b/69/b34e0cb5336f09c6866d53b4a19d76c227cdec1bbc7ac4de63ca7d58c9c7/sphinx_design-0.6.1.tar.gz", hash = "sha256:b44eea3719386d04d765c1a8257caca2b3e6f8421d7b3a5e742c0fd45f84e632", size = 2193689, upload-time = "2024-08-02T13:48:44.277Z" } wheels = [ @@ -1404,7 +1454,7 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/13/7b/804f311da4663a4aecc6cf7abd83443f3d4ded970826d0c958edc77d4527/sphinx_design-0.7.0.tar.gz", hash = "sha256:d2a3f5b19c24b916adb52f97c5f00efab4009ca337812001109084a740ec9b7a", size = 2203582, upload-time = "2026-01-19T13:12:53.297Z" } wheels = [ From 60ee3baec8af469d45308700e89f932c0146d23a Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Fri, 16 Oct 2026 08:46:29 +0900 Subject: [PATCH 4/8] Backport #1169: Describe Polars map columns of UNLOAD results as map Polars 2.0 reads a Parquet map as a Map column, which get_athena_type() did not know, so UNLOAD results described map columns as string. Nested types are now matched by their outer type, which also describes a struct containing a list as row instead of array. The docs now say that Polars reads with fsspec when use_pyarrow=True is given. (cherry picked from commit a473e9cd816c3b58d2d782080b49d90b2af3458c) 3.x resolution: - docs/polars.md: unchanged. The sentence this commit edits belongs to the "S3 read options" section, which 3.x does not have; the preceding #1169 commit 6ec3e008, which added that sentence and reworded two test comments of tests 3.x does not have, was skipped as empty. - tests/pyathena/polars/test_cursor.py: test_nested_type_description_unload runs without chunksize only. On 3.x, a chunked UNLOAD result describes its columns as the UNLOAD statement's own "rows" column, because the Parquet schema is not read on that path; master fixed that in #1012, which 3.x does not have. - pyathena/polars/util.py and tests/pyathena/polars/test_util.py apply unchanged. Co-Authored-By: Claude Opus 5.5 --- pyathena/polars/util.py | 14 +++++++++---- tests/pyathena/polars/test_cursor.py | 22 ++++++++++++++++++++ tests/pyathena/polars/test_util.py | 31 ++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 tests/pyathena/polars/test_util.py diff --git a/pyathena/polars/util.py b/pyathena/polars/util.py index 91edd06d3..a046bdc32 100644 --- a/pyathena/polars/util.py +++ b/pyathena/polars/util.py @@ -97,12 +97,18 @@ def get_athena_type(dtype: Any) -> tuple[str, int, int]: if base_dtype == polars_type or dtype == polars_type: return athena_info - # Handle parameterized types that didn't match above - dtype_str = str(dtype).lower() - if "list" in dtype_str: + # Nested types are matched by their outer type, not by the types they contain. + if base_dtype == pl.List: return ("array", 0, 0) - if "struct" in dtype_str: + if base_dtype == pl.Struct: return ("row", 0, 0) + # Polars 2.0 reads a Parquet map as Map; Polars 1.x reads it as a list of structs. + map_dtype = getattr(pl, "Map", None) + if map_dtype is not None and base_dtype == map_dtype: + return ("map", 0, 0) + + # Handle parameterized types that didn't match above + dtype_str = str(dtype).lower() if "decimal" in dtype_str: # Extract precision and scale from Decimal type if available if hasattr(dtype, "precision") and hasattr(dtype, "scale"): diff --git a/tests/pyathena/polars/test_cursor.py b/tests/pyathena/polars/test_cursor.py index 145a995dd..fb8ebaae4 100644 --- a/tests/pyathena/polars/test_cursor.py +++ b/tests/pyathena/polars/test_cursor.py @@ -635,6 +635,28 @@ def test_iterator_with_chunksize_unload(self, polars_cursor): rows = list(polars_cursor) assert len(rows) == 15 + @pytest.mark.parametrize( + "polars_cursor", + [ + {"cursor_kwargs": {"unload": True}}, + ], + indirect=["polars_cursor"], + ) + def test_nested_type_description_unload(self, polars_cursor): + """UNLOAD results describe nested Parquet columns by their outer type.""" + polars_cursor.execute( + """ + SELECT + ARRAY[1, 2] AS a + ,CAST(ROW(ARRAY[1]) AS ROW(x ARRAY(INTEGER))) AS r + ,MAP(ARRAY['k'], ARRAY[1]) AS m + """ + ) + # Polars 1.x reads a Parquet map as a list of key-value structs. + map_type = "map" if hasattr(pl, "Map") else "array" + description = [d[:2] for d in polars_cursor.description] + assert description == [("a", "array"), ("r", "row"), ("m", map_type)] + @pytest.mark.parametrize( "polars_cursor", [ diff --git a/tests/pyathena/polars/test_util.py b/tests/pyathena/polars/test_util.py new file mode 100644 index 000000000..800387ccc --- /dev/null +++ b/tests/pyathena/polars/test_util.py @@ -0,0 +1,31 @@ +# Copyright 2026 The PyAthena authors +# +# Licensed under the MIT License. +# See LICENSE or https://opensource.org/licenses/MIT. +# +# SPDX-License-Identifier: MIT + +import polars as pl +import pytest + +from pyathena.polars.util import get_athena_type + + +@pytest.mark.parametrize( + ("dtype", "expected"), + [ + (pl.List(pl.Int32), ("array", 0, 0)), + (pl.List(pl.Struct({"x": pl.Int32})), ("array", 0, 0)), + (pl.Struct({"x": pl.Int32}), ("row", 0, 0)), + (pl.Struct({"x": pl.List(pl.Int32)}), ("row", 0, 0)), + ], +) +def test_get_athena_type_nested(dtype, expected): + assert get_athena_type(dtype) == expected + + +@pytest.mark.skipif(not hasattr(pl, "Map"), reason="Polars 1.x has no Map type") +def test_get_athena_type_map(): + assert get_athena_type(pl.Map(pl.String, pl.Int32)) == ("map", 0, 0) + assert get_athena_type(pl.Map(pl.String, pl.List(pl.Int32))) == ("map", 0, 0) + assert get_athena_type(pl.List(pl.Map(pl.String, pl.Int32))) == ("array", 0, 0) From a2e6899107115314058183d3f388025eb7932b7c Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Wed, 14 Oct 2026 22:31:37 +0900 Subject: [PATCH 5/8] Backport #1169: Expect Polars 2.0's map conversion in ArrowCursor.as_polars() Polars 2.0 converts an Arrow map to a Map column whose values are dicts; Polars 1.x converts it to a list of key-value structs. (cherry picked from commit 22de45412887431ca90c180b46b243abc5ef39c7) Co-Authored-By: Claude Opus 5.5 --- tests/pyathena/arrow/test_cursor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/pyathena/arrow/test_cursor.py b/tests/pyathena/arrow/test_cursor.py index 7f8fed60c..1a5e54fc0 100644 --- a/tests/pyathena/arrow/test_cursor.py +++ b/tests/pyathena/arrow/test_cursor.py @@ -609,6 +609,13 @@ def test_complex_unload_as_polars(self, arrow_cursor): ).as_polars() assert df.height == 1 assert df.width == 16 + # Polars 2.0 converts an Arrow map to Map; Polars 1.x to a list of key-value structs. + if hasattr(pl, "Map"): + map_dtype = pl.Map(pl.Int32, pl.Int32) + map_value = {1: 2, 3: 4} + else: + map_dtype = pl.List(pl.Struct([pl.Field("key", pl.Int32), pl.Field("value", pl.Int32)])) + map_value = [{"key": 1, "value": 2}, {"key": 3, "value": 4}] dtypes = tuple(df.dtypes) assert dtypes == ( pl.Boolean, @@ -624,7 +631,7 @@ def test_complex_unload_as_polars(self, arrow_cursor): pl.Date, pl.Binary, pl.List(pl.Int32), - pl.List(pl.Struct([pl.Field("key", pl.Int32), pl.Field("value", pl.Int32)])), + map_dtype, pl.Struct([pl.Field("a", pl.Int32), pl.Field("b", pl.Int32)]), pl.Decimal(precision=10, scale=1), ) @@ -644,7 +651,7 @@ def test_complex_unload_as_polars(self, arrow_cursor): "col_date": datetime(2017, 1, 2).date(), "col_binary": b"123", "col_array": [1, 2], - "col_map": [{"key": 1, "value": 2}, {"key": 3, "value": 4}], + "col_map": map_value, "col_struct": {"a": 1, "b": 2}, "col_decimal": Decimal("0.1"), } From cb349989da628ccf9aa9fb898b06d4ccf6af6980 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Fri, 16 Oct 2026 11:21:31 +0900 Subject: [PATCH 6/8] Backport #1200: Let DefaultDialect.do_ping() run the ping with the API cursor Since #1185, AthenaDialect.do_ping() ran SELECT 1 itself. Before SQLAlchemy 2.0.5, DefaultDialect.do_ping() is also where a failed ping is checked for a disconnect (is_disconnect() and the handle_error listeners), and it returns False so that the pool reconnects; the override made such a checkout raise instead. The override now passes DefaultDialect.do_ping() a connection whose cursor() opens the API cursor, so the ping keeps #1185's cursor and each SQLAlchemy version keeps its own error handling. Closes #1199 (cherry picked from commit 3aae3c72f924082312a3bbd7eb93a0bcfda74f0e) 3.x resolution, as in the #1185 backport: - pyathena/sqlalchemy/base.py: the namespace's cursor() opens the API cursor with dbapi_connection.cursor(Cursor), since 3.x has no Connection._internal_cursor() (#1102). - tests/pyathena/sqlalchemy/test_base.py: the test uses the 3.x adapter name AsyncAdapt_pyathena_connection, and only DefaultDialect is added to 3.x's imports; the other master tests and imports in the conflict hunks are not part of this change. Co-Authored-By: Claude Opus 5.5 --- pyathena/sqlalchemy/base.py | 10 ++++-- tests/pyathena/sqlalchemy/test_base.py | 45 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/pyathena/sqlalchemy/base.py b/pyathena/sqlalchemy/base.py index 1568e68b4..81473aadb 100644 --- a/pyathena/sqlalchemy/base.py +++ b/pyathena/sqlalchemy/base.py @@ -4,6 +4,7 @@ import re from collections.abc import Mapping, MutableMapping from re import Pattern +from types import SimpleNamespace from typing import ( TYPE_CHECKING, Any, @@ -290,9 +291,12 @@ def do_ping(self, dbapi_connection: DBAPIConnection) -> bool: # When unload is enabled in the default cursor, SELECT 1 is wrapped in UNLOAD # which Athena rejects with MISSING_COLUMN_NAME because the column is unnamed. # Using the plain cursor here performs the SELECT without the UNLOAD. - with dbapi_connection.cursor(Cursor) as cursor: # type: ignore[attr-defined] - cursor.execute(self._dialect_specific_select_one) - return True + # The ping still runs in DefaultDialect.do_ping(), which before SQLAlchemy 2.0.5 + # is also where a failed ping is checked for a disconnect. + api_connection = SimpleNamespace( + cursor=lambda: dbapi_connection.cursor(Cursor) # type: ignore[call-arg] + ) + return super().do_ping(cast("DBAPIConnection", api_connection)) @reflection.cache def _get_tables(self, connection, schema: str | None = None, **kw): diff --git a/tests/pyathena/sqlalchemy/test_base.py b/tests/pyathena/sqlalchemy/test_base.py index bc16e2244..90de4a34e 100644 --- a/tests/pyathena/sqlalchemy/test_base.py +++ b/tests/pyathena/sqlalchemy/test_base.py @@ -14,6 +14,7 @@ import pytest import sqlalchemy from sqlalchemy import create_engine, func, literal_column, select, text, types +from sqlalchemy.engine.default import DefaultDialect from sqlalchemy.exc import NoSuchTableError from sqlalchemy.sql import expression, type_coerce from sqlalchemy.sql.ddl import CreateTable @@ -136,6 +137,50 @@ def test_do_ping_uses_api_cursor_without_unload( finally: conn.close() + @pytest.mark.parametrize( + ("dialect_class", "connection_class", "cursor_class", "expected"), + [ + (AthenaDialect, Connection, PandasCursor, Cursor), + (AthenaAioDialect, AioConnection, AioPandasCursor, AioCursor), + ], + ) + def test_do_ping_keeps_default_dialect_ping( + self, monkeypatch, dialect_class, connection_class, cursor_class, expected + ): + """DefaultDialect.do_ping() runs the ping with the API cursor and decides its result. + + Before SQLAlchemy 2.0.5, it returns False when a failed ping is a disconnect, so + that the pool reconnects; the stand-in here returns False the same way. + """ + conn = connection_class( + region_name="us-west-2", + s3_staging_dir="s3://bucket/path/", + aws_access_key_id="access_key", + aws_secret_access_key="secret_key", + cursor_class=cursor_class, + cursor_kwargs={"unload": True}, + ) + cursors = [] + + def do_ping(self, dbapi_connection): + cursors.append(dbapi_connection.cursor()) + return False + + monkeypatch.setattr(DefaultDialect, "do_ping", do_ping) + dbapi_connection = ( + AsyncAdapt_pyathena_connection(MagicMock(), conn) + if connection_class is AioConnection + else conn + ) + try: + assert dialect_class().do_ping(dbapi_connection) is False + [cursor] = cursors + if connection_class is AioConnection: + cursor = cursor._cursor + assert type(cursor) is expected + finally: + conn.close() + @pytest.mark.parametrize( ("map_type", "struct_type"), [ From e5c4fc45ccdf2c3507f632178485dfdbfd4595ce Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Fri, 16 Oct 2026 11:38:28 +0900 Subject: [PATCH 7/8] Backport #1200: Show is_disconnect() the pinged connection's attributes Before SQLAlchemy 2.0.5, DefaultDialect.do_ping() passes the connection it pings to is_disconnect(). It received a namespace with only cursor(), so an is_disconnect() override that reads the connection, such as its schema_name or an adapter's driver_connection, raised AttributeError and the checkout failed instead of reconnecting. The ping now passes a small proxy whose cursor() opens the API cursor and whose other attributes are the pinged connection's. The regression test now fails a real ping: its stand-in for DefaultDialect.do_ping() asks is_disconnect() as SQLAlchemy before 2.0.5 does, so neither the earlier override nor the namespace pass it, and no AWS request is made. (cherry picked from commit 29d61f39769f98c67c182f43981289e43ac9af5f) 3.x resolution: - pyathena/sqlalchemy/base.py: _PingConnection.cursor() opens the API cursor with self._connection.cursor(Cursor), since 3.x has no Connection._internal_cursor() (#1102). The async adapter maps Cursor to AioCursor since the #798 commit ported before. - tests/pyathena/sqlalchemy/test_base.py: the test uses the 3.x adapter names AsyncAdapt_pyathena_connection and AsyncAdapt_pyathena_cursor, and OperationalError is added to 3.x's imports. Co-Authored-By: Claude Opus 5.5 --- pyathena/sqlalchemy/base.py | 33 ++++++++++++++++--- tests/pyathena/sqlalchemy/test_base.py | 45 +++++++++++++++++++------- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/pyathena/sqlalchemy/base.py b/pyathena/sqlalchemy/base.py index 81473aadb..d714dc665 100644 --- a/pyathena/sqlalchemy/base.py +++ b/pyathena/sqlalchemy/base.py @@ -4,7 +4,6 @@ import re from collections.abc import Mapping, MutableMapping from re import Pattern -from types import SimpleNamespace from typing import ( TYPE_CHECKING, Any, @@ -88,6 +87,33 @@ } +class _PingConnection: + """A DBAPI connection whose ``cursor()`` opens an API cursor. + + ``AthenaDialect.do_ping()`` passes it to ``DefaultDialect.do_ping()``, so the ping + does not run through the connection's default cursor class. Other attributes are + those of the pinged connection, which ``is_disconnect()`` receives before + SQLAlchemy 2.0.5. + + Args: + connection: The pinged DBAPI connection, or the async adapter around one. + """ + + def __init__(self, connection: Any) -> None: + self._connection = connection + + def cursor(self) -> Any: + """Open an API cursor on the pinged connection. + + Returns: + The cursor, wrapped by the async adapter for an async connection. + """ + return self._connection.cursor(Cursor) + + def __getattr__(self, name: str) -> Any: + return getattr(self._connection, name) + + class AthenaDialect(DefaultDialect): """SQLAlchemy dialect for Amazon Athena. @@ -293,10 +319,7 @@ def do_ping(self, dbapi_connection: DBAPIConnection) -> bool: # Using the plain cursor here performs the SELECT without the UNLOAD. # The ping still runs in DefaultDialect.do_ping(), which before SQLAlchemy 2.0.5 # is also where a failed ping is checked for a disconnect. - api_connection = SimpleNamespace( - cursor=lambda: dbapi_connection.cursor(Cursor) # type: ignore[call-arg] - ) - return super().do_ping(cast("DBAPIConnection", api_connection)) + return super().do_ping(cast("DBAPIConnection", _PingConnection(dbapi_connection))) @reflection.cache def _get_tables(self, connection, schema: str | None = None, **kw): diff --git a/tests/pyathena/sqlalchemy/test_base.py b/tests/pyathena/sqlalchemy/test_base.py index 90de4a34e..bbc80b21b 100644 --- a/tests/pyathena/sqlalchemy/test_base.py +++ b/tests/pyathena/sqlalchemy/test_base.py @@ -32,6 +32,7 @@ ) from pyathena.connection import Connection from pyathena.cursor import Cursor +from pyathena.error import OperationalError from pyathena.pandas.cursor import PandasCursor from pyathena.sqlalchemy.base import AthenaDialect from pyathena.sqlalchemy.rest import AthenaRestDialect @@ -149,32 +150,54 @@ def test_do_ping_keeps_default_dialect_ping( ): """DefaultDialect.do_ping() runs the ping with the API cursor and decides its result. - Before SQLAlchemy 2.0.5, it returns False when a failed ping is a disconnect, so - that the pool reconnects; the stand-in here returns False the same way. + The stand-in checks a failed ping as SQLAlchemy before 2.0.5 does: it asks + is_disconnect() with the pinged connection and the cursor, and returns False for + a disconnect so that the pool reconnects. """ conn = connection_class( region_name="us-west-2", s3_staging_dir="s3://bucket/path/", aws_access_key_id="access_key", aws_secret_access_key="secret_key", + schema_name="pinged_schema", cursor_class=cursor_class, cursor_kwargs={"unload": True}, ) - cursors = [] + error = OperationalError("ExpiredTokenException") + checked = [] + + def execute(self, operation): + raise error + + def is_disconnect(self, e, connection, cursor): + checked.append((e, connection.schema_name, cursor)) + return True def do_ping(self, dbapi_connection): - cursors.append(dbapi_connection.cursor()) - return False + cursor = dbapi_connection.cursor() + try: + cursor.execute(self._dialect_specific_select_one) + except OperationalError as e: + if self.is_disconnect(e, dbapi_connection, cursor): + return False + raise + finally: + cursor.close() + return True + if connection_class is AioConnection: + dbapi_connection = AsyncAdapt_pyathena_connection(MagicMock(), conn) + monkeypatch.setattr(AsyncAdapt_pyathena_cursor, "execute", execute) + else: + dbapi_connection = conn + monkeypatch.setattr(Cursor, "execute", execute) monkeypatch.setattr(DefaultDialect, "do_ping", do_ping) - dbapi_connection = ( - AsyncAdapt_pyathena_connection(MagicMock(), conn) - if connection_class is AioConnection - else conn - ) + monkeypatch.setattr(dialect_class, "is_disconnect", is_disconnect) try: assert dialect_class().do_ping(dbapi_connection) is False - [cursor] = cursors + [(e, schema_name, cursor)] = checked + assert e is error + assert schema_name == "pinged_schema" if connection_class is AioConnection: cursor = cursor._cursor assert type(cursor) is expected From 90a9922fda4f0fc7e2803db8a2a58bda31f9c206 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Fri, 16 Oct 2026 11:51:21 +0900 Subject: [PATCH 8/8] Backport #1200: Document the ping proxy's constructor argument on __init__ The repository's docstring rules describe constructor arguments in the Args: section of __init__, not of the class. (cherry picked from commit 35734209a2162fed99da8bb2f2421bbb82f208e6) Co-Authored-By: Claude Opus 5.5 --- pyathena/sqlalchemy/base.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyathena/sqlalchemy/base.py b/pyathena/sqlalchemy/base.py index d714dc665..082033caa 100644 --- a/pyathena/sqlalchemy/base.py +++ b/pyathena/sqlalchemy/base.py @@ -94,12 +94,14 @@ class _PingConnection: does not run through the connection's default cursor class. Other attributes are those of the pinged connection, which ``is_disconnect()`` receives before SQLAlchemy 2.0.5. - - Args: - connection: The pinged DBAPI connection, or the async adapter around one. """ def __init__(self, connection: Any) -> None: + """Wrap the pinged connection. + + Args: + connection: The pinged DBAPI connection, or the async adapter around one. + """ self._connection = connection def cursor(self) -> Any: