From 844c25c05ea26a87988c939d62510a9f912233f7 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 2 Nov 2026 08:35:34 -0800 Subject: [PATCH] try pyiceberg-core 0.11.0rc3 Pin pyiceberg-core to the Apache Iceberg Rust 0.11.0 RC3 build and move the datafusion extra to 54.x, matching the datafusion-ffi 54.1 bundled in pyiceberg-core 0.11. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- mkdocs/docs/api.md | 2 +- pyproject.toml | 6 +++--- uv.lock | 38 ++++++++++++++++++++++---------------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index a732d2c0a3..407227fda8 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -2247,7 +2247,7 @@ PyIceberg integrates with [Apache DataFusion](https://datafusion.apache.org/) th The integration has a few caveats: - - Only works with `datafusion == 53`, aligns with the version used in `pyiceberg-core` + - Only works with `datafusion == 54`, aligns with the version used in `pyiceberg-core` - Depends directly on `iceberg-rust` instead of PyIceberg's implementation - Has limited features compared to the full PyIceberg API diff --git a/pyproject.toml b/pyproject.toml index 895d15c524..594a0ed819 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,9 +94,9 @@ rest-sigv4 = ["boto3>=1.24.59"] hf = ["huggingface-hub>=0.24.0"] # `pyiceberg-core` bundles a `datafusion-ffi` build, and the DataFusion FFI ABI is not stable # across major versions (https://github.com/apache/datafusion/issues/17374). The two ranges below -# must name the same DataFusion major version: pyiceberg-core 0.10.x bundles datafusion-ffi 53.x. -pyiceberg-core = ["pyiceberg-core>=0.10.1,<0.11.0"] -datafusion = ["datafusion>=53,<54"] +# must name the same DataFusion major version: pyiceberg-core 0.11.x bundles datafusion-ffi 54.x. +pyiceberg-core = ["pyiceberg-core==0.11.0rc3"] +datafusion = ["datafusion>=54,<55"] gcp-auth = ["google-auth>=2.4.0"] entra-auth = ["azure-identity>=1.25.1"] geoarrow = ["geoarrow-pyarrow>=0.2.0"] diff --git a/uv.lock b/uv.lock index dd2523b5f0..ad804f397b 100644 --- a/uv.lock +++ b/uv.lock @@ -1129,19 +1129,25 @@ wheels = [ [[package]] name = "datafusion" -version = "53.0.0" +version = "54.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "cloudpickle" }, { name = "pyarrow" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/58/2b/0f96f12b70839c93930c4e17d767fc32b6c77d548c78784128049e944701/datafusion-53.0.0.tar.gz", hash = "sha256:ba9a5ec06b5453fbd8710d6aeeb515a8bcac4b6c140e254409bb53a5f322ef22", size = 224267, upload-time = "2026-04-13T00:45:02.686Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/3d/d564d983812e78614a62cd977e347827fec3c353b528e58ed0ffd0396d8b/datafusion-54.1.0.tar.gz", hash = "sha256:0045864afee13a137b92c40406fe59f0dba46deb8f4f0c933bf7c74591946c58", size = 276477, upload-time = "2026-10-05T12:21:58.370Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/4c/60e052813d81f1ffe3123ead013dbdd2cf961daa576cb9056cbb80228e6b/datafusion-53.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a0bd1a98d736571321416dc4ed361a9d1225da1ec9f6c5fad818d75f547697a7", size = 35774913, upload-time = "2026-04-13T00:44:46.235Z" }, - { url = "https://files.pythonhosted.org/packages/6e/59/beabe5301df3338d8206446cd624079e43bdad46e20377a6336017fb6ccf/datafusion-53.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ce186a8d2405afd67e11e2fb75715019f16b00d070b8d0da89d8aa61cc74c8b5", size = 32667118, upload-time = "2026-04-13T00:44:50.269Z" }, - { url = "https://files.pythonhosted.org/packages/ae/94/636ab61ade98395daea6e733e225e9c7beef111c7c5b575ac851513e203c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:288a00a7ef03e2807a4667683f7560efd80d60ed1d41696ac15ca9ded14c8251", size = 35585824, upload-time = "2026-04-13T00:44:53.683Z" }, - { url = "https://files.pythonhosted.org/packages/34/80/b9f4889209af02f8d14bccb0e6f0519c329b072bc4d2595025a1303f144c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8fef0004f0161fcfc556c025a7201f9cc3169aa3adb97a86419ebb34182d9efb", size = 38083690, upload-time = "2026-04-13T00:44:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/4b/1a/ea4831fc6aeefedbcf186c9f6a273d507b1787c03cbb905bded7e1149a6a/datafusion-53.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:4c8410f5f659b926677be6c7d443bbc05d825c078c970b7d8cf977ebcf948314", size = 38120687, upload-time = "2026-04-13T00:45:00.633Z" }, + { url = "https://files.pythonhosted.org/packages/37/07/fba9fc887b72c6e069b2daf06e34be2e2beabd3e41a3df4669d0a33b80fc/datafusion-54.1.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:77c6575baaa7526911bf442f679971955985ad7a6095d009f61ea676129fdbbc", size = 39140971, upload-time = "2026-10-05T12:21:06.308Z" }, + { url = "https://files.pythonhosted.org/packages/aa/bd/5c211a56113144fc928b2f2ce079e53c41f9ddcb8d5ea2d6726a05f39ae0/datafusion-54.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b8019bd12ee41d043fc4875867a5d1a13c5757e1adee239056efef2fd7191ea7", size = 36584427, upload-time = "2026-10-05T12:21:11.404Z" }, + { url = "https://files.pythonhosted.org/packages/54/00/2d905992da4ec20a5ab438fedc006f852207f4d5ca32902ac74844a34587/datafusion-54.1.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0b4270eb031f841de729d883625d7d554ad993900bf8fcc43864710e379e6b21", size = 38149873, upload-time = "2026-10-05T12:21:17.337Z" }, + { url = "https://files.pythonhosted.org/packages/54/28/33fe20840071394fbd236aadee6bcfd3275be55bf302244ab85fc1d55910/datafusion-54.1.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e4bba2ab6afba3c5939d334430f56a2c25285b69d7c23489afb6c51988e29123", size = 40311653, upload-time = "2026-10-05T12:21:22.469Z" }, + { url = "https://files.pythonhosted.org/packages/37/57/f244efebc049663fdeec1f778f9e81b5cf1cb8c2c20230fd3efdd221d2aa/datafusion-54.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:6a007203478e09a6e204eb968013c60a86752b830cdbb15e20ed4095857eb304", size = 42858869, upload-time = "2026-10-05T12:21:28.800Z" }, + { url = "https://files.pythonhosted.org/packages/c4/7b/04e7fdd5741b45d2b2b510291fc54c1567f837cd07fb747b46619f206dfc/datafusion-54.1.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ca790f6aaa93065a44047f931248cd2ab98bed52c81acd84aaf8f0b120d2f696", size = 39100935, upload-time = "2026-10-05T12:21:34.265Z" }, + { url = "https://files.pythonhosted.org/packages/56/5c/4e4706e8193795127d97e0974dd14d2daae3390254e47a80abee51622152/datafusion-54.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:12a5fb5c1b18ac90a4aa5813ea0d3c89ced3c394b242ebe07ab73ca80ebac5ac", size = 36575605, upload-time = "2026-10-05T12:21:39.245Z" }, + { url = "https://files.pythonhosted.org/packages/15/e7/23fad288be09bcc0acc497dfd5eea4ed440ac62f2ccd5d080975145fe87f/datafusion-54.1.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:5448d7e0cca95e205b75566c115db60270e4d40895a9a013afcf3f0c8584bc3c", size = 38167545, upload-time = "2026-10-05T12:21:44.748Z" }, + { url = "https://files.pythonhosted.org/packages/43/77/b530f938fd82ff9d1e10b2779d6024e09c997360a709afde275140b8c779/datafusion-54.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3890c605e13127732b12e8301e124dfda4e05d36293135a52260f07094455df9", size = 40317700, upload-time = "2026-10-05T12:21:50.649Z" }, + { url = "https://files.pythonhosted.org/packages/1d/82/2fbc21f676b427ccdc94b5f994cd3b187518ae4be622043171936d27f152/datafusion-54.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:abf3c7f84d3936015320fc8fc842c7fff4d5d8b24fb344a0ee9b8c1fa41e69d5", size = 42862503, upload-time = "2026-10-05T12:21:56.345Z" }, ] [[package]] @@ -4630,7 +4636,7 @@ requires-dist = [ { name = "click", specifier = ">=7.1.1" }, { name = "cryptography", marker = "extra == 'encryption'", specifier = ">=42.0.0" }, { name = "daft", marker = "extra == 'daft'", specifier = ">=0.7.10" }, - { name = "datafusion", marker = "extra == 'datafusion'", specifier = ">=53,<54" }, + { name = "datafusion", marker = "extra == 'datafusion'", specifier = ">=54,<55" }, { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=0.5.0" }, { name = "fsspec", specifier = ">=2023.1.0" }, { name = "gcsfs", marker = "extra == 'gcsfs'", specifier = ">=2023.1.0" }, @@ -4649,7 +4655,7 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'pyarrow'", specifier = ">=18.0.0" }, { name = "pyarrow", marker = "extra == 'ray'", specifier = ">=18.0.0" }, { name = "pydantic", specifier = ">=2.0,!=2.4.0,!=2.4.1,!=2.12.0,!=2.12.1,<3.0" }, - { name = "pyiceberg-core", marker = "extra == 'pyiceberg-core'", specifier = ">=0.10.1,<0.11.0" }, + { name = "pyiceberg-core", marker = "extra == 'pyiceberg-core'", specifier = "==0.11.0rc3" }, { name = "pyparsing", specifier = ">=3.1.0" }, { name = "pyroaring", specifier = ">=1.0.0" }, { name = "python-snappy", marker = "extra == 'snappy'", specifier = ">=0.6.0" }, @@ -4713,15 +4719,15 @@ notebook = [{ name = "jupyterlab", specifier = ">=4.0.0" }] [[package]] name = "pyiceberg-core" -version = "0.10.1" +version = "0.11.0rc3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/0a/fa73e70a8af2c600fa8089a009c9be99587f4f62a1dd674acbb15f5dab91/pyiceberg_core-0.10.1.tar.gz", hash = "sha256:c5e600728071032a4027c4c36680e4806c98f443057a26523532a2f830db4c89", size = 883913, upload-time = "2026-08-01T18:34:38.955Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/f1/0428e51019324eee0a8129f0db6edf2522a93f16c8f9e91e1d91e7b96fd5/pyiceberg_core-0.11.0rc3.tar.gz", hash = "sha256:556a38cb94bc58ecdffb39ee7cdc104bee168b93a2581112d4541ae56197e076", size = 1017266, upload-time = "2026-10-28T21:30:53.980Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/8e/a22c7b1798023bc2a2bcdbe12930d06509be034ad7ec448cfdf5308fe3a7/pyiceberg_core-0.10.1-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5ae7490fa3d03d32eab6e15116ddbeb0899cb3ba8af9332e302f2f3ca8e7667c", size = 24865938, upload-time = "2026-08-01T18:34:23.329Z" }, - { url = "https://files.pythonhosted.org/packages/3c/5a/f97796aff09011e0d91f6e8d2715933159547b36711c00f55c179214a4d2/pyiceberg_core-0.10.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb6e7188243e1cf34d3897d6642078b3cc170935bb1955bc1c5096dd32f6719a", size = 11696981, upload-time = "2026-08-01T18:34:26.734Z" }, - { url = "https://files.pythonhosted.org/packages/76/72/7a259abb1b3bfee4216c9e307c6b5f96850a45bd88b4be58a46143dfc051/pyiceberg_core-0.10.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413bb2e699d1957c98302c1bce5a0bf36fc5acf11797c48de5c08067df7b27b4", size = 14046735, upload-time = "2026-08-01T18:34:29.802Z" }, - { url = "https://files.pythonhosted.org/packages/82/a1/b4ffa500ea9681ebb1db173b4bd6c0bfdd8707e7ac6d5b7c0a9a49ddfa2f/pyiceberg_core-0.10.1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a8974a5c93282455ed023e28f5291bb899b91731631d93f0ccf0411ad32efb71", size = 14502489, upload-time = "2026-08-01T18:34:32.808Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c1/f0fdd495b8312b295726135c16a84e3b63a0e21977af465401a8d6f10925/pyiceberg_core-0.10.1-cp310-abi3-win_amd64.whl", hash = "sha256:884969c030be824d5ce7998d96215741d0e34351cbe995df6a156947b1eb7472", size = 13293439, upload-time = "2026-08-01T18:34:36.634Z" }, + { url = "https://files.pythonhosted.org/packages/4c/37/a20ecd469353e423ed560f55ec7b708364213df7df638e00e83499e3b480/pyiceberg_core-0.11.0rc3-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:e20d9abda0cf1bc7118daedfcb049ea3d430809fdb4c5066e40f80b00611b05b", size = 25191704, upload-time = "2026-10-28T21:30:39.996Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f4/78f61e8c608d2d7065db41e1173f256930f87e6846b3174f227ab85e9c35/pyiceberg_core-0.11.0rc3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0387a34c91a36d16862f88d2038022a2b98ca0976803cbcc45822d7b997ddd56", size = 12421098, upload-time = "2026-10-28T21:30:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/e3/d6/f8ac94500f834aebac94888ba255c7626a1c3592c90dc90146fb4098808b/pyiceberg_core-0.11.0rc3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f510f489aaf612c6d9a1d210e907df48941bf9c5b58fee5d7f8f5a36c01a4548", size = 14663126, upload-time = "2026-10-28T21:30:45.638Z" }, + { url = "https://files.pythonhosted.org/packages/0a/6d/24037539640a22ef85f92e1dc8e8765947198313c6fd2e871fea248a5729/pyiceberg_core-0.11.0rc3-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d1a240c983488ae8209b7b27aa1c64b7db07e90d851424b9ac5a23026b287e66", size = 15099810, upload-time = "2026-10-28T21:30:48.444Z" }, + { url = "https://files.pythonhosted.org/packages/cc/70/ecd48d1fe7f6480ed172bf991e76d5444d0b67925cfddd7a8de6aea3adae/pyiceberg_core-0.11.0rc3-cp310-abi3-win_amd64.whl", hash = "sha256:020f0387daabf7f60deef59329603d97f700cda16ff8a15a0465510555dd0cf0", size = 13739426, upload-time = "2026-10-28T21:30:51.713Z" }, ] [[package]]