diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index aa7c111329..f823a2fc60 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -311,8 +311,8 @@ Next, write the data to the table. Both `append` and `overwrite` produce the sam -!!! note inline end "Fast append" - PyIceberg defaults to the [fast append](https://iceberg.apache.org/spec/#snapshots) to minimize the amount of data written. This enables fast commit operations, reducing the possibility of conflicts. The downside of the fast append is that it creates more metadata than a merge commit. [Compaction is planned](https://github.com/apache/iceberg-python/issues/270) and will automatically rewrite all the metadata when a threshold is hit, to maintain performant reads. +!!! note inline end "Merge and fast append" + PyIceberg defaults to [merge append](https://iceberg.apache.org/spec/#snapshots), which automatically merges manifests when the configured threshold is reached to keep metadata compact and reads performant. Fast append remains available by setting the table property `commit.manifest-merge.enabled` to `False`. Fast append minimizes the metadata written during a commit, enabling faster commits and reducing the possibility of conflicts, but accumulates more manifest metadata over time. diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index c2f3ef5870..f3e4e85845 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -106,12 +106,12 @@ Iceberg tables support table properties to configure table behavior. | ------------------------------------ | ------------------- | ------------- | ----------------------------------------------------------- | | `commit.manifest.target-size-bytes` | Size in bytes | 8388608 (8MB) | Target size when merging manifest files | | `commit.manifest.min-count-to-merge` | Number of manifests | 100 | Minimum number of manifests to accumulate before merging | -| `commit.manifest-merge.enabled` | Boolean | False | Controls whether to automatically merge manifests on writes | +| `commit.manifest-merge.enabled` | Boolean | True | Controls whether to automatically merge manifests on writes | -!!! note "Fast append" - Unlike Java implementation, PyIceberg default to the [fast append](api.md#write-to-a-table) and thus `commit.manifest-merge.enabled` is set to `False` by default. +!!! note "Append modes" + PyIceberg defaults to [merge append](api.md#write-to-a-table) to keep manifest metadata compact. Set `commit.manifest-merge.enabled` to `False` on a table to use fast append, which writes less metadata during each commit but accumulates more manifests over time. diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..05ec45094f 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -226,7 +226,7 @@ class TableProperties: MANIFEST_MIN_MERGE_COUNT_DEFAULT = 100 MANIFEST_MERGE_ENABLED = "commit.manifest-merge.enabled" - MANIFEST_MERGE_ENABLED_DEFAULT = False + MANIFEST_MERGE_ENABLED_DEFAULT = True METADATA_COMPRESSION = "write.metadata.compression-codec" METADATA_COMPRESSION_DEFAULT = "none" diff --git a/tests/integration/test_writes/test_partitioned_writes.py b/tests/integration/test_writes/test_partitioned_writes.py index 1d1488255f..040c0cfeea 100644 --- a/tests/integration/test_writes/test_partitioned_writes.py +++ b/tests/integration/test_writes/test_partitioned_writes.py @@ -748,8 +748,11 @@ def test_dynamic_partition_overwrite_evolve_partition(spark: SparkSession, sessi tbl.dynamic_partition_overwrite(arrow_table) result = tbl.scan().to_arrow() - assert result["place"].to_pylist() == ["Groningen", "Amsterdam", "Drachten"] - assert result["inhabitants"].to_pylist() == [238147, 921402, 44940] + assert sorted(zip(result["place"].to_pylist(), result["inhabitants"].to_pylist(), strict=True)) == [ + ("Amsterdam", 921402), + ("Drachten", 44940), + ("Groningen", 238147), + ] @pytest.mark.integration diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index e0d647c2e9..ba367b0f9c 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1572,7 +1572,7 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab tbl_a = _create_table( session_catalog, "default.merge_manifest_a", - {"commit.manifest-merge.enabled": "true", "commit.manifest.min-count-to-merge": "1", "format-version": format_version}, + {"commit.manifest.min-count-to-merge": "1", "format-version": format_version}, [], ) tbl_b = _create_table( @@ -1589,7 +1589,11 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab tbl_c = _create_table( session_catalog, "default.merge_manifest_c", - {"commit.manifest.min-count-to-merge": "1", "format-version": format_version}, + { + "commit.manifest-merge.enabled": "false", + "commit.manifest.min-count-to-merge": "1", + "format-version": format_version, + }, [], ) diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 84169a7d1c..3ecf75fc68 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -814,6 +814,25 @@ def test_overwrite_rejects_explicit_delete_missing_from_base_snapshot(catalog: C assert _total_data_file_count(committed) == 0 +def test_append_producer_defaults_to_merge_append(table_v2: Table) -> None: + from pyiceberg.table.update.snapshot import _MergeAppendFiles + + append = table_v2.transaction()._append_snapshot_producer({}) + + assert isinstance(append, _MergeAppendFiles) + + +def test_append_producer_uses_fast_append_when_merge_disabled(table_v2: Table) -> None: + from pyiceberg.table.update.snapshot import _FastAppendFiles + + transaction = table_v2.transaction() + transaction.set_properties({"commit.manifest-merge.enabled": "false"}) + + append = transaction._append_snapshot_producer({}) + + assert isinstance(append, _FastAppendFiles) + + def test_overwrite_rejects_explicit_delete_without_parent_snapshot( catalog: Catalog, overwrite_table: Table, arrow_table_simple: pa.Table ) -> None: