diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 3a1c2380aa7..1d0a50c7a41 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -546,6 +546,11 @@ jobs: org.apache.comet.CometIcebergResidualPushdownSuite org.apache.comet.iceberg.IcebergReflectionSuite org.apache.comet.serde.operator.IcebergWriteProtoTranslationSuite + org.apache.comet.CometIcebergCompactionSuite + org.apache.comet.iceberg.compaction.BinPackPlannerSuite + org.apache.comet.iceberg.compaction.CompactionSourceSuite + org.apache.comet.iceberg.compaction.CompactionScanIcebergSuite + org.apache.comet.iceberg.compaction.CompactionOptionsSuite org.apache.comet.csv.CometCsvNativeReadSuite org.apache.comet.text.CometTextNativeReadSuite org.apache.comet.CometFuzzTestSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index 48375ba31fd..20607845393 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -184,6 +184,11 @@ jobs: org.apache.comet.CometIcebergResidualPushdownSuite org.apache.comet.iceberg.IcebergReflectionSuite org.apache.comet.serde.operator.IcebergWriteProtoTranslationSuite + org.apache.comet.CometIcebergCompactionSuite + org.apache.comet.iceberg.compaction.BinPackPlannerSuite + org.apache.comet.iceberg.compaction.CompactionSourceSuite + org.apache.comet.iceberg.compaction.CompactionScanIcebergSuite + org.apache.comet.iceberg.compaction.CompactionOptionsSuite org.apache.comet.csv.CometCsvNativeReadSuite org.apache.comet.text.CometTextNativeReadSuite org.apache.comet.CometFuzzTestSuite diff --git a/docs/source/contributor-guide/iceberg-compaction.md b/docs/source/contributor-guide/iceberg-compaction.md new file mode 100644 index 00000000000..4330f6573d2 --- /dev/null +++ b/docs/source/contributor-guide/iceberg-compaction.md @@ -0,0 +1,213 @@ + + +# Iceberg Compaction + +This document describes how Comet runs Iceberg's `rewrite_data_files` procedure with the `binpack` +strategy: how a `CALL` reaches Comet, how the rewrite is planned, run and committed, and how to test +a change. For the user-facing view (configuration, supported options, and when Comet leaves a call to +Iceberg), see [Compaction](../user-guide/latest/iceberg.md#compaction). + +The rule that governs the whole feature is the one that governs the [Iceberg +writes](iceberg-writes.md): **Comet must produce the outcome iceberg-java would have produced, or +decline.** A call Comet does not fully reproduce runs Iceberg's own procedure, and it is decided +before anything is written. + +## Overview + +```text +CALL system.rewrite_data_files(...) + -> optimizer rule swaps Iceberg's procedure for CometRewriteDataFilesProcedure (Spark 4.x) + -> eligible? no: Iceberg's procedure runs the call + -> CometRewriteDataFiles.prepare plan the file groups, change nothing + -> PreparedRewrite.execute plan the jobs, run them, commit + bins job: CompactionSource scan, one partition per bin (CometIcebergNativeScan) + Iceberg rewrite write (CometIcebergWrite when eligible) + staged job: Iceberg staged scan of one group (CometIcebergNativeScan) + Iceberg rewrite write (CometIcebergWrite when eligible) + commit: Table.newRewrite() on the driver +``` + +| Component | Location | Role | +| ------------------------------------- | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `CometIcebergProcedureRule` | `spark/src/main/scala/org/apache/comet/iceberg/compaction/` | Optimizer rule. Looks at the root of every optimized plan for a `CALL`. | +| `CometProcedureShim` | `spark-4.x/` and `spark-3.x/`, same package | Swaps the procedure on Spark 4. A no-op on Spark 3. | +| `CometRewriteDataFilesProcedure` | `spark-4.x/`, same package | The `BoundProcedure` Spark invokes. Decodes the arguments, asks for a plan, and falls back to Iceberg's procedure. | +| `CometRewriteDataFiles` | `spark/src/main/scala/org/apache/comet/iceberg/compaction/` | The rewrite: planning, running the groups, committing, cleaning up. | +| `BinPackPlanner`, `CompactionOptions` | same | The planning logic and the option parsing. Pure Scala with no Iceberg classes, so they are unit tested on every Spark profile. | +| `CompactionSource`, `CompactionScan` | same | The Spark data source of a bins job. A scan whose partitions are the bins the planner chose. | +| `IcebergRewriteReflection` | same | Every call Comet makes on Iceberg. Comet does not link Iceberg, so these are reflective. | + +## How a `CALL` reaches Comet + +Spark 4 runs a `CALL` while it analyzes the statement. The `InvokeProcedures` rule optimizes the +`Call` node and then invokes the procedure it holds, and no physical operator exists. Resolving, +binding and invoking the procedure happen in one pass of the analyzer, so a resolution rule would +run too late. The optimizer is the one place between binding and invoking. `CometIcebergProcedureRule` +is injected with `injectOptimizerRule`, and replaces the `BoundProcedure` inside the +`ResolvedProcedure` when it is Iceberg's `RewriteDataFilesProcedure` in the `system` namespace and +`spark.comet.iceberg.compaction.enabled` is on. + +The rule runs on every optimized plan, so it checks only the root, and the conf lookup happens after +the match. + +Spark 3 is different. With Iceberg's SQL extensions the statement becomes Iceberg's own `Call` +logical node, which the extension's planner strategy turns into `CallExec`. Intercepting it needs a +planner strategy that runs before Iceberg's, and reflection on Iceberg's `Procedure` interface, +since that interface ships in the Iceberg jar. This is not implemented, and `CometProcedureShim` on +Spark 3 returns the plan unchanged. + +## Deciding whether Comet runs a call + +`CometRewriteDataFilesProcedure.prepare` changes nothing, so any answer other than "run it" falls +back to `delegate.call(input)` safely. It declines when: + +- the result schema of Iceberg's procedure is not the five columns Comet produces. Iceberg keeps it + in the private `OUTPUT_TYPE`, and a release that adds a column must not be answered with a row of + the wrong shape +- the strategy is not `binpack`, or a sort order is given +- the table is not in the procedure's catalog, or is not a `SparkTable` +- `CompactionOptions.parse` returns a reason: an unknown option, a value Iceberg would reject, or + an option Comet does not reproduce +- anything in this list throws. Any non-fatal exception while preparing is logged and handled as a + decline, which gives the user Iceberg's error for a call that is invalid. + +Once `prepare` returns, `execute` is the call: its failures are the call's failures. + +## Planning + +`BinPackPlanner` is `BinPackRewriteFilePlanner` from iceberg-java in Scala. The steps are the ones +of iceberg-go's `compaction.PlanCompaction`: + +| Step | iceberg-go (`table/compaction`) | Comet | +| ------------------------------ | ------------------------------------------------------- | ------------------------------------------- | +| Scan the snapshot | `Analyze` plans the files of the table | `IcebergRewriteReflection.planFiles` | +| Bucket by partition | `partitionBucketKey(specID, partition)` | the `partition` key of `CompactionFile` | +| Keep the files worth rewriting | `isCandidate` | `isCandidate` | +| Pack a bucket | `SlicePacker.PackEnd` into bins of the target file size | `pack` into groups of `max-file-group-size` | +| Drop bins not worth a rewrite | `MinInputFiles` | `isWorthRewriting` | + +Two differences are on purpose. Defaults and option names are iceberg-java's, because the `CALL` +surface is iceberg-java's: `delete-file-threshold` is unbounded there and 5 in iceberg-go. And +iceberg-java packs into groups of up to 100 GB, which are the unit that fails and commits together, +then splits a group into read tasks of `inputSplitSize` when it reads it. iceberg-go packs bins of the +target file size directly, each of which is about one output file. `BinPackPlanner.planReadTasks` is +the second step of the iceberg-java approach, and what a scan with explicit bins would use. + +Every number comes from `SizeBasedFileRewritePlanner`: `expectedOutputFiles`, `inputSplitSize` and +the writer's rolling size `writeMaxFileSize`. The unit tests pin them with the examples in that +class's Javadoc. + +## Running the groups + +The rewrite is a list of jobs, and a job is one Spark job. `planJobs` makes the list, and +`max-concurrent-file-group-rewrites` of them run at a time. There are two kinds. + +**A staged job** rewrites one group. It does what `SparkBinPackFileRewriteRunner` does, with +DataFrames: stage the group's `FileScanTask`s under a fresh id in Iceberg's `ScanTaskSetManager`, +read them back with the `scan-task-set-id` option, and write with `rewritten-file-scan-task-set-id`. +That keeps the data movement in the two paths that already carry the parity guarantees: the staged +scan is converted to `CometIcebergNativeScan` by `CometScanRule`, and the write goes through +`IcebergWriteStrategy` and the eligibility gate. `CometIcebergRewriteActionSuite` is the suite that +shows both engage on the groups of Iceberg's own action. + +**A bins job** rewrites several groups in one job. Iceberg's staged scan packs the tasks of its +group into Spark partitions of about the split size, and a group is the unit it does that for, so +a table with a thousand partitions runs a thousand small jobs, a few at a time. A bins job instead +plans the partitions itself, with `BinPackPlanner.planReadTasks`, so that each is a bin of one +group, and reads them all in one job. The pieces are: + +- `IcebergRewriteReflection.splitTasks` splits each task of a group into read tasks of about + `inputSplitSize` with Iceberg's own `SplittableScanTask.split`, because `TableScanUtil.planTaskGroups` + does, and reads each task's `sizeBytes()` and `filesCount()`, which include its delete files. + Packing by the data file size alone would put bins in different places. +- `CompactionSource` is a Spark `TableProvider` that reads a `CompactionBatch` registered under a + `batch-id` option, the way Iceberg's reader looks a staged group up by id. Its scan, + `CompactionScan`, has one `CompactionInputPartition` per bin and no reader, because only Comet + can read it. `IcebergReflection.ICEBERG_SCAN_CLASSES` lists its class name, and it exposes the + `table()`, `expectedSchema()` and `tasks()` accessors that `CometScanRule` reads off an Iceberg + scan. From there the plan is the one of any Iceberg read: `CometIcebergNativeScan` serializes the + tasks of each partition's `taskGroup()`. `CompactionSourceSuite` pins the names. +- Because Spark cannot read a `CompactionScan`, the job is only used when Comet's native scan takes it + over. `binsJob` plans the scan DataFrame and looks for a `CometIcebergNativeScanExec` first, + which writes nothing, and a plan without one gives its groups staged jobs. The scan is declined for + the same reasons as any Iceberg scan, such as a file that is not Parquet. +- The write is the one of a staged job, `AppendData` with `rewritten-file-scan-task-set-id`, over + the scan of the bins. Its files are staged in `FileRewriteCoordinator` under the batch id. + `distribution-mode` is `none`, because every bin is within one group, which is within one + partition. A group whose files are not in the output spec needs a shuffle, so it gets a staged job. + A table with row lineage (format version 3) reads and writes extra columns in a rewrite that + only Iceberg's staged scan provides, so all of its groups get staged jobs. + +A bins job covers at most `groupsPerJob` groups and `MaxBinsPerJob` bins. Without partial progress +`groupsPerJob` is unbounded. With it, it is the number of groups in a commit, so that a job is a +commit and a failed job fails one commit. `MaxBinsPerJob` keeps the commit messages, which the +driver holds until the job ends, under `spark.driver.maxResultSize`. + +Both kinds of job run in a clone of the session with adaptive execution off, because it can change +the output partitioning of the write, and with Iceberg's executor cache for delete files off, because +each partition is rewritten once. Iceberg's action does the same. + +The Iceberg write for a rewrite does not commit. `SparkWrite.RewriteFiles` stages its files in +`FileRewriteCoordinator`, so `IcebergCommitExec` only collects the task messages, and the files are +read back from the coordinator when the job ends. If the job fails, `IcebergCommitExec` deletes the +files of the tasks that finished. + +## Committing + +`IcebergRewriteReflection.commitRewrite` is `RewriteDataFilesCommitManager.commitFileGroups`: +`Table.newRewrite()` validated from the starting snapshot, optionally pinned to its data sequence +number, deleting the rewritten data files and the deletion vectors of the rewritten tasks, and +adding the new files. `commitOrClean` keeps the files when the commit state is unknown, deletes them +when the failure is cleanable, and turns a conflict into the same message Iceberg gives. + +With `partial-progress.enabled`, `BatchCommitter` commits the finished jobs once they hold at +least `ceil(groups / partial-progress.max-commits)` groups, ignores a batch that fails, and fails the +call when more than `partial-progress.max-failed-commits` batches did. A commit takes whole jobs, so +it can hold more groups than that, but never so many that there are more commits than the option +allows. + +Deletion vectors are identified by their location, offset and length, as Iceberg's `DeleteFileSet` +does, because the vectors of many data files can share one Puffin file. A key of the location alone +would commit one of them and leave the others pointing at removed data files. + +## Testing + +| Suite | What it covers | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `BinPackPlannerSuite` | Which files are selected, how groups and read tasks are packed and ordered, and the size arithmetic. No Iceberg needed. | +| `CompactionOptionsSuite` | Defaults, the values Iceberg rejects, and the options Comet declines. | +| `CompactionSourceSuite` | How a registered batch reads through Spark: its schema, one partition per bin, and the accessor names Comet reads. No Iceberg needed. | +| `CompactionScanIcebergSuite` | Comet's native Iceberg scan over a `CompactionScan` of a real table: one native partition per bin, and the table's rows. Spark 4 only. | +| `CometIcebergCompactionSuite` | `CALL` end to end on Spark 4: the result row, the job group that shows which engine ran and how many jobs, the fallbacks, `where`, partial progress, and deletes. | + +When writing a test for a change here, assert which engine ran. Every fallback is silent to the +query, so a passing comparison can mean Iceberg's procedure ran. `CometIcebergCompactionSuite` reads +the job group of the jobs that ran. + +## Not done yet + +- **Spark 3.x.** See above. +- **The `sort` and `zorder` strategies.** They shuffle, and they need `SparkShufflingDataRewritePlanner`'s + options. +- **Writing row lineage.** On a format-version 3 table the Iceberg write for a rewrite adds the row + lineage columns, which the native writer does not write, so the groups of such a table are written by + Iceberg's writer. See [#6669](https://github.com/apache/datafusion-comet/issues/6669). +- **Overlapping the commit of one job with the next job.** Partial progress commits as jobs finish, + but a bins job covers a whole commit, so the first commit waits for its job. diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md index 1be327be39e..21eeb53a7a8 100644 --- a/docs/source/contributor-guide/index.md +++ b/docs/source/contributor-guide/index.md @@ -52,6 +52,7 @@ JVM Shuffle Native Shuffle Memory Management Iceberg Writes +Iceberg Compaction ANSI Error Propagation S3 Credential Provider Design ``` diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 9278616c652..8ac6d0e2f7a 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -25,8 +25,11 @@ Comet accelerates Iceberg tables whose data files are Parquet, in two ways: files in Rust. It is enabled by default. - **[Writing](#writes)**: with `spark.comet.write.iceberg.enabled=true`, Comet writes the data files of eligible writes natively. It is disabled by default. +- **[Compaction](#compaction)**: with `spark.comet.iceberg.compaction.enabled=true`, Comet runs + the `rewrite_data_files` procedure's bin-pack rewrite itself, on Spark 4.0 and later. It is + disabled by default. -This page covers both, followed by the [known issues](#known-issues). Behavior that applies to +This page covers all three, followed by the [known issues](#known-issues). Behavior that applies to Parquet files in general is documented on the [Parquet](parquet.md) page. ## Native Reader @@ -888,6 +891,96 @@ must match iceberg-java exactly, or the write falls back. The open bugs in which still differs are listed in [Known Issues](#writes-that-differ-from-spark). +## Compaction + +On Spark 4.0 and later, Comet can run Iceberg's `rewrite_data_files` procedure with the `binpack` +strategy itself. It is disabled by default. Set `spark.comet.iceberg.compaction.enabled=true` to +turn it on, and `spark.comet.write.iceberg.enabled=true` as well to have the new files written by +the [native writer](#writes). + +```sql +CALL catalog.system.rewrite_data_files( + table => 'db.events', + where => 'event_date = "2026-10-27"', + options => map('min-input-files', '10', 'partial-progress.enabled', 'true')) +``` + +The call, its arguments, its options and its result row are Iceberg's. Comet follows the steps of +Iceberg's own `RewriteDataFilesSparkAction`, so a call plans the same file groups with either +engine. + +### How Comet runs a rewrite + +1. **Plan.** Comet lists the data files of the table's current snapshot that match `where`, and + groups them the way Iceberg's `BinPackRewriteFilePlanner` does. A file is a candidate when its + size is outside `min-file-size-bytes` and `max-file-size-bytes`, or when it has at least + `delete-file-threshold` delete files, or when `delete-ratio-threshold` of its rows are deleted. + Candidates are packed within each partition into groups of up to `max-file-group-size-bytes`. + A group is kept when it has several files and at least `min-input-files` of them, when it has + several files and more than the target size, when it holds more than `max-file-size-bytes`, or + when one of its files qualifies for its deletes. +2. **Rewrite.** Comet reads the files of a group with the [native Iceberg reader](#native-reader), + in splits sized so that each becomes about one new file, and writes them back with the + [native writer](#writes) when it is eligible and Iceberg's writer otherwise. Deletes are + applied while reading, so the new files do not contain deleted rows. Iceberg rewrites each + group in a Spark job of its own, `max-concurrent-file-group-rewrites` jobs at a time, so a table + with many partitions has many small jobs and most of the cluster is idle. Comet rewrites all the + groups of a commit in one job, in which every Spark task reads one split and writes about one + file. A job covers at most 10,000 splits. A group is rewritten in a job of its own, as Iceberg + does it, when the native reader cannot read it, when its files are not in the output + partition spec, when the table is format version 3, or when + `spark.comet.iceberg.compaction.singleJob.enabled` is `false`. +3. **Commit.** The groups are committed as one `replace` snapshot that is validated against the + snapshot the plan read, so a concurrent change to a rewritten file fails the commit instead of + losing it. With `partial-progress.enabled`, groups are committed in up to + `partial-progress.max-commits` snapshots as they finish, and a job covers the groups of one + commit. If a commit fails, the files it wrote are deleted. + +### Supported calls + +- `strategy` is `binpack` or not given. `sort` and `zorder` run Iceberg's procedure. +- `sort_order` is not supported. A call that has one runs Iceberg's procedure. +- `where` is supported and converted by Iceberg. +- These options are supported, with Iceberg's defaults: `target-file-size-bytes`, + `min-file-size-bytes`, `max-file-size-bytes`, `min-input-files`, `rewrite-all`, + `max-file-group-size-bytes`, `delete-file-threshold`, `delete-ratio-threshold`, + `rewrite-job-order`, `max-files-to-rewrite`, `max-concurrent-file-group-rewrites`, + `partial-progress.enabled`, `partial-progress.max-commits`, + `partial-progress.max-failed-commits` and `use-starting-sequence-number`. +- `output-spec-id` is supported for the table's current spec only. +- `remove-dangling-deletes` is supported when it is `false`. + +### When Comet leaves a call to Iceberg + +Comet runs Iceberg's own procedure, with the call unchanged, when the call has something Comet does +not reproduce. That covers the strategies and options listed above, an option Comet does not know, +a value Iceberg would reject, a `where` that does not convert, and a table that is not an Iceberg +table. Running Iceberg's procedure is also how an invalid call gets Iceberg's error message. The +check happens before anything is written, so a call that Comet declines is never half done. The +driver logs the reason at `INFO`. + +A failure after the rewrite has started is the call's failure, as it is with Iceberg. + +### Telling which engine ran + +The jobs of a rewrite that Comet ran have the job group `COMET-REWRITE-DATA-FILES-` in the +Spark UI, and Iceberg's have `REWRITE-DATA-FILES-`. + +### Differences from Iceberg + +- A Spark job covers several file groups, so a task failure fails all of them. Without partial + progress the rewrite fails as it does in Iceberg when any group fails. With partial progress, a + job covers the groups of one commit, so a failure costs that commit. The files the job wrote are + deleted, and the rest of the rewrite goes on. +- `max-concurrent-file-group-rewrites` counts Spark jobs, which cover several groups each. +- Groups are started in the order the table's scan returned their partitions when + `rewrite-job-order` is `none`. Iceberg's order depends on how it hashes partitions. The other job + orders sort the same way. +- Comet commits one batch of groups at a time in `partial-progress` mode. Iceberg commits from + the thread that finished the group, so batches can commit at the same time. +- Spark 3.x is not supported yet. There, Iceberg's SQL extensions plan their own `CALL` node, which + Comet does not intercept. + ## Known Issues This section lists open issues in which Comet's Iceberg reads and writes give a different result diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 04fb3e537f7..af8916471a3 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -154,6 +154,34 @@ object CometConf extends ShimCometConf { .booleanConf .createWithDefault(false) + val COMET_ICEBERG_COMPACTION_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.iceberg.compaction.enabled") + .category(CATEGORY_EXEC) + .doc( + "Whether Comet runs the Iceberg `rewrite_data_files` procedure with the `binpack` " + + "strategy itself. Comet plans the file groups the way Iceberg does, rewrites each " + + "group through its native Iceberg scan and, when `spark.comet.write.iceberg.enabled` " + + "is also true, its native Iceberg writer, and commits the result with the same " + + "options and defaults as Iceberg. A call Comet cannot reproduce, such as a sort or " + + "z-order strategy, an unsupported option or a non-Iceberg table, runs Iceberg's own " + + "procedure instead. Requires Spark 4.0 or later.") + .booleanConf + .createWithDefault(false) + + val COMET_ICEBERG_COMPACTION_SINGLE_JOB_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.iceberg.compaction.singleJob.enabled") + .category(CATEGORY_EXEC) + .doc( + "Whether a compaction run by `spark.comet.iceberg.compaction.enabled` rewrites the file " + + "groups of each commit in one Spark job, in which every Spark task reads and rewrites " + + "about one output file of data. Iceberg runs one job per file group, " + + "`max-concurrent-file-group-rewrites` of them at a time, so a table with many small " + + "groups leaves most of the cluster idle. When false, or when Comet's native Iceberg " + + "scan cannot read a group, each group is rewritten in a job of its own as Iceberg " + + "does it.") + .booleanConf + .createWithDefault(true) + val COMET_ICEBERG_DATA_FILE_CONCURRENCY_LIMIT: ConfigEntry[Int] = conf("spark.comet.scan.icebergNative.dataFileConcurrencyLimit") .category(CATEGORY_SCAN) diff --git a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala index 81ed81a7028..5004d1279d5 100644 --- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala +++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala @@ -34,6 +34,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf._ import org.apache.comet.iceberg.IcebergWriteStrategy +import org.apache.comet.iceberg.compaction.CometIcebergProcedureRule import org.apache.comet.rules.{CometCoalesceShufflePartitions, CometPlanAdaptiveDynamicPruningFilters, CometReuseSubquery, CometRule, CometSpark34AqeDppFallbackRule} import org.apache.comet.shims.ShimCometSparkSessionExtensions @@ -116,6 +117,9 @@ class CometSparkSessionExtensions injectQueryStageOptimizerRuleShim(extensions, CometReuseSubquery) injectQueryStageOptimizerRuleShim(extensions, CometCoalesceShufflePartitions) extensions.injectPlannerStrategy { session => IcebergWriteStrategy(session) } + // Spark 4 invokes a CALL's procedure while it analyzes the statement, after optimizing it, so + // an optimizer rule is where Comet's rewrite_data_files replaces Iceberg's. + extensions.injectOptimizerRule { session => CometIcebergProcedureRule(session) } } case class CometColumnar(session: SparkSession) extends ColumnarRule { diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index 58278b488aa..a7de2ba29ba 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -57,6 +57,10 @@ object IcebergReflection extends Logging { val UNBOUND_PREDICATE = "org.apache.iceberg.expressions.UnboundPredicate" val SPARK_BATCH_QUERY_SCAN = "org.apache.iceberg.spark.source.SparkBatchQueryScan" val SPARK_STAGED_SCAN = "org.apache.iceberg.spark.source.SparkStagedScan" + + // Comet's own scan of a compaction job's bins, which exposes the accessors Comet reads off + // an Iceberg scan. + val COMPACTION_SCAN = "org.apache.comet.iceberg.compaction.CompactionScan" val SPARK_SCHEMA_UTIL = "org.apache.iceberg.spark.SparkSchemaUtil" val TABLE = "org.apache.iceberg.Table" val DEFAULT_LOCATION_PROVIDER = @@ -85,9 +89,15 @@ object IcebergReflection extends Logging { * `SparkStagedScan` also backs reads against Iceberg metadata tables (e.g. `POSITION_DELETES`), * but the gate for that lives in `getMetadataLocation`, which returns None for metadata-table * instances. + * + * `CompactionScan` is Comet's own scan of the bins of a compaction job. It is not an Iceberg + * class, but it exposes the accessors Comet reads off an Iceberg scan. */ val ICEBERG_SCAN_CLASSES: Set[String] = - Set(ClassNames.SPARK_BATCH_QUERY_SCAN, ClassNames.SPARK_STAGED_SCAN) + Set( + ClassNames.SPARK_BATCH_QUERY_SCAN, + ClassNames.SPARK_STAGED_SCAN, + ClassNames.COMPACTION_SCAN) def isIcebergScanClass(name: String): Boolean = ICEBERG_SCAN_CLASSES.contains(name) diff --git a/spark/src/main/scala/org/apache/comet/iceberg/compaction/BinPackPlanner.scala b/spark/src/main/scala/org/apache/comet/iceberg/compaction/BinPackPlanner.scala new file mode 100644 index 00000000000..0c728487839 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/iceberg/compaction/BinPackPlanner.scala @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import scala.collection.mutable + +/** + * One data file of a compaction scan, reduced to the facts the bin-pack planner decides on. + * + * `task` is an opaque handle to the Iceberg `FileScanTask` the facts were read from. The planner + * never looks inside it, which keeps the planning logic free of Iceberg classes. + * + * @param task + * handle to the `FileScanTask`, handed back unchanged in the planned groups + * @param length + * `FileScanTask.length()`, the number of bytes the task reads. This is the weight every packing + * and threshold decision uses. + * @param recordCount + * rows in the data file + * @param specId + * the id of the partition spec the file was written with + * @param partition + * the grouping key. Files with equal keys are packed together. Files written under an older + * partition spec than the table's current one carry an all-NULL key, the way Iceberg groups + * them. + * @param deleteFileCount + * `FileScanTask.deletes().size()` + * @param fileScopedDeletedRows + * records in the delete files of this task that apply to this data file alone (deletion vectors + * and position deletes scoped to one file) + */ +final case class CompactionFile( + task: AnyRef, + length: Long, + recordCount: Long, + specId: Int, + partition: Seq[Any], + deleteFileCount: Int, + fileScopedDeletedRows: Long) + +/** + * Files that are rewritten together, within one partition. This is Iceberg's `RewriteFileGroup` + * and is the unit that fails, retries and commits as a whole in partial-progress mode. + * + * @param globalIndex + * 1-based position among the selected groups + * @param partitionIndex + * 1-based position among the selected groups of the same partition + * @param partition + * the grouping key shared by every file + * @param files + * the input files, in scan order + * @param outputSpecId + * the partition spec the output files are written with + * @param maxOutputFileSize + * the file size the writer rolls at, which is a little above the target size so that a group + * that compresses slightly worse than estimated does not leave a tiny remainder file + * @param inputSplitSize + * how many bytes one read task takes from the group + * @param expectedOutputFiles + * how many files the group should produce + */ +final case class FileGroup( + globalIndex: Int, + partitionIndex: Int, + partition: Seq[Any], + files: Seq[CompactionFile], + outputSpecId: Int, + maxOutputFileSize: Long, + inputSplitSize: Long, + expectedOutputFiles: Int) { + + def numFiles: Int = files.size + + def inputBytes: Long = files.foldLeft(0L)(_ + _.length) +} + +/** + * The result of planning. + * + * @param groups + * the groups to rewrite, in job order + * @param totalGroupCount + * the number of groups before `max-files-to-rewrite` trimmed the selection. Partial-progress + * commits are sized from it. + */ +final case class CompactionPlan(groups: Seq[FileGroup], totalGroupCount: Int) + +/** + * Plans which data files are rewritten together by a bin-pack compaction. + * + * It follows iceberg-java's `BinPackRewriteFilePlanner` so that a `CALL rewrite_data_files` + * produces the same groups either way. The steps match iceberg-go's `PlanCompaction`: bucket the + * files by partition, keep the ones worth rewriting, pack each bucket into bins, and drop the + * bins that would not pay for a rewrite. The two differ in bin size. Iceberg-java packs up to + * `max-file-group-size-bytes` (100 GB by default) and splits a group into read tasks later, while + * iceberg-go packs bins of the target file size directly. This planner keeps the iceberg-java + * groups, and [[BinPackPlanner.planReadTasks]] makes the second step available. + */ +object BinPackPlanner { + + /** Bytes added to the split size so that a split holding a full file's worth never splits. */ + private val SplitOverhead = 5L * 1024 + + /** The planning lookback for read tasks, `read.split.planning-lookback` in Iceberg. */ + val DefaultSplitLookback = 10 + + def plan(files: Seq[CompactionFile], options: CompactionOptions): CompactionPlan = { + val byPartition = mutable.LinkedHashMap.empty[Seq[Any], mutable.ArrayBuffer[CompactionFile]] + files.foreach { f => + byPartition.getOrElseUpdate(f.partition, mutable.ArrayBuffer.empty[CompactionFile]) += f + } + + val groupsByPartition: Seq[(Seq[Any], Seq[Seq[CompactionFile]])] = + byPartition.toSeq.map { case (partition, partitionFiles) => + partition -> planFileGroups(partitionFiles.toSeq, options) + } + + val selected = mutable.ArrayBuffer.empty[FileGroup] + val partitionCounts = mutable.HashMap.empty[Seq[Any], Int] + var filesSelected = 0 + + def newGroup(partition: Seq[Any], groupFiles: Seq[CompactionFile], inputSize: Long) = { + val partitionIndex = partitionCounts.getOrElse(partition, 0) + 1 + partitionCounts(partition) = partitionIndex + FileGroup( + globalIndex = selected.size + 1, + partitionIndex = partitionIndex, + partition = partition, + files = groupFiles, + outputSpecId = options.outputSpecId, + maxOutputFileSize = options.writeMaxFileSize, + inputSplitSize = inputSplitSize(inputSize, options), + expectedOutputFiles = expectedOutputFiles(inputSize, options)) + } + + for { + (partition, groups) <- groupsByPartition + groupFiles <- groups + } { + // The split size and the output file count come from the whole group even when + // `max-files-to-rewrite` trims it, which is what iceberg-java does. + val inputSize = groupFiles.foldLeft(0L)(_ + _.length) + options.maxFilesToRewrite match { + case None => + selected += newGroup(partition, groupFiles, inputSize) + case Some(maxFiles) if filesSelected < maxFiles => + val take = math.min(groupFiles.size, maxFiles - filesSelected) + selected += newGroup(partition, groupFiles.take(take), inputSize) + filesSelected += take + case _ => + } + } + + val totalGroupCount = groupsByPartition.foldLeft(0)(_ + _._2.size) + CompactionPlan(orderGroups(selected.toSeq, options.jobOrder), totalGroupCount) + } + + /** + * Packs the read tasks of one group into bins, each of which becomes one Spark partition and, + * with the rolling writer, about one output file. Iceberg does this when it plans the read of a + * staged file group, with the group's [[FileGroup.inputSplitSize]] as the bin size and no cost + * for opening a file. This is `TableScanUtil.planTaskGroups` minus the splitting of large + * files, which the caller does first because it needs Iceberg's own tasks. + * + * @param tasks + * the read tasks, after large files have been split + * @param sizeBytes + * the bytes a task reads, including the delete files it applies + * @param filesCount + * the files a task opens, the data file and its delete files + */ + def planReadTasks[T]( + tasks: Seq[T], + splitSize: Long, + lookback: Int = DefaultSplitLookback, + openFileCost: Long = 0L)(sizeBytes: T => Long, filesCount: T => Int): Seq[Seq[T]] = { + pack(tasks, splitSize, lookback, largestBinFirst = true) { task => + math.max(sizeBytes(task), filesCount(task).toLong * openFileCost) + } + } + + /** + * Splits the groups of a rewrite into the jobs that run them, in order. A job takes groups + * until it has `maxGroups` of them, or until the next one would take it past `maxWeight`, which + * is counted in bins. A group heavier than `maxWeight` still gets a job of its own. + */ + private[compaction] def chunk[T](entries: Seq[T], maxGroups: Int, maxWeight: Int)( + weightOf: T => Int): Seq[Seq[T]] = { + val chunks = mutable.ArrayBuffer.empty[Seq[T]] + var current = mutable.ArrayBuffer.empty[T] + var weight = 0 + entries.foreach { entry => + val entryWeight = weightOf(entry) + val full = current.size >= maxGroups || weight.toLong + entryWeight > maxWeight + if (current.nonEmpty && full) { + chunks.append(current.toVector) + current = mutable.ArrayBuffer.empty[T] + weight = 0 + } + current.append(entry) + weight += entryWeight + } + if (current.nonEmpty) { + chunks.append(current.toVector) + } + chunks.toSeq + } + + private def planFileGroups( + files: Seq[CompactionFile], + options: CompactionOptions): Seq[Seq[CompactionFile]] = { + val candidates = if (options.rewriteAll) files else files.filter(isCandidate(_, options)) + val groups = + pack(candidates, options.maxGroupSize, lookback = 1, largestBinFirst = false)(_.length) + if (options.rewriteAll) groups else groups.filter(isWorthRewriting(_, options)) + } + + private def isCandidate(file: CompactionFile, options: CompactionOptions): Boolean = + outsideDesiredFileSizeRange(file, options) || + tooManyDeletes(file, options) || + tooHighDeleteRatio(file, options) + + private def isWorthRewriting( + group: Seq[CompactionFile], + options: CompactionOptions): Boolean = { + val size = group.foldLeft(0L)(_ + _.length) + val enoughInputFiles = group.size > 1 && group.size >= options.minInputFiles + val enoughContent = group.size > 1 && size > options.targetFileSize + val tooMuchContent = size > options.maxFileSize + enoughInputFiles || enoughContent || tooMuchContent || + group.exists(tooManyDeletes(_, options)) || + group.exists(tooHighDeleteRatio(_, options)) + } + + private def outsideDesiredFileSizeRange(file: CompactionFile, options: CompactionOptions) = + file.length < options.minFileSize || file.length > options.maxFileSize + + private def tooManyDeletes(file: CompactionFile, options: CompactionOptions): Boolean = + file.deleteFileCount >= options.deleteFileThreshold + + private def tooHighDeleteRatio(file: CompactionFile, options: CompactionOptions): Boolean = { + if (file.deleteFileCount == 0) { + false + } else { + val deletedRecords = math.min(file.fileScopedDeletedRows, file.recordCount).toDouble + // A data file with no records divides to NaN, which compares false. + deletedRecords / file.recordCount >= options.deleteRatioThreshold + } + } + + /** + * The number of files a group should be rewritten into. Rounds the last, partial file into the + * others when that keeps the average within 10 percent of the target, so a 10.1 GB group with a + * 1 GB target gives ten files and not eleven. + */ + private[compaction] def expectedOutputFiles( + inputSize: Long, + options: CompactionOptions): Int = { + val target = options.targetFileSize + if (inputSize < target) { + 1 + } else { + val withoutRemainder = inputSize / target + val withRemainder = if (inputSize % target == 0) withoutRemainder else withoutRemainder + 1 + val averageWithoutRemainder = inputSize / withoutRemainder + if (inputSize % target > options.minFileSize) { + withRemainder.toInt + } else if (averageWithoutRemainder < + math.min(1.1 * target, options.writeMaxFileSize.toDouble)) { + withoutRemainder.toInt + } else { + withRemainder.toInt + } + } + } + + /** + * How many bytes one read task takes: the input divided by the expected file count, but at + * least the target size and at most the size the writer rolls at. + */ + private[compaction] def inputSplitSize(inputSize: Long, options: CompactionOptions): Long = { + val estimated = inputSize / expectedOutputFiles(inputSize, options) + SplitOverhead + if (estimated < options.targetFileSize) { + options.targetFileSize + } else { + math.min(estimated, options.writeMaxFileSize) + } + } + + private def orderGroups(groups: Seq[FileGroup], order: RewriteJobOrder): Seq[FileGroup] = + order match { + case RewriteJobOrder.BytesAsc => groups.sortBy(_.inputBytes) + case RewriteJobOrder.BytesDesc => groups.sortBy(_.inputBytes)(Ordering[Long].reverse) + case RewriteJobOrder.FilesAsc => groups.sortBy(_.numFiles) + case RewriteJobOrder.FilesDesc => groups.sortBy(_.numFiles)(Ordering[Int].reverse) + case RewriteJobOrder.NoOrder => groups + } + + /** + * Iceberg's `BinPacking.PackingIterator`. Items stay in input order. Each goes into the first + * open bin it fits in, or opens a new one. When more than `lookback` bins are open, the oldest + * (or the heaviest, with `largestBinFirst`) is closed and emitted. An item heavier than the + * target gets a bin of its own. + */ + private[compaction] def pack[T]( + items: Seq[T], + targetWeight: Long, + lookback: Int, + largestBinFirst: Boolean)(weightOf: T => Long): Seq[Seq[T]] = { + require(lookback > 0, s"Bin look-back size must be greater than 0: $lookback") + + final class Bin { + val members = mutable.ArrayBuffer.empty[T] + var weight = 0L + def canAdd(w: Long): Boolean = weight + w <= targetWeight + def add(item: T, w: Long): Unit = { + weight += w + members.append(item) + } + } + + val out = mutable.ArrayBuffer.empty[Seq[T]] + val bins = new java.util.LinkedList[Bin]() + + def takeBin(): Bin = { + if (largestBinFirst) { + // The first bin with the largest weight, as Collections.max picks it. + var largest = bins.getFirst + val iter = bins.iterator() + while (iter.hasNext) { + val bin = iter.next() + if (bin.weight > largest.weight) largest = bin + } + bins.remove(largest) + largest + } else { + bins.removeFirst() + } + } + + items.foreach { item => + val weight = weightOf(item) + var target: Bin = null + val iter = bins.iterator() + while (target == null && iter.hasNext) { + val bin = iter.next() + if (bin.canAdd(weight)) target = bin + } + if (target != null) { + target.add(item, weight) + } else { + val bin = new Bin + bin.add(item, weight) + bins.addLast(bin) + if (bins.size() > lookback) { + out.append(takeBin().members.toVector) + } + } + } + while (!bins.isEmpty) { + out.append(bins.removeFirst().members.toVector) + } + out.toSeq + } +} diff --git a/spark/src/main/scala/org/apache/comet/iceberg/compaction/CometIcebergProcedureRule.scala b/spark/src/main/scala/org/apache/comet/iceberg/compaction/CometIcebergProcedureRule.scala new file mode 100644 index 00000000000..cebebb75e67 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/iceberg/compaction/CometIcebergProcedureRule.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule + +/** + * Hands Iceberg's `rewrite_data_files` procedure to Comet when `spark.comet.iceberg.compaction + * .enabled` is on. See [[CometProcedureShim]] for how the procedure is intercepted on each Spark + * version. + * + * Spark optimizes the plan of a `CALL` before it invokes the procedure, so this is an optimizer + * rule. A resolution rule would be too late, because Spark resolves, binds and invokes the + * procedure in one pass. + * + * The rule runs on every optimized plan, so it looks only at the root, where a `CALL` always is. + */ +case class CometIcebergProcedureRule(session: SparkSession) extends Rule[LogicalPlan] { + + override def apply(plan: LogicalPlan): LogicalPlan = CometProcedureShim.wrap(session, plan) +} diff --git a/spark/src/main/scala/org/apache/comet/iceberg/compaction/CometRewriteDataFiles.scala b/spark/src/main/scala/org/apache/comet/iceberg/compaction/CometRewriteDataFiles.scala new file mode 100644 index 00000000000..922748d2f08 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/iceberg/compaction/CometRewriteDataFiles.scala @@ -0,0 +1,704 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import java.util.UUID +import java.util.concurrent.{ConcurrentLinkedQueue, Executors, ExecutorService, ThreadFactory} +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} + +import scala.jdk.CollectionConverters._ +import scala.util.control.NonFatal + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.comet.CometIcebergNativeScanExec +import org.apache.spark.sql.types.StructType + +import org.apache.comet.CometConf +import org.apache.comet.iceberg.IcebergReflection + +/** + * Comet's bin-pack `rewrite_data_files`. It follows iceberg-java's `RewriteDataFilesSparkAction` + * step for step, so that a call plans the same groups either way: + * + * 1. Plan the data files of the current snapshot into file groups ([[BinPackPlanner]]). + * 1. Rewrite the groups in Spark jobs, which read the files with Comet's native Iceberg scan + * and write them back with Iceberg's rewrite write, which Comet runs with its native Iceberg + * writer when `spark.comet.write.iceberg.enabled` is set. Up to + * `max-concurrent-file-group-rewrites` jobs run at once. + * 1. Commit the rewritten groups as one `RewriteFiles` snapshot, or in several snapshots with + * `partial-progress.enabled`. + * + * Iceberg runs a job per group. Each job reads the group's files through Iceberg's staged scan, + * which packs them into Spark partitions of about the target file size. A table with many + * partitions has many small groups, so only a few Spark tasks run at once. A job here covers + * every group of a commit, with one Spark partition per bin the planner chose, so all of the + * commit's bins run together. A group that cannot be read this way is rewritten as Iceberg does. + * + * The data movement stays in Iceberg's write path on purpose. It carries everything that must + * match iceberg-java (metrics, partition values, sort order ids, file naming), and the + * eligibility gates that fall back to Iceberg's own writer when Comet cannot match it. + */ +object CometRewriteDataFiles extends Logging { + + /** What a `CALL` hands to the rewrite. */ + final case class Request( + session: SparkSession, + table: AnyRef, + options: Map[String, String], + filter: Option[AnyRef], + refreshCache: () => Unit) + + /** The columns of the procedure's result row. */ + final case class Result( + rewrittenDataFilesCount: Int, + addedDataFilesCount: Int, + rewrittenBytesCount: Long, + failedDataFilesCount: Int, + removedDeleteFilesCount: Int) + + object Result { + val Empty: Result = Result(0, 0, 0L, 0, 0) + } + + /** + * Plans a rewrite without changing anything. Returns the reason when Comet leaves the call to + * Iceberg, which the caller must then run instead. + */ + def prepare(request: Request): Either[String, PreparedRewrite] = { + val table = request.table + IcebergRewriteReflection.currentSnapshotId(table) match { + case None => + // An empty table has nothing to rewrite, and Iceberg does not look at the options. + Right(new PreparedRewrite(request, 0L, None, CompactionPlan(Nil, 0))) + case Some(startingSnapshotId) => + val specId = IcebergRewriteReflection.currentSpecId(table) + CompactionOptions.parse( + request.options, + IcebergRewriteReflection.tableProperties(table), + specId) match { + case Left(reason) => Left(reason) + case Right(options) => + val files = IcebergRewriteReflection.planFiles( + request.session, + table, + startingSnapshotId, + request.filter, + specId) + val plan = BinPackPlanner.plan(files, options) + Right(new PreparedRewrite(request, startingSnapshotId, Some(options), plan)) + } + } + } + + /** Groups whose files have been rewritten and not yet committed. */ + private[compaction] final case class RewrittenGroups( + groups: Seq[FileGroup], + addedFiles: Seq[AnyRef], + danglingDeletionVectors: Seq[AnyRef]) + + /** One Spark job of a rewrite. */ + private sealed trait RewriteJob { + def groups: Seq[FileGroup] + } + + /** A single group, read through Iceberg's staged scan the way Iceberg's action reads it. */ + private final case class StagedJob(group: FileGroup) extends RewriteJob { + override def groups: Seq[FileGroup] = Seq(group) + } + + /** + * Several groups, read through [[CompactionSource]] with one Spark partition per bin. + * + * @param scan + * the DataFrame that reads the bins. It is planned already, and Comet's native scan has taken + * it over. + */ + private final case class BinsJob( + groups: Seq[FileGroup], + batchId: String, + scan: DataFrame, + binCount: Int) + extends RewriteJob + + /** What a bins job needs from the table, which does not change between jobs. */ + private final case class BinsSupport( + icebergSchema: AnyRef, + sparkSchema: StructType, + lookback: Int) + + /** + * The most Spark partitions one bins job reads. Every task returns its commit message to the + * driver, which holds them all until the job ends, and `spark.driver.maxResultSize` limits what + * they add up to. A table with wide statistics has messages of tens of kilobytes, so this keeps + * a job well under the default limit of 1 GiB. + */ + private[compaction] val MaxBinsPerJob = 10000 + + /** The default executor, daemon threads so a stuck job never keeps the JVM alive. */ + private def newPool(size: Int): ExecutorService = + Executors.newFixedThreadPool( + math.max(1, size), + new ThreadFactory { + private val counter = new AtomicInteger(0) + override def newThread(runnable: Runnable): Thread = { + val thread = new Thread(runnable, s"Comet-Rewrite-Service-${counter.getAndIncrement()}") + thread.setDaemon(true) + thread + } + }) + + /** The Spark job group id prefix of the jobs that rewrite file groups. */ + val JobGroupPrefix = "COMET-REWRITE-DATA-FILES" + + private val jobCounter = new AtomicInteger(0) + + /** + * A rewrite that has been planned and is ready to run. + * + * @param snapshotId + * the snapshot the plan was read from, which commits validate against. Unused when the table + * has no snapshot. + * @param options + * None only when the table has no snapshot + */ + final class PreparedRewrite private[compaction] ( + request: Request, + snapshotId: Long, + options: Option[CompactionOptions], + plan: CompactionPlan) { + + private val table = request.table + + /** The batches registered for bins jobs, which are released when the rewrite ends. */ + private val registeredBatches = new ConcurrentLinkedQueue[String]() + + def groupCount: Int = plan.groups.size + + def execute(): Result = { + val result = options match { + case Some(opts) if plan.groups.nonEmpty => + val work = configuredSession() + try { + if (opts.partialProgress) { + executeWithPartialProgress(work, opts) + } else { + executeAtomic(work, opts) + } + } finally { + releaseBatches() + } + case _ => + logInfo(s"Nothing found to rewrite in ${IcebergRewriteReflection.tableName(table)}") + Result.Empty + } + request.refreshCache() + result + } + + /** + * A copy of the session for the rewrite's own jobs. Adaptive execution is off because it can + * change the output partitioning of the write, and delete files are not cached on executors + * because every partition is rewritten once. Iceberg's action does the same. + */ + private def configuredSession(): SparkSession = { + val work = request.session.getClass + .getMethod("cloneSession") + .invoke(request.session) + .asInstanceOf[SparkSession] + work.conf.set("spark.sql.adaptive.enabled", "false") + work.conf.set("spark.sql.iceberg.executor-cache.delete-files.enabled", "false") + work + } + + // -- Atomic ---------------------------------------------------------------------------- + + private def executeAtomic(work: SparkSession, opts: CompactionOptions): Result = { + val jobs = planJobs(work, plan.groups, groupsPerJob = Int.MaxValue) + val pool = newPool(math.min(opts.maxConcurrentGroupRewrites, jobs.size)) + val rewritten = new ConcurrentLinkedQueue[RewrittenGroups]() + val failure = new AtomicReference[Throwable](null) + try { + val futures = jobs.map { job => + pool.submit(new Runnable { + override def run(): Unit = { + // Once a job has failed, jobs that have not started are skipped. + if (failure.get() == null) { + try { + val _ = rewritten.add(runJob(work, job)) + } catch { + case e: Throwable => + logWarning(s"Failure during rewrite process for ${describe(job)}", e) + val _ = failure.compareAndSet(null, e) + } + } + } + }) + } + futures.foreach(_.get()) + } finally { + pool.shutdown() + } + + val completed = rewritten.asScala.toSeq + if (failure.get() != null) { + logError( + s"Cannot complete rewrite, ${CompactionOptions.PartialProgressEnabled} is not " + + "enabled and one of the file groups failed to be rewritten. This error occurred " + + "during the writing of new files, not during the commit process. Cleaning up " + + s"${completed.map(_.groups.size).sum} groups which finished being written.") + completed.foreach(abort) + throw failure.get() + } + + try { + commitOrClean(completed, opts) + } catch { + case e: Exception if IcebergRewriteReflection.isCommitConflict(e) => + throw new RuntimeException( + "Cannot commit rewrite because of a ValidationException or CommitFailedException. " + + "This usually means that this rewrite has conflicted with another concurrent " + + "Iceberg operation. To reduce the likelihood of conflicts, set " + + s"${CompactionOptions.PartialProgressEnabled} which will break up the rewrite " + + s"into multiple smaller commits controlled by " + + s"${CompactionOptions.PartialProgressMaxCommits}. Separate smaller rewrite " + + "commits can succeed independently while any commits that conflict with " + + "another Iceberg operation will be ignored. This mode will create additional " + + "snapshots in the table history, one for each commit.", + e) + } + resultOf(completed, failedFiles = 0) + } + + // -- Partial progress ------------------------------------------------------------------ + + private def executeWithPartialProgress( + work: SparkSession, + opts: CompactionOptions): Result = { + val groupsPerCommit = math.max(1, ceilDiv(plan.totalGroupCount, opts.maxCommits)) + // A job covers at most the groups of one commit, so that a failed job fails one commit. + val jobs = planJobs(work, plan.groups, groupsPerJob = groupsPerCommit) + val pool = newPool(math.min(opts.maxConcurrentGroupRewrites, jobs.size)) + val committer = new BatchCommitter(groupsPerCommit, batch => commitOrClean(batch, opts)) + val failedFiles = new AtomicInteger(0) + try { + val futures = jobs.map { job => + pool.submit(new Runnable { + override def run(): Unit = { + try { + committer.offer(runJob(work, job)) + } catch { + case e: Exception => + logError(s"Failure during rewrite of ${describe(job)}", e) + val _ = failedFiles.addAndGet(job.groups.map(_.numFiles).sum) + } + } + }) + } + futures.foreach(_.get()) + } finally { + pool.shutdown() + } + committer.finish() + + val totalCommits = math.min(plan.totalGroupCount, opts.maxCommits) + val failedCommits = totalCommits - committer.succeededCommits + if (failedCommits > 0 && failedCommits <= opts.maxFailedCommits) { + logWarning( + s"${CompactionOptions.PartialProgressEnabled} is true but $failedCommits rewrite " + + "commits failed. Check the logs to determine why the individual commits failed. " + + s"If this is persistent it may help to increase " + + s"${CompactionOptions.PartialProgressMaxCommits} which will split the rewrite " + + "operation into smaller commits.") + } else if (failedCommits > opts.maxFailedCommits) { + throw new RuntimeException( + s"${CompactionOptions.PartialProgressEnabled} is true but $failedCommits rewrite " + + "commits failed. This is more than the maximum allowed failures of " + + s"${opts.maxFailedCommits}. Check the logs to determine why the individual " + + "commits failed. If this is persistent it may help to increase " + + s"${CompactionOptions.PartialProgressMaxCommits} which will split the rewrite " + + "operation into smaller commits.") + } + resultOf(committer.committedGroups, failedFiles.get()) + } + + // -- Planning the jobs ----------------------------------------------------------------- + + /** + * Decides how the groups are rewritten. Groups that can be read as bins are collected into + * jobs of at most `groupsPerJob` groups and [[MaxBinsPerJob]] bins, and each other group gets + * a job of its own. Nothing is written here, so a bins job that cannot run is replaced by its + * groups' own jobs. + */ + private def planJobs( + work: SparkSession, + groups: Seq[FileGroup], + groupsPerJob: Int): Seq[RewriteJob] = { + binsSupport(work) match { + case None => groups.map(StagedJob) + case Some(support) => + // A group whose files are not in the output spec is repartitioned by the write, which + // the bins job does not do. + val (direct, repartitioned) = + groups.partition(group => group.files.head.specId == group.outputSpecId) + // A group whose tasks cannot be split and packed here is left to Iceberg's staged scan. + val (planned, unplanned) = + direct.foldLeft((Vector.empty[(FileGroup, Seq[BinTasks])], Vector.empty[FileGroup])) { + case ((ok, failed), group) => + try { + (ok :+ (group -> binsOf(group, support.lookback)), failed) + } catch { + case NonFatal(e) => + logInfo(s"Comet cannot plan the read of ${describe(group)}: ${e.getMessage}") + (ok, failed :+ group) + } + } + val jobs = + BinPackPlanner.chunk(planned, groupsPerJob, MaxBinsPerJob)(_._2.size).flatMap { + chunked => + binsJob(work, support, chunked) match { + case Some(job) => Seq(job) + case None => chunked.map { case (group, _) => StagedJob(group) } + } + } + jobs ++ (unplanned ++ repartitioned).map(StagedJob) + } + } + + /** + * What a bins job needs, or None when the table or the session cannot use one. A table with + * row lineage reads and writes the lineage columns in a rewrite, which only Iceberg's own + * staged scan provides. + */ + private def binsSupport(work: SparkSession): Option[BinsSupport] = { + val conf = work.sessionState.conf + if (!CometConf.COMET_ICEBERG_COMPACTION_SINGLE_JOB_ENABLED.get(conf) || + !CometConf.COMET_ICEBERG_NATIVE_ENABLED.get(conf) || + !CometConf.COMET_EXEC_ENABLED.get(conf)) { + None + } else { + try { + if (IcebergReflection.getFormatVersion(table).exists(_ < 3)) { + val lookback = IcebergRewriteReflection + .tableProperties(table) + .get("read.split.planning-lookback") + .map(_.toInt) + .getOrElse(BinPackPlanner.DefaultSplitLookback) + val schema = IcebergReflection.getSchema(table).map(_.asInstanceOf[AnyRef]) + if (lookback > 0 && schema.isDefined) { + Some(BinsSupport(schema.get, IcebergReflection.toSparkSchema(schema.get), lookback)) + } else { + None + } + } else { + None + } + } catch { + case NonFatal(e) => + logInfo(s"Comet rewrites each file group in its own job: ${e.getMessage}") + None + } + } + } + + /** + * The bins of a group: its large files split into read tasks, packed into Spark partitions of + * about `inputSplitSize`, which is how Iceberg's staged scan plans the read. + */ + private def binsOf(group: FileGroup, lookback: Int): Seq[BinTasks] = { + val readTasks = + IcebergRewriteReflection.splitTasks(group.files.map(_.task), group.inputSplitSize) + BinPackPlanner + .planReadTasks(readTasks, group.inputSplitSize, lookback, openFileCost = 0L)( + _.sizeBytes, + _.filesCount) + .map { bin => + val merged = IcebergRewriteReflection.mergeTasks(bin.map(_.task)) + new BinTasks(new java.util.ArrayList[AnyRef](merged.asJava)) + } + } + + /** + * A job that reads the bins of `chunked`, or None when Comet's native scan does not take over + * the read. Spark cannot read a [[CompactionScan]], so the plan is checked before the job is + * committed to. The scan is planned here, which changes nothing. + */ + private def binsJob( + work: SparkSession, + support: BinsSupport, + chunked: Seq[(FileGroup, Seq[BinTasks])]): Option[BinsJob] = { + val batchId = UUID.randomUUID().toString + val batch = new CompactionBatch( + table, + support.icebergSchema, + support.sparkSchema, + chunked.flatMap(_._2)) + CompactionBatches.register(batchId, batch) + registeredBatches.add(batchId) + val scan = + try { + val df = work.read + .format(classOf[CompactionSource].getName) + .option(CompactionSource.BatchIdOption, batchId) + .load() + if (df.queryExecution.executedPlan.exists(_.isInstanceOf[CometIcebergNativeScanExec])) { + Some(df) + } else { + logInfo( + "Comet's native Iceberg scan does not read these file groups, so each is " + + "rewritten in its own job") + None + } + } catch { + case NonFatal(e) => + logInfo(s"Comet cannot plan one job for these file groups: ${e.getMessage}") + None + } + scan match { + case Some(df) => Some(BinsJob(chunked.map(_._1), batchId, df, batch.bins.size)) + case None => + CompactionBatches.remove(batchId) + None + } + } + + private def releaseBatches(): Unit = + registeredBatches.asScala.foreach { batchId => + CompactionBatches.remove(batchId) + IcebergRewriteReflection.release(table, batchId) + } + + // -- Running a job --------------------------------------------------------------------- + + private def runJob(work: SparkSession, job: RewriteJob): RewrittenGroups = job match { + case StagedJob(group) => rewriteStaged(work, group) + case bins: BinsJob => rewriteBins(work, bins) + } + + /** + * Rewrites one group, which is what iceberg-java's bin-pack runner does: read the group's + * files in splits of `inputSplitSize`, and write them back so that each split becomes a new + * file. A shuffle is added only when the files are not in the output spec. + */ + private def rewriteStaged(work: SparkSession, group: FileGroup): RewrittenGroups = { + val groupId = UUID.randomUUID().toString + val description = s"Rewriting ${group.numFiles} files (BIN-PACK, Comet, file group " + + s"${group.globalIndex}/${plan.totalGroupCount}) in " + + IcebergRewriteReflection.tableName(table) + withJobGroup(work, description) { + try { + val tasks = new java.util.ArrayList[AnyRef](group.numFiles) + group.files.foreach(f => tasks.add(f.task)) + IcebergRewriteReflection.stage(table, groupId, tasks) + + val repartition = group.files.head.specId != group.outputSpecId + work.read + .format("iceberg") + .option("scan-task-set-id", groupId) + .option("split-size", group.inputSplitSize.toString) + .option("file-open-cost", "0") + .load(groupId) + .write + .format("iceberg") + .option("rewritten-file-scan-task-set-id", groupId) + .option("target-file-size-bytes", group.maxOutputFileSize.toString) + .option("distribution-mode", if (repartition) "range" else "none") + .option("output-spec-id", group.outputSpecId.toString) + .mode("append") + .save(groupId) + + val added = IcebergRewriteReflection.newFiles(table, groupId) + val dvs = IcebergRewriteReflection.deletionVectors(group.files.map(_.task)) + logInfo(s"Rewrite files ready to be committed - $description") + RewrittenGroups(Seq(group), added, dvs) + } finally { + IcebergRewriteReflection.release(table, groupId) + } + } + } + + /** + * Rewrites the groups of a bins job in one Spark job. Every Spark task reads one bin and + * writes it back, so a bin of about the target file size becomes about one file. The groups + * share the output settings, because those come from the options and not from the group. + */ + private def rewriteBins(work: SparkSession, job: BinsJob): RewrittenGroups = { + val first = job.groups.head + val description = s"Rewriting ${job.groups.map(_.numFiles).sum} files (BIN-PACK, Comet, " + + s"${job.groups.size} file groups, ${job.binCount} read tasks) in " + + IcebergRewriteReflection.tableName(table) + withJobGroup(work, description) { + try { + IcebergRewriteReflection.cacheTable(table, job.batchId) + job.scan.write + .format("iceberg") + .option("rewritten-file-scan-task-set-id", job.batchId) + .option("target-file-size-bytes", first.maxOutputFileSize.toString) + .option("distribution-mode", "none") + .option("output-spec-id", first.outputSpecId.toString) + .mode("append") + .save(job.batchId) + + val added = IcebergRewriteReflection.newFiles(table, job.batchId) + val dvs = + IcebergRewriteReflection.deletionVectors(job.groups.flatMap(_.files.map(_.task))) + logInfo(s"Rewrite files ready to be committed - $description") + RewrittenGroups(job.groups, added, dvs) + } finally { + IcebergRewriteReflection.release(table, job.batchId) + CompactionBatches.remove(job.batchId) + } + } + } + + /** Runs `body` under a job group of its own, so that the Spark UI shows what it rewrites. */ + private def withJobGroup[T](work: SparkSession, description: String)(body: => T): T = { + val sparkContext = work.sparkContext + // Iceberg names its jobs REWRITE-DATA-FILES, so the prefix shows who ran the rewrite. + sparkContext.setJobGroup( + s"$JobGroupPrefix-${jobCounter.incrementAndGet()}", + description, + interruptOnCancel = false) + try { + body + } finally { + sparkContext.clearJobGroup() + } + } + + // -- Committing ------------------------------------------------------------------------ + + /** Serializes commits, which iceberg-java's partial-progress service does not guarantee. */ + private val commitLock = new Object + + private def commitOrClean(rewritten: Seq[RewrittenGroups], opts: CompactionOptions): Unit = { + try { + commitLock.synchronized { + IcebergRewriteReflection.commitRewrite( + table, + snapshotId, + opts.useStartingSequenceNumber, + rewrittenDataFiles = rewritten.flatMap( + _.groups.flatMap(_.files.map(f => IcebergRewriteReflection.dataFileOf(f.task)))), + addedDataFiles = rewritten.flatMap(_.addedFiles), + danglingDeletionVectors = rewritten.flatMap(_.danglingDeletionVectors)) + } + } catch { + case e: Exception if IcebergRewriteReflection.isCommitStateUnknown(e) => + logError( + "Commit state unknown, cannot clean up files because they may have been " + + "committed successfully.", + e) + throw e + case e: Exception => + if (IcebergRewriteReflection.isCleanableFailure(e)) { + logError("Cannot commit groups, attempting to clean up written files", e) + rewritten.foreach(abort) + } + throw e + } + } + + private def abort(rewritten: RewrittenGroups): Unit = + IcebergRewriteReflection.deleteFiles(table, rewritten.addedFiles) + + private def resultOf(rewritten: Seq[RewrittenGroups], failedFiles: Int): Result = { + val groups = rewritten.flatMap(_.groups) + Result( + rewrittenDataFilesCount = groups.map(_.numFiles).sum, + addedDataFilesCount = + rewritten.map(r => IcebergRewriteReflection.distinctDataFiles(r.addedFiles)).sum, + rewrittenBytesCount = groups.map(_.inputBytes).sum, + failedDataFilesCount = failedFiles, + removedDeleteFilesCount = rewritten + .map(r => IcebergRewriteReflection.distinctDeleteFiles(r.danglingDeletionVectors)) + .sum) + } + + private def describe(job: RewriteJob): String = + job.groups.map(describe).mkString("file groups ", ", ", "") + + private def describe(group: FileGroup): String = + s"{globalIndex=${group.globalIndex}, partitionIndex=${group.partitionIndex}, " + + s"partition=${group.partition.mkString("[", ", ", "]")}}" + + private def ceilDiv(a: Int, b: Int): Int = (a + b - 1) / b + } + + /** + * Commits finished jobs in batches of `groupsPerCommit` groups while the others are still + * running, the way iceberg-java's commit service does for `partial-progress.enabled`. A failed + * batch is logged and dropped, and its files are cleaned up by the commit. + */ + private final class BatchCommitter(groupsPerCommit: Int, commit: Seq[RewrittenGroups] => Unit) { + private val pending = new java.util.ArrayDeque[RewrittenGroups]() + private var pendingGroups = 0 + private val committed = new ConcurrentLinkedQueue[RewrittenGroups]() + private val succeeded = new AtomicInteger(0) + + def succeededCommits: Int = succeeded.get() + + def committedGroups: Seq[RewrittenGroups] = committed.asScala.toSeq + + def offer(rewritten: RewrittenGroups): Unit = { + pending.synchronized { + pending.add(rewritten) + pendingGroups += rewritten.groups.size + } + commitReadyBatches(finalPass = false) + } + + /** Commits what is left once every job has finished. */ + def finish(): Unit = commitReadyBatches(finalPass = true) + + private def commitReadyBatches(finalPass: Boolean): Unit = { + var batch = nextBatch(finalPass) + while (batch.nonEmpty) { + try { + commit(batch) + batch.foreach(committed.add) + succeeded.incrementAndGet() + } catch { + case e: Exception => + logError( + "Failure during rewrite commit process, partial progress enabled. Ignoring", + e) + } + batch = nextBatch(finalPass) + } + } + + private def nextBatch(finalPass: Boolean): Seq[RewrittenGroups] = pending.synchronized { + if (pendingGroups >= groupsPerCommit || (finalPass && !pending.isEmpty)) { + val batch = Seq.newBuilder[RewrittenGroups] + var taken = 0 + while (taken < groupsPerCommit && !pending.isEmpty) { + val next = pending.poll() + pendingGroups -= next.groups.size + batch += next + taken += next.groups.size + } + batch.result() + } else { + Nil + } + } + } +} diff --git a/spark/src/main/scala/org/apache/comet/iceberg/compaction/CompactionOptions.scala b/spark/src/main/scala/org/apache/comet/iceberg/compaction/CompactionOptions.scala new file mode 100644 index 00000000000..04c6c89a83d --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/iceberg/compaction/CompactionOptions.scala @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +/** The order Iceberg starts file groups in (`rewrite-job-order`). */ +sealed abstract class RewriteJobOrder(val orderName: String) + +object RewriteJobOrder { + case object BytesAsc extends RewriteJobOrder("bytes-asc") + case object BytesDesc extends RewriteJobOrder("bytes-desc") + case object FilesAsc extends RewriteJobOrder("files-asc") + case object FilesDesc extends RewriteJobOrder("files-desc") + case object NoOrder extends RewriteJobOrder("none") + + private val all: Seq[RewriteJobOrder] = Seq(BytesAsc, BytesDesc, FilesAsc, FilesDesc, NoOrder) + + def fromName(name: String): Option[RewriteJobOrder] = + all.find(_.orderName.equalsIgnoreCase(name)) +} + +/** + * The options of a bin-pack `rewrite_data_files` call, with Iceberg's defaults applied. + * + * Only options Comet reproduces are accepted. [[CompactionOptions.parse]] reports everything + * else, including a value Iceberg would reject, so that the call falls back to Iceberg's + * procedure and the user sees Iceberg's own error. + */ +final case class CompactionOptions( + targetFileSize: Long, + minFileSize: Long, + maxFileSize: Long, + minInputFiles: Int, + rewriteAll: Boolean, + maxGroupSize: Long, + outputSpecId: Int, + deleteFileThreshold: Int, + deleteRatioThreshold: Double, + jobOrder: RewriteJobOrder, + maxFilesToRewrite: Option[Int], + maxConcurrentGroupRewrites: Int, + partialProgress: Boolean, + maxCommits: Int, + maxFailedCommits: Int, + useStartingSequenceNumber: Boolean) { + + /** + * The size the writer rolls files at. It sits halfway between the target and the maximum size, + * so that data compressing a little worse than the estimate does not leave a tiny extra file. + */ + def writeMaxFileSize: Long = (targetFileSize + ((maxFileSize - targetFileSize) * 0.5)).toLong +} + +object CompactionOptions { + val TargetFileSizeBytes = "target-file-size-bytes" + val MinFileSizeBytes = "min-file-size-bytes" + val MaxFileSizeBytes = "max-file-size-bytes" + val MinInputFiles = "min-input-files" + val RewriteAll = "rewrite-all" + val MaxFileGroupSizeBytes = "max-file-group-size-bytes" + val DeleteFileThreshold = "delete-file-threshold" + val DeleteRatioThreshold = "delete-ratio-threshold" + val JobOrder = "rewrite-job-order" + val MaxFilesToRewrite = "max-files-to-rewrite" + val MaxConcurrentFileGroupRewrites = "max-concurrent-file-group-rewrites" + val PartialProgressEnabled = "partial-progress.enabled" + val PartialProgressMaxCommits = "partial-progress.max-commits" + val PartialProgressMaxFailedCommits = "partial-progress.max-failed-commits" + val UseStartingSequenceNumber = "use-starting-sequence-number" + val RemoveDanglingDeletes = "remove-dangling-deletes" + val OutputSpecId = "output-spec-id" + + /** The table property that sets the default target file size. */ + val WriteTargetFileSizeBytes = "write.target-file-size-bytes" + + /** Iceberg's `TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT`, 512 MiB. */ + val WriteTargetFileSizeBytesDefault: Long = 512L * 1024 * 1024 + + private val MaxFileGroupSizeBytesDefault = 100L * 1024 * 1024 * 1024 + private val MinInputFilesDefault = 5 + private val MaxConcurrentFileGroupRewritesDefault = 5 + private val PartialProgressMaxCommitsDefault = 10 + + private val supportedKeys: Set[String] = Set( + TargetFileSizeBytes, + MinFileSizeBytes, + MaxFileSizeBytes, + MinInputFiles, + RewriteAll, + MaxFileGroupSizeBytes, + DeleteFileThreshold, + DeleteRatioThreshold, + JobOrder, + MaxFilesToRewrite, + MaxConcurrentFileGroupRewrites, + PartialProgressEnabled, + PartialProgressMaxCommits, + PartialProgressMaxFailedCommits, + UseStartingSequenceNumber, + RemoveDanglingDeletes, + OutputSpecId) + + /** + * Parses the `options` map of a `CALL`. Returns the reason when Comet should leave the call to + * Iceberg. That covers an option Comet does not reproduce, a value Iceberg would reject, and a + * combination Iceberg refuses. + * + * @param options + * the `options` argument + * @param tableProperties + * the table's properties, for the default target file size + * @param currentSpecId + * the table's current partition spec id, the default and only supported `output-spec-id` + */ + def parse( + options: Map[String, String], + tableProperties: Map[String, String], + currentSpecId: Int): Either[String, CompactionOptions] = { + val unsupported = options.keySet.diff(supportedKeys) + if (unsupported.nonEmpty) { + return Left(s"unsupported option(s): ${unsupported.toSeq.sorted.mkString(", ")}") + } + try { + // Iceberg parses booleans with Boolean.parseBoolean, so anything but "true" is false. + def bool(key: String, default: Boolean): Boolean = + options.get(key).map(java.lang.Boolean.parseBoolean).getOrElse(default) + + val defaultTarget = tableProperties + .get(WriteTargetFileSizeBytes) + .map(_.toLong) + .getOrElse(WriteTargetFileSizeBytesDefault) + val target = options.get(TargetFileSizeBytes).map(_.toLong).getOrElse(defaultTarget) + val min = options.get(MinFileSizeBytes).map(_.toLong).getOrElse((target * 0.75).toLong) + val max = options.get(MaxFileSizeBytes).map(_.toLong).getOrElse((target * 1.80).toLong) + require(target > 0, s"'$TargetFileSizeBytes' must be > 0") + require(min >= 0, s"'$MinFileSizeBytes' must be >= 0") + require(target > min, s"'$TargetFileSizeBytes' must be > '$MinFileSizeBytes'") + require(target < max, s"'$TargetFileSizeBytes' must be < '$MaxFileSizeBytes'") + + val minInputFiles = options.get(MinInputFiles).map(_.toInt).getOrElse(MinInputFilesDefault) + require(minInputFiles > 0, s"'$MinInputFiles' must be > 0") + + val maxGroupSize = + options.get(MaxFileGroupSizeBytes).map(_.toLong).getOrElse(MaxFileGroupSizeBytesDefault) + require(maxGroupSize > 0, s"'$MaxFileGroupSizeBytes' must be > 0") + + val deleteFileThreshold = + options.get(DeleteFileThreshold).map(_.toInt).getOrElse(Integer.MAX_VALUE) + require(deleteFileThreshold >= 0, s"'$DeleteFileThreshold' must be >= 0") + + val deleteRatioThreshold = options.get(DeleteRatioThreshold).map(_.toDouble).getOrElse(0.3) + require( + deleteRatioThreshold > 0 && deleteRatioThreshold <= 1, + s"'$DeleteRatioThreshold' must be in (0, 1]") + + val jobOrder = options + .get(JobOrder) + .map(name => + RewriteJobOrder + .fromName(name) + .getOrElse(throw new IllegalArgumentException(s"invalid $JobOrder: $name"))) + .getOrElse(RewriteJobOrder.NoOrder) + + val maxFilesToRewrite = options.get(MaxFilesToRewrite).map(_.toInt) + require(maxFilesToRewrite.forall(_ > 0), s"'$MaxFilesToRewrite' must be positive") + + val maxConcurrent = options + .get(MaxConcurrentFileGroupRewrites) + .map(_.toInt) + .getOrElse(MaxConcurrentFileGroupRewritesDefault) + require(maxConcurrent >= 1, s"'$MaxConcurrentFileGroupRewrites' must be positive") + + val partialProgress = bool(PartialProgressEnabled, default = false) + val maxCommits = options + .get(PartialProgressMaxCommits) + .map(_.toInt) + .getOrElse(PartialProgressMaxCommitsDefault) + val maxFailedCommits = + options.get(PartialProgressMaxFailedCommits).map(_.toInt).getOrElse(maxCommits) + require( + !partialProgress || maxCommits > 0, + s"'$PartialProgressMaxCommits' must be positive when '$PartialProgressEnabled' is true") + + require( + !bool(RemoveDanglingDeletes, default = false), + s"'$RemoveDanglingDeletes' is not supported") + val outputSpecId = options.get(OutputSpecId).map(_.toInt).getOrElse(currentSpecId) + require(outputSpecId == currentSpecId, s"'$OutputSpecId' other than the current spec") + + Right( + CompactionOptions( + targetFileSize = target, + minFileSize = min, + maxFileSize = max, + minInputFiles = minInputFiles, + rewriteAll = bool(RewriteAll, default = false), + maxGroupSize = maxGroupSize, + outputSpecId = outputSpecId, + deleteFileThreshold = deleteFileThreshold, + deleteRatioThreshold = deleteRatioThreshold, + jobOrder = jobOrder, + maxFilesToRewrite = maxFilesToRewrite, + maxConcurrentGroupRewrites = maxConcurrent, + partialProgress = partialProgress, + maxCommits = maxCommits, + maxFailedCommits = maxFailedCommits, + useStartingSequenceNumber = bool(UseStartingSequenceNumber, default = true))) + } catch { + // NumberFormatException is an IllegalArgumentException, as is a failed require. + case e: IllegalArgumentException => Left(e.getMessage) + } + } +} diff --git a/spark/src/main/scala/org/apache/comet/iceberg/compaction/CompactionScan.scala b/spark/src/main/scala/org/apache/comet/iceberg/compaction/CompactionScan.scala new file mode 100644 index 00000000000..e4982c3f93b --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/iceberg/compaction/CompactionScan.scala @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import java.util.concurrent.ConcurrentHashMap + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability, TableProvider} +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.connector.read.{Batch, InputPartition, PartitionReader, PartitionReaderFactory, Scan, ScanBuilder} +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +/** + * The read tasks one Spark task of a rewrite reads: a bin of the planner, which is about one + * output file of data. Comet's native Iceberg scan reads the `FileScanTask`s of an input + * partition through a `taskGroup()` that has a `tasks()` method, which is this. + */ +final class BinTasks(val tasks: java.util.List[AnyRef]) extends Serializable + +/** + * What a rewrite job reads: the table, and the bins of its file groups, one per Spark partition. + * + * @param table + * the Iceberg `Table` + * @param icebergSchema + * the Iceberg `Schema` the scan reads, which is the table's + * @param sparkSchema + * the same schema as Spark's `StructType` + * @param bins + * the read tasks of each Spark partition + */ +final class CompactionBatch( + val table: AnyRef, + val icebergSchema: AnyRef, + val sparkSchema: StructType, + val bins: Seq[BinTasks]) { + + /** Every task of every bin, for the scan accessor Comet reads the tasks from. */ + val allTasks: java.util.List[AnyRef] = { + val all = new java.util.ArrayList[AnyRef]() + bins.foreach(bin => all.addAll(bin.tasks)) + all + } +} + +/** + * The batches a driver has registered for [[CompactionSource]] to read. Iceberg keeps the tasks + * of a staged rewrite the same way, in a registry its reader looks them up in by id. + */ +object CompactionBatches { + + private val batches = new ConcurrentHashMap[String, CompactionBatch]() + + def register(id: String, batch: CompactionBatch): Unit = { + batches.put(id, batch) + () + } + + def get(id: String): Option[CompactionBatch] = Option(batches.get(id)) + + def remove(id: String): Unit = { + batches.remove(id) + () + } +} + +/** + * A Spark data source that reads a [[CompactionBatch]], given the id it was registered under as + * the `batch-id` option. + * + * Iceberg reads a staged rewrite through `SparkStagedScan`, which packs the tasks into Spark + * partitions itself, without regard for the file group a task came from. A rewrite that runs + * every group of a commit in one job needs the partitions to be the bins the planner chose, so + * each stays within one group and one partition of the table. This source gives it that, and + * nothing else: it only describes the scan, and Comet's native Iceberg scan reads it. The scan is + * never read by Spark, so the caller checks that Comet takes it over before running anything. + */ +class CompactionSource extends TableProvider { + + override def inferSchema(options: CaseInsensitiveStringMap): StructType = + batchOf(options).sparkSchema + + override def getTable( + schema: StructType, + partitioning: Array[Transform], + properties: java.util.Map[String, String]): Table = + new CompactionTable(batchOf(new CaseInsensitiveStringMap(properties))) + + override def supportsExternalMetadata(): Boolean = false + + private def batchOf(options: CaseInsensitiveStringMap): CompactionBatch = { + val id = options.get(CompactionSource.BatchIdOption) + CompactionBatches + .get(id) + .getOrElse(throw new IllegalStateException(s"No compaction batch is registered as '$id'")) + } +} + +object CompactionSource { + + /** The option that names the registered [[CompactionBatch]]. */ + val BatchIdOption = "batch-id" +} + +private[compaction] final class CompactionTable(batch: CompactionBatch) + extends Table + with SupportsRead { + + override def name(): String = "comet_iceberg_compaction" + + override def schema(): StructType = batch.sparkSchema + + override def capabilities(): java.util.Set[TableCapability] = + java.util.EnumSet.of(TableCapability.BATCH_READ) + + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = + new ScanBuilder { + override def build(): Scan = new CompactionScan(batch) + } +} + +/** + * The scan of a [[CompactionBatch]]. Comet recognizes it by its class name in `CometScanRule` and + * reads the table, schema and tasks off the accessors below, the same ones it reads off Iceberg's + * own scans. + */ +class CompactionScan(private val batch: CompactionBatch) extends Scan with Batch { + + override def readSchema(): StructType = batch.sparkSchema + + override def description(): String = + s"CometIcebergCompactionScan(bins=${batch.bins.size}, tasks=${batch.allTasks.size()})" + + override def toBatch: Batch = this + + private val partitions: Array[InputPartition] = + batch.bins.map(bin => new CompactionInputPartition(bin): InputPartition).toArray + + override def planInputPartitions(): Array[InputPartition] = partitions + + override def createReaderFactory(): PartitionReaderFactory = CompactionReaderFactory + + /** The Iceberg `Table`, read reflectively by Comet. */ + def table(): AnyRef = batch.table + + /** The Iceberg `Schema` the scan reads, read reflectively by Comet. */ + def expectedSchema(): AnyRef = batch.icebergSchema + + /** Every task of the scan, read reflectively by Comet. */ + def tasks(): java.util.List[AnyRef] = batch.allTasks + + override def equals(other: Any): Boolean = other match { + case that: CompactionScan => this.batch eq that.batch + case _ => false + } + + override def hashCode(): Int = System.identityHashCode(batch) +} + +/** + * One Spark partition of a [[CompactionScan]]. Comet's native Iceberg scan reads its tasks + * through `taskGroup()`, the name Iceberg's own input partitions give theirs. + */ +final class CompactionInputPartition(val taskGroup: BinTasks) extends InputPartition + +/** Spark must never read a [[CompactionScan]] itself, because only Comet's native scan can. */ +private object CompactionReaderFactory extends PartitionReaderFactory { + override def createReader(partition: InputPartition): PartitionReader[InternalRow] = + throw new UnsupportedOperationException( + "A compaction scan can only be read by Comet's native Iceberg scan") +} diff --git a/spark/src/main/scala/org/apache/comet/iceberg/compaction/IcebergRewriteReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/compaction/IcebergRewriteReflection.scala new file mode 100644 index 00000000000..90fc26ccc9a --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/iceberg/compaction/IcebergRewriteReflection.scala @@ -0,0 +1,523 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import java.lang.reflect.{InvocationTargetException, Method} +import java.nio.ByteBuffer + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.SparkSession + +import org.apache.comet.iceberg.IcebergReflection +import org.apache.comet.iceberg.IcebergReflection.{getMethod, loadClass} + +/** + * Everything the compaction calls on Iceberg, reached by reflection because Comet does not link + * Iceberg. Each call is one a version of iceberg-java's own `RewriteDataFilesSparkAction` and + * `RewriteDataFilesCommitManager` make, so it exists in every Iceberg release the Spark profile + * that uses it runs against. + * + * The objects passed around are Iceberg's: a `Table`, `DataFile`s, `FileScanTask`s and an + * `Expression`. They are typed `AnyRef` here. + */ +private[compaction] object IcebergRewriteReflection extends Logging { + + private val TABLE = "org.apache.iceberg.Table" + private val SNAPSHOT = "org.apache.iceberg.Snapshot" + private val PARTITION_SPEC = "org.apache.iceberg.PartitionSpec" + private val FILE_SCAN_TASK = "org.apache.iceberg.FileScanTask" + private val SCAN_TASK = "org.apache.iceberg.ScanTask" + private val SPLITTABLE_SCAN_TASK = "org.apache.iceberg.SplittableScanTask" + private val DELETE_FILE = "org.apache.iceberg.DeleteFile" + private val CONTENT_FILE = "org.apache.iceberg.ContentFile" + private val STRUCT_LIKE = "org.apache.iceberg.StructLike" + private val TABLE_SCAN = "org.apache.iceberg.TableScan" + private val SCAN = "org.apache.iceberg.Scan" + private val EXPRESSION = "org.apache.iceberg.expressions.Expression" + private val REWRITE_FILES = "org.apache.iceberg.RewriteFiles" + private val PENDING_UPDATE = "org.apache.iceberg.PendingUpdate" + private val FILE_IO = "org.apache.iceberg.io.FileIO" + private val CONTENT_FILE_UTIL = "org.apache.iceberg.util.ContentFileUtil" + private val TABLE_SCAN_UTIL = "org.apache.iceberg.util.TableScanUtil" + private val SCAN_TASK_SET_MANAGER = "org.apache.iceberg.spark.ScanTaskSetManager" + private val SPARK_TABLE_CACHE = "org.apache.iceberg.spark.SparkTableCache" + private val FILE_REWRITE_COORDINATOR = "org.apache.iceberg.spark.FileRewriteCoordinator" + private val SPARK_3_UTIL = "org.apache.iceberg.spark.Spark3Util" + private val SPARK_TABLE = "org.apache.iceberg.spark.source.SparkTable" + private val EXPRESSION_CONVERTER = + "org.apache.spark.sql.execution.datasources.SparkExpressionConverter" + + private def boxedLong(value: Long): AnyRef = java.lang.Long.valueOf(value) + + /** Invokes `method`, and throws what the callee threw instead of the reflection wrapper. */ + private def call(method: Method, target: AnyRef, args: AnyRef*): AnyRef = + try method.invoke(target, args: _*) + catch { + case e: InvocationTargetException if e.getCause != null => throw e.getCause + } + + private def longResult(value: AnyRef): Long = value.asInstanceOf[java.lang.Long].longValue() + + private def intResult(value: AnyRef): Int = value.asInstanceOf[java.lang.Integer].intValue() + + // -- Table facts ------------------------------------------------------------------------ + + def isSparkTable(table: Any): Boolean = + loadClass(SPARK_TABLE).isInstance(table) + + /** The Iceberg `Table` behind a Spark `SparkTable`. */ + def icebergTable(sparkTable: AnyRef): AnyRef = + call(getMethod(loadClass(SPARK_TABLE), "table"), sparkTable) + + def tableName(table: AnyRef): String = + call(getMethod(loadClass(TABLE), "name"), table).asInstanceOf[String] + + def tableProperties(table: AnyRef): Map[String, String] = + call(getMethod(loadClass(TABLE), "properties"), table) + .asInstanceOf[java.util.Map[String, String]] + .asScala + .toMap + + /** The id of the table's current snapshot, or None for a table that has no data yet. */ + def currentSnapshotId(table: AnyRef): Option[Long] = { + val snapshot = call(getMethod(loadClass(TABLE), "currentSnapshot"), table) + if (snapshot == null) { + None + } else { + Some(longResult(call(getMethod(loadClass(SNAPSHOT), "snapshotId"), snapshot))) + } + } + + def snapshotSequenceNumber(table: AnyRef, snapshotId: Long): Long = { + val snapshot = + call( + getMethod(loadClass(TABLE), "snapshot", java.lang.Long.TYPE), + table, + boxedLong(snapshotId)) + longResult(call(getMethod(loadClass(SNAPSHOT), "sequenceNumber"), snapshot)) + } + + private def currentSpec(table: AnyRef): AnyRef = + call(getMethod(loadClass(TABLE), "spec"), table) + + def currentSpecId(table: AnyRef): Int = + intResult(call(getMethod(loadClass(PARTITION_SPEC), "specId"), currentSpec(table))) + + private def currentSpecFieldCount(table: AnyRef): Int = + call(getMethod(loadClass(PARTITION_SPEC), "fields"), currentSpec(table)) + .asInstanceOf[java.util.List[_]] + .size() + + // -- Reading a call's arguments ----------------------------------------------------------- + + /** + * Resolves the `table` argument against the procedure's catalog, as Iceberg's `ProcedureInput` + * does, and returns the identifier when the table belongs to that catalog. + */ + def resolveIdentifier( + session: SparkSession, + catalog: AnyRef, + tableArgument: String): Option[AnyRef] = { + val util = loadClass(SPARK_3_UTIL) + val method = util.getMethods + .find(m => m.getName == "catalogAndIdentifier" && m.getParameterCount == 4) + .getOrElse(throw new NoSuchMethodException(s"$SPARK_3_UTIL.catalogAndIdentifier/4")) + val resolved = + call(method, null, "identifier for parameter 'table'", session, tableArgument, catalog) + val resolvedCatalog = call(getMethod(resolved.getClass, "catalog"), resolved) + if (resolvedCatalog == catalog) { + Some(call(getMethod(resolved.getClass, "identifier"), resolved)) + } else { + None + } + } + + /** Converts the `where` argument to an Iceberg expression, as Iceberg's procedure does. */ + def whereExpression( + session: SparkSession, + catalogName: String, + identifier: AnyRef, + where: String): AnyRef = { + val util = loadClass(SPARK_3_UTIL) + val quote = util.getMethods + .find(m => m.getName == "quotedFullIdentifier" && m.getParameterCount == 2) + .getOrElse(throw new NoSuchMethodException(s"$SPARK_3_UTIL.quotedFullIdentifier/2")) + val name = call(quote, null, catalogName, identifier).asInstanceOf[String] + val converter = loadClass(EXPRESSION_CONVERTER) + val collect = converter.getMethods + .find(m => m.getName == "collectResolvedSparkExpression" && m.getParameterCount == 3) + .getOrElse(throw new NoSuchMethodException(s"$EXPRESSION_CONVERTER.collect/3")) + val convert = converter.getMethods + .find(m => m.getName == "convertToIcebergExpression" && m.getParameterCount == 1) + .getOrElse(throw new NoSuchMethodException(s"$EXPRESSION_CONVERTER.convert/1")) + call(convert, null, call(collect, null, session, name, where)) + } + + // -- Planning ---------------------------------------------------------------------------- + + /** + * Plans the table's data files the way iceberg-java's `BinPackRewriteFilePlanner` does: a scan + * of the starting snapshot with the call's filter, residuals ignored, so that a file is + * selected whole whenever any of its rows may match. + */ + def planFiles( + session: SparkSession, + table: AnyRef, + snapshotId: Long, + filter: Option[AnyRef], + currentSpecId: Int): Seq[CompactionFile] = { + val scanClass = loadClass(SCAN) + val expressionClass = loadClass(EXPRESSION) + var scan = call(getMethod(loadClass(TABLE), "newScan"), table) + filter.foreach { expression => + scan = call(getMethod(scanClass, "filter", expressionClass), scan, expression) + } + val caseSensitive = + java.lang.Boolean.parseBoolean(session.conf.get("spark.sql.caseSensitive")) + scan = call( + getMethod(scanClass, "caseSensitive", java.lang.Boolean.TYPE), + scan, + java.lang.Boolean.valueOf(caseSensitive)) + scan = call(getMethod(scanClass, "ignoreResiduals"), scan) + scan = call( + getMethod(loadClass(TABLE_SCAN), "useSnapshot", java.lang.Long.TYPE), + scan, + boxedLong(snapshotId)) + + val tasks = call(getMethod(scanClass, "planFiles"), scan) + try { + val reader = new FileFacts(currentSpecId, currentSpecFieldCount(table)) + tasks.asInstanceOf[java.lang.Iterable[AnyRef]].asScala.map(reader.read).toVector + } finally { + tasks.asInstanceOf[java.io.Closeable].close() + } + } + + /** Reads the planner's facts off scan tasks, holding the reflective handles it needs. */ + private class FileFacts(currentSpecId: Int, currentSpecFieldCount: Int) { + private val taskClass = loadClass(FILE_SCAN_TASK) + private val contentFileClass = loadClass(CONTENT_FILE) + private val structClass = loadClass(STRUCT_LIKE) + private val fileOf = getMethod(taskClass, "file") + private val lengthOf = getMethod(taskClass, "length") + private val deletesOf = getMethod(taskClass, "deletes") + private val recordCountOf = getMethod(contentFileClass, "recordCount") + private val specIdOf = getMethod(contentFileClass, "specId") + private val partitionOf = getMethod(contentFileClass, "partition") + private val structSize = getMethod(structClass, "size") + private val structGet = + getMethod(structClass, "get", java.lang.Integer.TYPE, classOf[java.lang.Class[_]]) + private val isFileScoped = + getMethod(loadClass(CONTENT_FILE_UTIL), "isFileScoped", loadClass(DELETE_FILE)) + + def read(task: AnyRef): CompactionFile = { + val file = call(fileOf, task) + val specId = intResult(call(specIdOf, file)) + // Files of an older spec cannot be told apart by their partition values, so Iceberg + // groups them together under an all-NULL key. + val partition = + if (specId == currentSpecId) partitionValues(call(partitionOf, file)) + else Seq.fill[Any](currentSpecFieldCount)(null) + val deletes = call(deletesOf, task).asInstanceOf[java.util.List[AnyRef]] + var fileScopedDeletedRows = 0L + if (deletes != null) { + deletes.asScala.foreach { delete => + if (call(isFileScoped, null, delete).asInstanceOf[java.lang.Boolean]) { + fileScopedDeletedRows += longResult(call(recordCountOf, delete)) + } + } + } + CompactionFile( + task = task, + length = longResult(call(lengthOf, task)), + recordCount = longResult(call(recordCountOf, file)), + specId = specId, + partition = partition, + deleteFileCount = if (deletes == null) 0 else deletes.size(), + fileScopedDeletedRows = fileScopedDeletedRows) + } + + private def partitionValues(partition: AnyRef): Seq[Any] = { + val size = intResult(call(structSize, partition)) + (0 until size).map { i => + normalize(call(structGet, partition, Integer.valueOf(i), classOf[java.lang.Object])) + } + } + + /** Gives partition values the equality Iceberg's own partition comparator has. */ + private def normalize(value: AnyRef): Any = value match { + case chars: CharSequence => chars.toString + case bytes: ByteBuffer => + val copy = new Array[Byte](bytes.remaining()) + bytes.duplicate().get(copy) + copy.toSeq + case other => other + } + } + + // -- Read tasks ---------------------------------------------------------------------------- + + /** A task a Spark task reads, with the weight Iceberg packs it by. */ + final case class ReadTask(task: AnyRef, sizeBytes: Long, filesCount: Int) + + /** + * Splits large files into read tasks of about `splitSize`, as Iceberg does when it plans the + * read of a staged group, and reads the weight of each. A Parquet file splits at its row group + * boundaries. A task for a file smaller than the split size stays whole. + */ + def splitTasks(tasks: Seq[AnyRef], splitSize: Long): Seq[ReadTask] = { + val splittableClass = loadClass(SPLITTABLE_SCAN_TASK) + val split = getMethod(splittableClass, "split", java.lang.Long.TYPE) + val scanTaskClass = loadClass(SCAN_TASK) + val sizeBytes = getMethod(scanTaskClass, "sizeBytes") + val filesCount = getMethod(scanTaskClass, "filesCount") + def read(task: AnyRef): ReadTask = + ReadTask(task, longResult(call(sizeBytes, task)), intResult(call(filesCount, task))) + tasks.flatMap { task => + if (splittableClass.isInstance(task)) { + call(split, task, boxedLong(splitSize)) + .asInstanceOf[java.lang.Iterable[AnyRef]] + .asScala + .map(read) + .toVector + } else { + Seq(read(task)) + } + } + } + + /** + * Merges the tasks of a bin that read neighboring ranges of one file into a single task, as + * `TableScanUtil.planTaskGroups` does for each task group it plans. + */ + def mergeTasks(tasks: Seq[AnyRef]): Seq[AnyRef] = { + val merge = getMethod(loadClass(TABLE_SCAN_UTIL), "mergeTasks", classOf[java.util.List[_]]) + call(merge, null, new java.util.ArrayList[AnyRef](tasks.asJava)) + .asInstanceOf[java.util.List[AnyRef]] + .asScala + .toVector + } + + // -- Running a group ----------------------------------------------------------------------- + + /** + * Registers the table under the id a rewrite's write finds it by. A rewrite names its table by + * an id, which Iceberg's cached table catalog resolves. + */ + def cacheTable(table: AnyRef, id: String): Unit = { + val cacheClass = loadClass(SPARK_TABLE_CACHE) + val cache = call(getMethod(cacheClass, "get"), null) + call(getMethod(cacheClass, "add", classOf[String], loadClass(TABLE)), cache, id, table) + () + } + + /** Registers a group's tasks and the table under the id the group's reads and writes use. */ + def stage(table: AnyRef, groupId: String, tasks: java.util.List[AnyRef]): Unit = { + cacheTable(table, groupId) + val manager = call(getMethod(loadClass(SCAN_TASK_SET_MANAGER), "get"), null) + call( + getMethod( + loadClass(SCAN_TASK_SET_MANAGER), + "stageTasks", + loadClass(TABLE), + classOf[String], + classOf[java.util.List[_]]), + manager, + table, + groupId, + tasks) + () + } + + /** + * Releases everything [[stage]] and the group's write registered. It runs in a `finally`, so it + * must not replace the failure that got the group there. + */ + def release(table: AnyRef, groupId: String): Unit = { + try { + val tableClass = loadClass(TABLE) + val cacheClass = loadClass(SPARK_TABLE_CACHE) + val cache = call(getMethod(cacheClass, "get"), null) + val manager = call(getMethod(loadClass(SCAN_TASK_SET_MANAGER), "get"), null) + val coordinator = call(getMethod(loadClass(FILE_REWRITE_COORDINATOR), "get"), null) + call(getMethod(cacheClass, "remove", classOf[String]), cache, groupId) + call( + getMethod(loadClass(SCAN_TASK_SET_MANAGER), "removeTasks", tableClass, classOf[String]), + manager, + table, + groupId) + call( + getMethod( + loadClass(FILE_REWRITE_COORDINATOR), + "clearRewrite", + tableClass, + classOf[String]), + coordinator, + table, + groupId) + () + } catch { + case e: Exception => logWarning(s"Failed to release the staged rewrite $groupId", e) + } + } + + /** The data files a group's write produced. They are not part of the table yet. */ + def newFiles(table: AnyRef, groupId: String): Seq[AnyRef] = { + val coordinatorClass = loadClass(FILE_REWRITE_COORDINATOR) + val coordinator = call(getMethod(coordinatorClass, "get"), null) + val files = call( + getMethod(coordinatorClass, "fetchNewFiles", loadClass(TABLE), classOf[String]), + coordinator, + table, + groupId) + files.asInstanceOf[java.util.Set[AnyRef]].asScala.toVector + } + + /** The deletion vectors of a group's tasks, which are dangling once the group is rewritten. */ + def deletionVectors(tasks: Seq[AnyRef]): Seq[AnyRef] = { + val deletesOf = getMethod(loadClass(FILE_SCAN_TASK), "deletes") + val isDV = getMethod(loadClass(CONTENT_FILE_UTIL), "isDV", loadClass(DELETE_FILE)) + tasks.flatMap { task => + val deletes = call(deletesOf, task).asInstanceOf[java.util.List[AnyRef]] + if (deletes == null) Nil + else deletes.asScala.filter(d => call(isDV, null, d).asInstanceOf[java.lang.Boolean]) + } + } + + def dataFileOf(task: AnyRef): AnyRef = + call(getMethod(loadClass(FILE_SCAN_TASK), "file"), task) + + def fileLocation(file: AnyRef): String = + IcebergReflection + .extractFileLocation(file) + .getOrElse(throw new IllegalStateException(s"Cannot read the location of $file")) + + // -- Committing ---------------------------------------------------------------------------- + + /** + * Commits a rewrite, the way `RewriteDataFilesCommitManager.commitFileGroups` does: replace the + * rewritten data files with the added ones, drop the deletion vectors that targeted them, and + * validate against the snapshot the planning read so that a concurrent change to the rewritten + * files fails the commit. + */ + def commitRewrite( + table: AnyRef, + startingSnapshotId: Long, + useStartingSequenceNumber: Boolean, + rewrittenDataFiles: Seq[AnyRef], + addedDataFiles: Seq[AnyRef], + danglingDeletionVectors: Seq[AnyRef]): Unit = { + val rewriteClass = loadClass(REWRITE_FILES) + var rewrite = call(getMethod(loadClass(TABLE), "newRewrite"), table) + rewrite = call( + getMethod(rewriteClass, "validateFromSnapshot", java.lang.Long.TYPE), + rewrite, + boxedLong(startingSnapshotId)) + if (useStartingSequenceNumber) { + val sequenceNumber = snapshotSequenceNumber(table, startingSnapshotId) + rewrite = call( + getMethod(rewriteClass, "dataSequenceNumber", java.lang.Long.TYPE), + rewrite, + boxedLong(sequenceNumber)) + } + val setClass = classOf[java.util.Set[_]] + call( + getMethod(rewriteClass, "rewriteFiles", setClass, setClass, setClass, setClass), + rewrite, + uniqueDataFiles(rewrittenDataFiles), + uniqueDeleteFiles(danglingDeletionVectors), + uniqueDataFiles(addedDataFiles), + new java.util.LinkedHashSet[AnyRef]()) + call(getMethod(loadClass(PENDING_UPDATE), "commit"), rewrite) + () + } + + /** + * Keeps the first of the files that Iceberg's `DataFileSet` treats as one: data files with the + * same location. + */ + private def uniqueDataFiles(files: Seq[AnyRef]): java.util.Set[AnyRef] = + uniqueBy(files)(fileLocation) + + /** + * Keeps the first of the files that Iceberg's `DeleteFileSet` treats as one. That is the same + * location, offset and length, because the deletion vectors of many data files can share one + * Puffin file and differ only by where in it they are. + */ + private def uniqueDeleteFiles(files: Seq[AnyRef]): java.util.Set[AnyRef] = + uniqueBy(files)(deleteFileKey) + + private def uniqueBy(files: Seq[AnyRef])(key: AnyRef => String): java.util.Set[AnyRef] = { + val byKey = new java.util.LinkedHashMap[String, AnyRef]() + files.foreach(f => byKey.putIfAbsent(key(f), f)) + new java.util.LinkedHashSet[AnyRef](byKey.values()) + } + + /** The identity `DeleteFileSet` gives a delete file, as a string. */ + def deleteFileKey(file: AnyRef): String = { + val deleteFileClass = loadClass(DELETE_FILE) + def optional(name: String): Any = + IcebergReflection.findMethod(deleteFileClass, name).map(m => call(m, file)).orNull + s"${fileLocation(file)}|${optional("contentOffset")}|${optional("contentSizeInBytes")}" + } + + /** How many of the delete files are distinct, by the identity of `DeleteFileSet`. */ + def distinctDeleteFiles(files: Seq[AnyRef]): Int = files.map(deleteFileKey).distinct.size + + /** How many of the data files are distinct, by the identity of `DataFileSet`. */ + def distinctDataFiles(files: Seq[AnyRef]): Int = files.map(fileLocation).distinct.size + + /** Deletes files a failed or abandoned rewrite wrote. Failures are logged, never thrown. */ + def deleteFiles(table: AnyRef, files: Seq[AnyRef]): Unit = { + if (files.nonEmpty) { + try { + val io = call(getMethod(loadClass(TABLE), "io"), table) + val delete = getMethod(loadClass(FILE_IO), "deleteFile", classOf[String]) + files.foreach { file => + try call(delete, io, fileLocation(file)) + catch { + case e: Exception => logWarning(s"Failed to delete rewritten file $file", e) + } + } + } catch { + case e: Exception => logWarning("Failed to clean up files from a failed rewrite", e) + } + } + } + + private def isInstance(className: String, e: Throwable): Boolean = + try loadClass(className).isInstance(e) + catch { case _: ClassNotFoundException => false } + + /** A commit that may have succeeded, so the files it names must be kept. */ + def isCommitStateUnknown(e: Throwable): Boolean = + isInstance("org.apache.iceberg.exceptions.CommitStateUnknownException", e) + + /** A failure after which the files of the failed commit are safe to delete. */ + def isCleanableFailure(e: Throwable): Boolean = + isInstance("org.apache.iceberg.exceptions.CleanableFailure", e) + + /** A conflict with a concurrent operation. */ + def isCommitConflict(e: Throwable): Boolean = + isInstance("org.apache.iceberg.exceptions.ValidationException", e) || + isInstance("org.apache.iceberg.exceptions.CommitFailedException", e) +} diff --git a/spark/src/main/spark-3.x/org/apache/comet/iceberg/compaction/CometProcedureShim.scala b/spark/src/main/spark-3.x/org/apache/comet/iceberg/compaction/CometProcedureShim.scala new file mode 100644 index 00000000000..167f956f629 --- /dev/null +++ b/spark/src/main/spark-3.x/org/apache/comet/iceberg/compaction/CometProcedureShim.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan + +/** + * Procedures are only invoked through `InvokeProcedures` on Spark 4. Spark 3 plans Iceberg's own + * `CALL` node, which Comet does not intercept yet, so there is nothing to swap here. + */ +object CometProcedureShim { + + def wrap(session: SparkSession, plan: LogicalPlan): LogicalPlan = plan +} diff --git a/spark/src/main/spark-4.x/org/apache/comet/iceberg/compaction/CometProcedureShim.scala b/spark/src/main/spark-4.x/org/apache/comet/iceberg/compaction/CometProcedureShim.scala new file mode 100644 index 00000000000..ba6c77f5017 --- /dev/null +++ b/spark/src/main/spark-4.x/org/apache/comet/iceberg/compaction/CometProcedureShim.scala @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.analysis.ResolvedProcedure +import org.apache.spark.sql.catalyst.plans.logical.{Call, LogicalPlan} +import org.apache.spark.sql.connector.catalog.TableCatalog +import org.apache.spark.sql.connector.catalog.procedures.BoundProcedure + +import org.apache.comet.CometConf +import org.apache.comet.CometSparkSessionExtensions.isCometLoaded + +/** + * Spark 4 runs a `CALL` while it analyzes the statement. `InvokeProcedures` optimizes the `Call` + * and then invokes the procedure it holds, so the procedure object is the one place to intercept. + * This swaps Iceberg's `rewrite_data_files` for [[CometRewriteDataFilesProcedure]], which runs + * Comet's rewrite when it can and Iceberg's procedure otherwise. + */ +object CometProcedureShim { + + private val IcebergRewriteDataFiles = + "org.apache.iceberg.spark.procedures.RewriteDataFilesProcedure" + + def wrap(session: SparkSession, plan: LogicalPlan): LogicalPlan = plan match { + case call @ Call( + resolved @ ResolvedProcedure(catalog: TableCatalog, ident, procedure), + _, + true) + if isIcebergRewriteDataFiles(ident.namespace, ident.name, procedure) && + enabled(session) => + call.copy(procedure = resolved.copy(procedure = new CometRewriteDataFilesProcedure( + session, + catalog, + procedure.asInstanceOf[BoundProcedure]))) + case _ => plan + } + + private def isIcebergRewriteDataFiles( + namespace: Array[String], + name: String, + procedure: AnyRef): Boolean = + name == "rewrite_data_files" && + namespace.sameElements(Array("system")) && + procedure.isInstanceOf[BoundProcedure] && + procedure.getClass.getName == IcebergRewriteDataFiles + + private def enabled(session: SparkSession): Boolean = { + val conf = session.sessionState.conf + isCometLoaded(conf) && CometConf.COMET_ICEBERG_COMPACTION_ENABLED.get(conf) + } +} diff --git a/spark/src/main/spark-4.x/org/apache/comet/iceberg/compaction/CometRewriteDataFilesProcedure.scala b/spark/src/main/spark-4.x/org/apache/comet/iceberg/compaction/CometRewriteDataFilesProcedure.scala new file mode 100644 index 00000000000..fd490aef1ea --- /dev/null +++ b/spark/src/main/spark-4.x/org/apache/comet/iceberg/compaction/CometRewriteDataFilesProcedure.scala @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import scala.util.control.NonFatal + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.GenericInternalRow +import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} +import org.apache.spark.sql.connector.catalog.procedures.{BoundProcedure, ProcedureParameter} +import org.apache.spark.sql.connector.read.{LocalScan, Scan} +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType} + +import org.apache.comet.iceberg.IcebergRefreshCacheShim + +/** + * Iceberg's `rewrite_data_files` with Comet's rewrite in front of it. + * + * Every call is first checked against what [[CometRewriteDataFiles]] reproduces. A call that is + * not covered, such as the `sort` or `zorder` strategies, an option Comet does not know, a + * `where` that does not convert, or a table Comet cannot load, runs `delegate` instead. The check + * changes nothing, so the fallback is safe. Once the rewrite has started, a failure is the call's + * failure, the same as with Iceberg. + * + * @param delegate + * Iceberg's procedure. Its parameters, name and description are this procedure's. + */ +class CometRewriteDataFilesProcedure( + session: SparkSession, + catalog: TableCatalog, + delegate: BoundProcedure) + extends BoundProcedure + with Logging { + + import CometRewriteDataFilesProcedure._ + + override def name(): String = delegate.name() + + override def description(): String = delegate.description() + + override def parameters(): Array[ProcedureParameter] = delegate.parameters() + + override def isDeterministic: Boolean = delegate.isDeterministic + + override def call(input: InternalRow): java.util.Iterator[Scan] = { + val prepared = + try { + prepare(input) + } catch { + case NonFatal(e) => + logWarning( + "Comet cannot plan rewrite_data_files, so Iceberg's procedure runs it: " + + e.getMessage, + e) + None + } + prepared match { + case Some(rewrite) => + java.util.Collections.singleton[Scan](new Output(toRow(rewrite.execute()))).iterator() + case None => + delegate.call(input) + } + } + + /** The planned rewrite, or None when the call is left to Iceberg. Changes nothing. */ + private def prepare(input: InternalRow): Option[CometRewriteDataFiles.PreparedRewrite] = { + val params = parameters() + def ordinal(name: String): Int = params.indexWhere(_.name() == name) + def string(name: String): Option[String] = { + val i = ordinal(name) + if (i < 0 || input.isNullAt(i)) None else Some(input.getUTF8String(i).toString) + } + + if (!outputMatches(delegate)) { + return decline("the procedure's result schema is not the one Comet produces") + } + val strategy = string("strategy") + if (string("sort_order").isDefined || strategy.exists(!_.equalsIgnoreCase("binpack"))) { + return decline("only the binpack strategy without a sort order is supported") + } + + val tableArgument = string("table").filter(_.nonEmpty) + val identifier = tableArgument.flatMap { name => + IcebergRewriteReflection.resolveIdentifier(session, catalog, name) + } + if (identifier.isEmpty) { + return decline("the table is not in the procedure's catalog") + } + val ident = identifier.get.asInstanceOf[Identifier] + + val sparkTable = catalog.loadTable(ident) + if (!IcebergRewriteReflection.isSparkTable(sparkTable)) { + return decline("the table is not an Iceberg table") + } + val icebergTable = IcebergRewriteReflection.icebergTable(sparkTable) + + val filter = string("where").map { where => + IcebergRewriteReflection.whereExpression(session, catalog.name(), ident, where) + } + val options = { + val i = ordinal("options") + if (i < 0 || input.isNullAt(i)) { + Map.empty[String, String] + } else { + val map = input.getMap(i) + val keys = map.keyArray() + val values = map.valueArray() + (0 until map.numElements()) + .map(n => keys.getUTF8String(n).toString -> values.getUTF8String(n).toString) + .toMap + } + } + + val refresh = () => + IcebergRefreshCacheShim.refreshCache( + session, + DataSourceV2Relation.create(sparkTable, Some(catalog), Some(ident))) + val request = CometRewriteDataFiles.Request(session, icebergTable, options, filter, refresh) + CometRewriteDataFiles.prepare(request) match { + case Right(rewrite) => Some(rewrite) + case Left(reason) => decline(reason) + } + } + + private def decline[T](reason: String): Option[T] = { + logInfo(s"Comet leaves this rewrite_data_files call to Iceberg: $reason") + None + } +} + +object CometRewriteDataFilesProcedure { + + /** The result row Iceberg's procedure returns, which this one must match exactly. */ + private val OutputType = StructType( + Seq( + StructField("rewritten_data_files_count", IntegerType, nullable = false), + StructField("added_data_files_count", IntegerType, nullable = false), + StructField("rewritten_bytes_count", LongType, nullable = false), + StructField("failed_data_files_count", IntegerType, nullable = false), + StructField("removed_delete_files_count", IntegerType, nullable = false))) + + /** + * Iceberg keeps the result schema in a private constant. A release that adds a column would + * make Comet's row the wrong shape, so a mismatch sends the call to Iceberg. + */ + private def outputMatches(delegate: BoundProcedure): Boolean = + try { + val field = delegate.getClass.getDeclaredField("OUTPUT_TYPE") + field.setAccessible(true) + field.get(null) == OutputType + } catch { + case NonFatal(_) => false + } + + private def toRow(result: CometRewriteDataFiles.Result): Array[InternalRow] = + Array( + new GenericInternalRow( + Array[Any]( + result.rewrittenDataFilesCount, + result.addedDataFilesCount, + result.rewrittenBytesCount, + result.failedDataFilesCount, + result.removedDeleteFilesCount))) + + private final class Output(data: Array[InternalRow]) extends LocalScan { + override def readSchema(): StructType = OutputType + + override def rows(): Array[InternalRow] = data + } +} diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergCompactionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergCompactionSuite.scala new file mode 100644 index 00000000000..3b8f013d992 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/CometIcebergCompactionSuite.scala @@ -0,0 +1,628 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +import java.io.File +import java.util.concurrent.ConcurrentLinkedQueue + +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.fs.Path +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.spark.CometListenerBusUtils +import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart, SparkListenerStageSubmitted} +import org.apache.spark.sql.{CometTestBase, Row} + +import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.iceberg.compaction.CometRewriteDataFiles + +/** + * End to end tests for `CALL system.rewrite_data_files` with Comet's bin-pack rewrite + * (`spark.comet.iceberg.compaction.enabled`). Spark runs the procedure while it analyzes the + * statement, so the tests call it through SQL and look at the jobs it ran. Comet's jobs carry the + * job group prefix `COMET-REWRITE-DATA-FILES` and Iceberg's `REWRITE-DATA-FILES`. + */ +class CometIcebergCompactionSuite extends CometTestBase with CometIcebergTestBase { + + private val catalog = "test_cat" + + private val SmallFileOptions = "options => map('min-input-files', '2')" + + private def withCompaction(enabled: Boolean = true)(body: => Unit): Unit = { + assume(icebergAvailable, "Iceberg not available in classpath") + assume(isSpark40Plus, "Comet intercepts the CALL statement on Spark 4.0 and later only") + withHadoopCatalog(catalog) { + withSQLConf( + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_COMPACTION_ENABLED.key -> enabled.toString)(body) + } + } + + // -- Fixtures -------------------------------------------------------------------------- + + /** Creates `table` and appends `files` one-file commits of 100 rows each. */ + private def createTable(table: String, files: Int, partitioned: Boolean = false): Unit = { + val partitioning = if (partitioned) "PARTITIONED BY (part)" else "" + sql(s"CREATE TABLE $table (id INT, part STRING, value DOUBLE) USING iceberg $partitioning") + (0 until files).foreach(i => appendFile(table, i, "a")) + } + + private def appendFile(table: String, index: Int, part: String): Unit = + spark + .range(index * 100L, (index + 1) * 100L) + .selectExpr( + "CAST(id AS INT) AS id", + s"'$part' AS part", + "CAST(id * 1.5 AS DOUBLE) AS value") + .coalesce(1) + .write + .format("iceberg") + .mode("append") + .saveAsTable(table) + + /** Creates a table with the partitions `a`, `b` and `c`, each with three small files. */ + private def createThreePartitions(table: String): Unit = { + createTable(table, files = 0, partitioned = true) + Seq("a", "b", "c").zipWithIndex.foreach { case (part, p) => + (0 until 3).foreach(i => appendFile(table, p * 10 + i, part)) + } + } + + private def rows(table: String): Seq[Row] = + sql(s"SELECT * FROM $table ORDER BY id").collect().toSeq + + private def dataFileCount(table: String): Long = + sql(s"SELECT COUNT(*) FROM $table.data_files").collect().head.getLong(0) + + private def replaceSnapshotCount(table: String): Long = + sql(s"SELECT COUNT(*) FROM $table.snapshots WHERE operation = 'replace'") + .collect() + .head + .getLong(0) + + private def call(table: String, arguments: String): Row = + sql(s"CALL $catalog.system.rewrite_data_files(table => '$table', $arguments)") + .collect() + .head + + /** The job groups of the jobs that ran while `body` ran. */ + private def jobGroups(body: => Unit): Seq[String] = { + val groups = new ConcurrentLinkedQueue[String]() + val listener = new SparkListener { + override def onJobStart(jobStart: SparkListenerJobStart): Unit = { + val properties = jobStart.properties + if (properties != null) { + Option(properties.getProperty("spark.jobGroup.id")).foreach(groups.add) + } + } + } + CometListenerBusUtils.waitUntilEmpty(spark.sparkContext) + spark.sparkContext.addSparkListener(listener) + try { + body + CometListenerBusUtils.waitUntilEmpty(spark.sparkContext) + } finally { + spark.sparkContext.removeSparkListener(listener) + } + groups.asScala.toSeq + } + + /** The number of tasks of each stage that ran while `body` ran. */ + private def stageTaskCounts(body: => Unit): Seq[Int] = { + val counts = new ConcurrentLinkedQueue[Int]() + val listener = new SparkListener { + override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = { + counts.add(stageSubmitted.stageInfo.numTasks) + } + } + CometListenerBusUtils.waitUntilEmpty(spark.sparkContext) + spark.sparkContext.addSparkListener(listener) + try { + body + CometListenerBusUtils.waitUntilEmpty(spark.sparkContext) + } finally { + spark.sparkContext.removeSparkListener(listener) + } + counts.asScala.toSeq + } + + private def cometJobs(groups: Seq[String]): Seq[String] = + groups.filter(_.startsWith(CometRewriteDataFiles.JobGroupPrefix)) + + private def icebergJobs(groups: Seq[String]): Seq[String] = + groups.filter(_.startsWith("REWRITE-DATA-FILES")) + + /** + * Comet's native Iceberg writer stamps `(Comet)` into the footer's `created_by`. Reading the + * footer shows who wrote the file, which the plan alone does not. + */ + private def assertWrittenByCometNative(table: String): Unit = { + val paths = sql(s"SELECT file_path FROM $table.data_files").collect().map(_.getString(0)) + assert(paths.nonEmpty) + paths.foreach { path => + val reader = ParquetFileReader.open( + HadoopInputFile + .fromPath(new Path(path.stripPrefix("file:")), spark.sparkContext.hadoopConfiguration)) + val createdBy = + try reader.getFooter.getFileMetaData.getCreatedBy + finally reader.close() + assert( + createdBy != null && createdBy.contains("(Comet)"), + s"$path was written by $createdBy") + } + } + + // -- Tests ----------------------------------------------------------------------------- + + test("bin-pack rewrite compacts small files and returns Iceberg's result row") { + withCompaction() { + val table = s"$catalog.db.compact_basic" + try { + createTable(table, files = 4) + val before = rows(table) + assert(dataFileCount(table) == 4) + + var result: Row = null + val groups = jobGroups { + result = call("db.compact_basic", SmallFileOptions) + } + + assert(cometJobs(groups).nonEmpty, s"Comet did not run the rewrite, jobs: $groups") + assert(icebergJobs(groups).isEmpty, s"Iceberg ran a rewrite job too, jobs: $groups") + assert(result.length == 5) + assert(result.getInt(0) == 4, "rewritten_data_files_count") + assert(result.getInt(1) == 1, "added_data_files_count") + assert(result.getLong(2) > 0L, "rewritten_bytes_count") + assert(result.getInt(3) == 0, "failed_data_files_count") + assert(result.getInt(4) == 0, "removed_delete_files_count") + + assert(dataFileCount(table) == 1) + assert(rows(table) == before) + assert(replaceSnapshotCount(table) == 1) + assertWrittenByCometNative(table) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("the rewrite is left to Iceberg unless Comet's compaction is enabled") { + withCompaction(enabled = false) { + val table = s"$catalog.db.compact_disabled" + try { + createTable(table, files = 4) + val groups = jobGroups { + assert(call("db.compact_disabled", SmallFileOptions).getInt(0) == 4) + } + assert(cometJobs(groups).isEmpty) + assert(icebergJobs(groups).nonEmpty) + assert(dataFileCount(table) == 1) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("a sort strategy runs Iceberg's procedure") { + withCompaction() { + val table = s"$catalog.db.compact_sort" + try { + createTable(table, files = 4) + setIcebergSortOrderAsc(table, "id") + val before = rows(table) + val groups = jobGroups { + call("db.compact_sort", s"strategy => 'sort', $SmallFileOptions") + } + assert(cometJobs(groups).isEmpty) + assert(icebergJobs(groups).nonEmpty) + assert(rows(table) == before) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("an option Comet does not reproduce runs Iceberg's procedure and its error") { + withCompaction() { + val table = s"$catalog.db.compact_bad_option" + try { + createTable(table, files = 2) + val error = intercept[Exception] { + call("db.compact_bad_option", "options => map('no-such-option', '1')") + } + val messages = + Iterator.iterate[Throwable](error)(_.getCause).takeWhile(_ != null).map(_.getMessage) + assert(messages.exists(m => m != null && m.contains("no-such-option"))) + assert(dataFileCount(table) == 2) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("rewriting an empty table does nothing") { + withCompaction() { + val table = s"$catalog.db.compact_empty" + try { + sql(s"CREATE TABLE $table (id INT) USING iceberg") + val groups = jobGroups { + val result = call("db.compact_empty", SmallFileOptions) + assert(result == Row(0, 0, 0L, 0, 0)) + } + assert(groups.isEmpty) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("files already near the target size are left alone") { + withCompaction() { + val table = s"$catalog.db.compact_nothing" + try { + createTable(table, files = 3) + // Every file is within [size / 2, size * 4], so none is worth rewriting. + val size = sql(s"SELECT MIN(file_size_in_bytes) FROM $table.data_files") + .collect() + .head + .getLong(0) + val result = call( + "db.compact_nothing", + s"options => map('min-input-files', '10', 'min-file-size-bytes', '${size / 2}', " + + s"'target-file-size-bytes', '${size * 2}', 'max-file-size-bytes', '${size * 4}')") + assert(result == Row(0, 0, 0L, 0, 0)) + assert(dataFileCount(table) == 3) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("where limits the rewrite to the matching partitions") { + withCompaction() { + val table = s"$catalog.db.compact_where" + try { + createTable(table, files = 0, partitioned = true) + (0 until 3).foreach(i => appendFile(table, i, "a")) + (3 until 6).foreach(i => appendFile(table, i, "b")) + val before = rows(table) + + val result = call("db.compact_where", s"""where => "part = 'a'", $SmallFileOptions""") + + assert(result.getInt(0) == 3 && result.getInt(1) == 1) + def files(part: String): Long = + sql(s"SELECT COUNT(*) FROM $table.data_files WHERE partition.part = '$part'") + .collect() + .head + .getLong(0) + assert(files("a") == 1) + assert(files("b") == 3) + assert(rows(table) == before) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("one Spark job rewrites all the file groups of a commit") { + withCompaction() { + val table = s"$catalog.db.compact_one_job" + try { + createThreePartitions(table) + val before = rows(table) + + var result: Row = null + val groups = jobGroups { + result = call("db.compact_one_job", SmallFileOptions) + } + + // Three partitions are three file groups, which one job rewrites. + assert(cometJobs(groups).distinct.size == 1, s"Expected one rewrite job, got $groups") + assert(icebergJobs(groups).isEmpty) + assert(result.getInt(0) == 9 && result.getInt(1) == 3 && result.getInt(3) == 0) + assert(dataFileCount(table) == 3) + assert(replaceSnapshotCount(table) == 1) + assert(rows(table) == before) + assertWrittenByCometNative(table) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("each file group has a job of its own when the single job rewrite is off") { + withCompaction() { + withSQLConf(CometConf.COMET_ICEBERG_COMPACTION_SINGLE_JOB_ENABLED.key -> "false") { + val table = s"$catalog.db.compact_job_per_group" + try { + createThreePartitions(table) + val before = rows(table) + + var result: Row = null + val groups = jobGroups { + result = call("db.compact_job_per_group", SmallFileOptions) + } + + assert( + cometJobs(groups).distinct.size == 3, + s"Expected three rewrite jobs, got $groups") + assert(result.getInt(0) == 9 && result.getInt(1) == 3) + assert(dataFileCount(table) == 3) + assert(replaceSnapshotCount(table) == 1) + assert(rows(table) == before) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + } + + test("file groups are rewritten one job each when Comet's native scan is off") { + withCompaction() { + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "false") { + val table = s"$catalog.db.compact_no_native_scan" + try { + createThreePartitions(table) + val before = rows(table) + + val groups = jobGroups { + assert(call("db.compact_no_native_scan", SmallFileOptions).getInt(0) == 9) + } + + // Spark cannot read a bin without the native scan, so each group is read as Iceberg does. + assert( + cometJobs(groups).distinct.size == 3, + s"Expected three rewrite jobs, got $groups") + assert(dataFileCount(table) == 3) + assert(rows(table) == before) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + } + + test("large files are split across several read tasks") { + withCompaction() { + val table = s"$catalog.db.compact_split" + try { + // Row groups of 16 KiB, so that a file of about 100 KiB has several that a split can + // end at. + sql(s""" + CREATE TABLE $table (id INT, part STRING, value DOUBLE, payload STRING) USING iceberg + TBLPROPERTIES ('write.parquet.row-group-size-bytes' = '16384')""") + (0 until 4).foreach { i => + spark + .range(i * 3000L, (i + 1) * 3000L) + .selectExpr( + "CAST(id AS INT) AS id", + "'a' AS part", + "CAST(id * 1.5 AS DOUBLE) AS value", + "uuid() AS payload") + .coalesce(1) + .write + .format("iceberg") + .mode("append") + .saveAsTable(table) + } + val before = rows(table) + val inputBytes = sql(s"SELECT SUM(file_size_in_bytes) FROM $table.data_files") + .collect() + .head + .getLong(0) + + // A target of a third of a file puts every file past the maximum size, so it is rewritten + // into pieces of about the target. + val target = inputBytes / 4 / 3 + var result: Row = null + val taskCounts = stageTaskCounts { + result = call( + "db.compact_split", + s"options => map('target-file-size-bytes', '$target', 'min-input-files', '2')") + } + + assert(result.getInt(0) == 4, s"rewritten_data_files_count: $result") + assert(result.getInt(1) >= 6, s"Expected the files to be split into pieces: $result") + // Every read task is a Spark task of the write, so there is a stage with that many. + assert(taskCounts.max >= 6, s"Stages ran $taskCounts tasks") + assert(rows(table) == before) + assertWrittenByCometNative(table) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("partial progress commits file groups in several snapshots") { + withCompaction() { + val table = s"$catalog.db.compact_partial" + try { + createThreePartitions(table) + val before = rows(table) + + var result: Row = null + val groups = jobGroups { + result = call( + "db.compact_partial", + "options => map('min-input-files', '2', 'partial-progress.enabled', 'true', " + + "'partial-progress.max-commits', '3')") + } + + assert(result.getInt(0) == 9 && result.getInt(1) == 3 && result.getInt(3) == 0) + assert(dataFileCount(table) == 3) + assert(replaceSnapshotCount(table) == 3) + // A job covers the groups of one commit, so three commits are three jobs. + assert(cometJobs(groups).distinct.size == 3, s"Expected three rewrite jobs, got $groups") + assert(rows(table) == before) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("partial progress covers several file groups per job") { + withCompaction() { + val table = s"$catalog.db.compact_partial_batches" + try { + createThreePartitions(table) + val before = rows(table) + + // Three groups in at most two commits is two groups in the first and one in the second. + val groups = jobGroups { + val result = call( + "db.compact_partial_batches", + "options => map('min-input-files', '2', 'partial-progress.enabled', 'true', " + + "'partial-progress.max-commits', '2')") + assert(result.getInt(0) == 9 && result.getInt(1) == 3) + } + + // The jobs finish in either order, and a commit takes every job that has finished by + // then, so there are one or two commits but never more than the option allows. + assert(Set(1L, 2L).contains(replaceSnapshotCount(table))) + assert(cometJobs(groups).distinct.size == 2, s"Expected two rewrite jobs, got $groups") + assert(dataFileCount(table) == 3) + assert(rows(table) == before) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("rewrite preserves the rows of every column type, nulls included") { + withCompaction() { + val table = s"$catalog.db.compact_types" + try { + sql(s""" + CREATE TABLE $table ( + id INT, l BIGINT, d DOUBLE, f FLOAT, dec DECIMAL(10, 2), s STRING, b BOOLEAN, + dt DATE, ts TIMESTAMP, bin BINARY, arr ARRAY, st STRUCT, + m MAP) USING iceberg""") + (0 until 4).foreach { i => + sql(s""" + INSERT INTO $table + SELECT /*+ COALESCE(1) */ + CAST(id AS INT), + CASE WHEN id % 5 = 0 THEN NULL ELSE id * 1000 END, + CASE WHEN id % 5 = 1 THEN NULL ELSE id * 1.5 END, + CAST(id / 3 AS FLOAT), + CASE WHEN id % 5 = 2 THEN NULL ELSE CAST(id AS DECIMAL(10, 2)) / 7 END, + CASE WHEN id % 5 = 3 THEN NULL ELSE CAST(id AS STRING) END, + id % 2 = 0, + date_add(DATE '2020-01-01', CAST(id AS INT)), + timestamp_micros(id * 1000003), + CAST(CAST(id AS STRING) AS BINARY), + CASE WHEN id % 4 = 0 THEN NULL ELSE array(CAST(id AS INT), CAST(id + 1 AS INT)) END, + named_struct('a', CAST(id AS INT), 'b', CAST(id AS STRING)), + map(CAST(id AS STRING), CAST(id AS INT)) + FROM range(${i * 50}, ${(i + 1) * 50})""") + } + val before = rows(table) + assert(before.size == 200) + assert(dataFileCount(table) == 4) + + assert(call("db.compact_types", SmallFileOptions).getInt(0) == 4) + + assert(dataFileCount(table) == 1) + assert(rows(table) == before) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("a delete that lands before the commit fails the rewrite and leaves no new files") { + withCompaction() { + val table = s"$catalog.db.compact_conflict" + try { + sql(s""" + CREATE TABLE $table (id INT, part STRING, value DOUBLE) USING iceberg + TBLPROPERTIES ( + 'format-version' = '2', + 'write.delete.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read')""") + (0 until 4).foreach(appendFile(table, _, "a")) + val icebergTable = loadIcebergTable(spark, catalog, "db", "compact_conflict") + val location = icebergTable.getClass.getMethod("location").invoke(icebergTable).toString + def parquetFiles(): Int = + new File(location.stripPrefix("file:"), "data") + .listFiles() + .count(_.getName.endsWith(".parquet")) + + // The rewrite plans against the four files, and then a delete changes one of them. + val rewrite = CometRewriteDataFiles.prepare( + CometRewriteDataFiles + .Request(spark, icebergTable, Map("min-input-files" -> "2"), None, () => ())) match { + case Right(prepared) => prepared + case Left(reason) => fail(s"Comet declined the rewrite: $reason") + } + sql(s"DELETE FROM $table WHERE id = 5") + val before = rows(table) + val filesBefore = parquetFiles() + assert(before.size == 399) + + val error = intercept[RuntimeException](rewrite.execute()) + + assert(error.getMessage.contains("Cannot commit rewrite"), error.getMessage) + assert(dataFileCount(table) == 4) + assert(rows(table) == before) + // The file the rewrite wrote was deleted when the commit failed. + assert(parquetFiles() == filesBefore) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("rewrite applies position deletes of a merge-on-read table") { + withCompaction() { + val table = s"$catalog.db.compact_deletes" + try { + sql(s""" + CREATE TABLE $table (id INT, part STRING, value DOUBLE) USING iceberg + TBLPROPERTIES ( + 'format-version' = '2', + 'write.delete.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read')""") + appendFile(table, 0, "a") + appendFile(table, 1, "a") + sql(s"DELETE FROM $table WHERE id = 5") + val before = rows(table) + assert(before.size == 199) + + val groups = jobGroups { + call("db.compact_deletes", SmallFileOptions) + } + + assert(cometJobs(groups).nonEmpty) + assert(rows(table) == before) + // The deleted row is gone from the data files, not just masked by a delete file. + assert( + sql(s"SELECT SUM(record_count) FROM $table.data_files").collect().head.getLong(0) == + 199L) + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/iceberg/compaction/BinPackPlannerSuite.scala b/spark/src/test/scala/org/apache/comet/iceberg/compaction/BinPackPlannerSuite.scala new file mode 100644 index 00000000000..0a481e3d84b --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/iceberg/compaction/BinPackPlannerSuite.scala @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import org.scalatest.funsuite.AnyFunSuite + +/** + * Unit tests for the bin-pack planner. The planner has no Iceberg dependency, so these run on + * every Spark profile. The expected groups follow iceberg-java's `BinPackRewriteFilePlanner`. + */ +class BinPackPlannerSuite extends AnyFunSuite { + + private val MB = 1024L * 1024 + + // A 100 MB target gives a 75 MB minimum, a 180 MB maximum and a 140 MB rolling size. + private val DefaultOptions = Seq(CompactionOptions.TargetFileSizeBytes -> (100 * MB).toString) + + private def options(extra: (String, String)*): CompactionOptions = + CompactionOptions.parse((DefaultOptions ++ extra).toMap, Map.empty, currentSpecId = 0) match { + case Right(parsed) => parsed + case Left(reason) => fail(s"Cannot parse the options: $reason") + } + + private def file( + id: String, + megabytes: Long, + partition: Seq[Any] = Nil, + deleteFiles: Int = 0, + deletedRows: Long = 0L, + records: Long = 100L): CompactionFile = + CompactionFile( + task = id, + length = megabytes * MB, + recordCount = records, + specId = 0, + partition = partition, + deleteFileCount = deleteFiles, + fileScopedDeletedRows = deletedRows) + + private def small( + count: Int, + megabytes: Long, + partition: Seq[Any] = Nil, + prefix: String = "f") = + (0 until count).map(i => file(s"$prefix$i", megabytes, partition)) + + private def ids(group: FileGroup): Seq[String] = group.files.map(_.task.toString) + + // -- Which files are rewritten ---------------------------------------------------------- + + test("enough small files in a partition form one group") { + val plan = BinPackPlanner.plan(small(5, 10), options()) + assert(plan.groups.size == 1) + assert(plan.totalGroupCount == 1) + val group = plan.groups.head + assert(ids(group) == Seq("f0", "f1", "f2", "f3", "f4")) + assert(group.globalIndex == 1 && group.partitionIndex == 1) + assert(group.expectedOutputFiles == 1) + assert(group.inputBytes == 50 * MB) + } + + test("a partition with too few small files is left alone") { + val plan = BinPackPlanner.plan(small(4, 10), options()) + assert(plan.groups.isEmpty) + assert(plan.totalGroupCount == 0) + } + + test("min-input-files sets how many small files are enough") { + assert(BinPackPlanner.plan(small(2, 10), options("min-input-files" -> "2")).groups.size == 1) + assert(BinPackPlanner.plan(small(1, 10), options("min-input-files" -> "2")).groups.isEmpty) + } + + test("files that add up past the target size are rewritten even when there are few") { + val plan = BinPackPlanner.plan(Seq(file("a", 60), file("b", 60)), options()) + assert(plan.groups.size == 1) + assert(ids(plan.groups.head) == Seq("a", "b")) + } + + test("a single small file is never rewritten") { + assert(BinPackPlanner.plan(Seq(file("a", 10)), options()).groups.isEmpty) + } + + test("files within the desired size range are not rewritten") { + val plan = BinPackPlanner.plan(small(6, 100), options()) + assert(plan.groups.isEmpty) + } + + test("the size range is closed at both ends") { + // 75 MB is the minimum and 180 MB the maximum, and neither is outside the range. + val plan = BinPackPlanner.plan(small(3, 75) ++ small(3, 180, prefix = "g"), options()) + assert(plan.groups.isEmpty) + } + + test("an oversized file is rewritten to split it") { + val plan = BinPackPlanner.plan(Seq(file("big", 200)), options()) + assert(plan.groups.size == 1) + assert(plan.groups.head.expectedOutputFiles == 2) + } + + test("partitions are never mixed") { + val files = small(6, 10, partition = Seq("a")) ++ small(3, 10, partition = Seq("b"), "g") + val plan = BinPackPlanner.plan(files, options()) + assert(plan.groups.size == 1) + assert(plan.groups.head.partition == Seq("a")) + } + + test("each partition numbers its own groups") { + val files = small(5, 10, Seq("a"), "a") ++ small(5, 10, Seq("b"), "b") + val plan = BinPackPlanner.plan(files, options()) + assert(plan.groups.map(g => (g.globalIndex, g.partitionIndex)) == Seq((1, 1), (2, 1))) + } + + test("max-file-group-size-bytes splits a partition into several groups") { + val plan = BinPackPlanner.plan( + small(10, 30), + options("max-file-group-size-bytes" -> (100 * MB).toString, "min-input-files" -> "2")) + // 100 MB holds three 30 MB files. The last file is alone and too small to be worth it. + assert(plan.groups.map(_.numFiles) == Seq(3, 3, 3)) + assert(plan.totalGroupCount == 3) + } + + test("an item heavier than the group size gets a group of its own") { + val plan = BinPackPlanner.plan( + Seq(file("a", 10), file("huge", 500), file("b", 10)), + options("max-file-group-size-bytes" -> (100 * MB).toString, "min-input-files" -> "1")) + assert(plan.groups.map(ids) == Seq(Seq("huge"))) + } + + // -- Deletes ------------------------------------------------------------------------------- + + test("delete-file-threshold selects a right-sized file") { + val files = Seq(file("a", 100, deleteFiles = 2)) + assert(BinPackPlanner.plan(files, options()).groups.isEmpty) + val plan = BinPackPlanner.plan(files, options("delete-file-threshold" -> "2")) + assert(plan.groups.map(ids) == Seq(Seq("a"))) + } + + test("delete-ratio-threshold selects a file whose rows are mostly deleted") { + def plan(deleted: Long) = + BinPackPlanner.plan(Seq(file("a", 100, deleteFiles = 1, deletedRows = deleted)), options()) + assert(plan(29).groups.isEmpty) + assert(plan(30).groups.size == 1) + // Deleted rows are capped at the file's own record count. + assert(plan(1000).groups.size == 1) + } + + test("a file with no rows is not selected by the delete ratio") { + val files = Seq(file("a", 100, deleteFiles = 1, deletedRows = 5, records = 0)) + assert(BinPackPlanner.plan(files, options()).groups.isEmpty) + } + + test("a file in a group with a heavily deleted file is rewritten with it") { + val files = + Seq(file("a", 100, deleteFiles = 1, deletedRows = 50), file("b", 100), file("c", 20)) + val plan = BinPackPlanner.plan(files, options()) + // Only the files outside the size range or with too many deletes are candidates. + assert(plan.groups.map(ids) == Seq(Seq("a", "c"))) + } + + // -- Options ------------------------------------------------------------------------------- + + test("rewrite-all rewrites files of any size") { + val plan = BinPackPlanner.plan(small(2, 100), options("rewrite-all" -> "true")) + assert(plan.groups.map(ids) == Seq(Seq("f0", "f1"))) + } + + test("max-files-to-rewrite trims the selection but not the group count") { + val files = (0 until 3).flatMap(p => small(5, 10, Seq(s"p$p"), s"p$p-")) + val plan = BinPackPlanner.plan(files, options("max-files-to-rewrite" -> "7")) + assert(plan.totalGroupCount == 3) + assert(plan.groups.map(_.numFiles) == Seq(5, 2)) + assert(plan.groups.map(_.partition) == Seq(Seq("p0"), Seq("p1"))) + } + + test("rewrite-job-order sorts the groups") { + val files = + small(5, 10, Seq("small"), "s") ++ small(7, 10, Seq("many"), "m") ++ + small(5, 30, Seq("big"), "b") + def order(jobOrder: String) = + BinPackPlanner + .plan(files, options("rewrite-job-order" -> jobOrder)) + .groups + .map(_.partition.head) + assert(order("none") == Seq("small", "many", "big")) + assert(order("bytes-asc") == Seq("small", "many", "big")) + assert(order("bytes-desc") == Seq("big", "many", "small")) + assert(order("files-asc") == Seq("small", "big", "many")) + assert(order("files-desc") == Seq("many", "small", "big")) + } + + test("files from an older spec group under one key") { + val old = (0 until 3).map(i => file(s"old$i", 10, partition = Seq(null)).copy(specId = 1)) + val files = old ++ small(2, 10, partition = Seq("x"), prefix = "x") + val plan = BinPackPlanner.plan(files, options("min-input-files" -> "2")) + assert(plan.groups.map(ids).toSet == Set(Seq("old0", "old1", "old2"), Seq("x0", "x1"))) + } + + // -- Sizes -------------------------------------------------------------------------------- + + test("the output file count rounds a small remainder into the other files") { + val gb = 1024 * MB + val opts = CompactionOptions + .parse(Map(CompactionOptions.TargetFileSizeBytes -> gb.toString), Map.empty, 0) + .toOption + .get + def files(inputGb: Double) = BinPackPlanner.expectedOutputFiles((inputGb * gb).toLong, opts) + assert(files(0.5) == 1) + assert(files(1.0) == 1) + // A remainder under the minimum size is spread over the others while that keeps the + // average file within 10% of the target. + assert(files(10.1) == 10) + assert(files(10.5) == 10) + // A remainder over the minimum size is a file of its own. + assert(files(10.9) == 11) + // Spreading 0.6 GB over one file would make it 1.6 GB, which is too large. + assert(files(1.6) == 2) + } + + test("the split size is the input per output file, between the target and the rolling size") { + val opts = options() + // 200 MB into two files is 100 MB each plus the 5 KB overhead. + assert(BinPackPlanner.inputSplitSize(200 * MB, opts) == 100 * MB + 5 * 1024) + // Small input never splits finer than the target. + assert(BinPackPlanner.inputSplitSize(10 * MB, opts) == 100 * MB) + // And it never exceeds the size the writer rolls at. + assert(BinPackPlanner.inputSplitSize(1000 * MB, opts) <= opts.writeMaxFileSize) + } + + // -- Packing ------------------------------------------------------------------------------- + + test("pack with a lookback of one fills a bin and starts the next") { + val bins = BinPackPlanner.pack(Seq(10, 90, 20), 100L, 1, largestBinFirst = false)(_.toLong) + assert(bins == Seq(Seq(10, 90), Seq(20))) + } + + test("pack keeps an oversized item alone") { + val bins = BinPackPlanner.pack(Seq(10, 500, 10), 100L, 1, largestBinFirst = false)(_.toLong) + assert(bins == Seq(Seq(10), Seq(500), Seq(10))) + } + + test("pack of nothing is nothing") { + assert( + BinPackPlanner.pack(Seq.empty[Int], 100L, 1, largestBinFirst = false)(_.toLong).isEmpty) + } + + test("pack with a larger lookback puts an item in the first bin it fits") { + val bins = + BinPackPlanner.pack(Seq(60, 60, 30, 30), 100L, 10, largestBinFirst = false)(_.toLong) + assert(bins == Seq(Seq(60, 30), Seq(60, 30))) + } + + test("pack closes the oldest or the heaviest bin when the lookback is exceeded") { + val items = Seq(50, 70, 60, 20) + val oldest = BinPackPlanner.pack(items, 100L, 2, largestBinFirst = false)(_.toLong) + val heaviest = BinPackPlanner.pack(items, 100L, 2, largestBinFirst = true)(_.toLong) + assert(oldest == Seq(Seq(50), Seq(70, 20), Seq(60))) + assert(heaviest == Seq(Seq(70), Seq(50, 20), Seq(60))) + } + + test("read tasks split a group into bins no larger than the split size") { + val sizes = Seq.fill(6)(30L) + val bins = BinPackPlanner.planReadTasks(sizes, splitSize = 100L)(identity, _ => 1) + assert(bins == Seq(Seq(30L, 30L, 30L), Seq(30L, 30L, 30L))) + } + + test("read tasks pack the heaviest open bin first when there are too many") { + // A lookback of two closes the heaviest bin when a third opens, which is how Iceberg plans + // the read of a staged group. + val bins = + BinPackPlanner.planReadTasks(Seq(50L, 70L, 60L, 20L), splitSize = 100L, lookback = 2)( + identity, + _ => 1) + assert(bins == Seq(Seq(70L), Seq(50L, 20L), Seq(60L))) + } + + test("read tasks count an empty file as its open cost") { + val bins = BinPackPlanner.planReadTasks(Seq(0L, 0L, 0L), splitSize = 10L, openFileCost = 5L)( + identity, + _ => 1) + assert(bins.map(_.size) == Seq(2, 1)) + } + + test("read tasks weigh the delete files a task opens") { + // One byte each, but every task opens three files at a cost of 10: three tasks fit a bin of + // 100 and a fourth does not. + val bins = BinPackPlanner.planReadTasks( + Seq.fill(4)(1L), + splitSize = 100L, + openFileCost = 10L)(identity, _ => 3) + assert(bins.map(_.size) == Seq(3, 1)) + } + + // -- Splitting groups into jobs ------------------------------------------------------------ + + test("a job takes at most the number of groups asked for") { + val jobs = BinPackPlanner.chunk(Seq("a", "b", "c", "d", "e"), 2, Int.MaxValue)(_ => 1) + assert(jobs == Seq(Seq("a", "b"), Seq("c", "d"), Seq("e"))) + } + + test("a job stops before it passes the bin limit") { + val jobs = BinPackPlanner.chunk(Seq("a" -> 6, "b" -> 4, "c" -> 1, "d" -> 9), 10, 10)(_._2) + assert(jobs == Seq(Seq("a" -> 6, "b" -> 4), Seq("c" -> 1, "d" -> 9))) + } + + test("a group over the bin limit gets a job of its own") { + val jobs = BinPackPlanner.chunk(Seq("a" -> 1, "huge" -> 50, "b" -> 1), 10, 10)(_._2) + assert(jobs == Seq(Seq("a" -> 1), Seq("huge" -> 50), Seq("b" -> 1))) + } + + test("without limits all the groups are one job, and none is no job") { + assert( + BinPackPlanner.chunk(Seq(1, 2, 3), Int.MaxValue, Int.MaxValue)(_ => 1) == Seq(Seq(1, 2, 3))) + assert(BinPackPlanner.chunk(Seq.empty[Int], 2, 10)(_ => 1).isEmpty) + } +} diff --git a/spark/src/test/scala/org/apache/comet/iceberg/compaction/CompactionOptionsSuite.scala b/spark/src/test/scala/org/apache/comet/iceberg/compaction/CompactionOptionsSuite.scala new file mode 100644 index 00000000000..a0e01e5935b --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/iceberg/compaction/CompactionOptionsSuite.scala @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import org.scalatest.funsuite.AnyFunSuite + +/** + * Unit tests for the option parsing of a bin-pack `rewrite_data_files`. The defaults are + * iceberg-java's, and anything Comet does not reproduce has to come back as a reason so that the + * call runs Iceberg's procedure. + */ +class CompactionOptionsSuite extends AnyFunSuite { + + private val MB = 1024L * 1024 + + private def parse( + options: Map[String, String], + properties: Map[String, String] = Map.empty, + specId: Int = 3): Either[String, CompactionOptions] = + CompactionOptions.parse(options, properties, specId) + + private def parsed( + options: Map[String, String] = Map.empty, + properties: Map[String, String] = Map.empty): CompactionOptions = + parse(options, properties) match { + case Right(o) => o + case Left(reason) => fail(s"Expected the options to parse, got: $reason") + } + + test("defaults follow iceberg-java") { + val o = parsed() + assert(o.targetFileSize == 512 * MB) + assert(o.minFileSize == (512 * MB * 0.75).toLong) + assert(o.maxFileSize == (512 * MB * 1.8).toLong) + assert(o.minInputFiles == 5) + assert(!o.rewriteAll) + assert(o.maxGroupSize == 100L * 1024 * MB) + assert(o.outputSpecId == 3) + assert(o.deleteFileThreshold == Int.MaxValue) + assert(o.deleteRatioThreshold == 0.3) + assert(o.jobOrder == RewriteJobOrder.NoOrder) + assert(o.maxFilesToRewrite.isEmpty) + assert(o.maxConcurrentGroupRewrites == 5) + assert(!o.partialProgress) + assert(o.maxCommits == 10) + assert(o.maxFailedCommits == 10) + assert(o.useStartingSequenceNumber) + } + + test("the target size defaults to the table property") { + val o = parsed(properties = Map("write.target-file-size-bytes" -> (64 * MB).toString)) + assert(o.targetFileSize == 64 * MB) + assert(o.minFileSize == (64 * MB * 0.75).toLong) + assert(o.maxFileSize == (64 * MB * 1.8).toLong) + // The option wins over the table. + val explicit = parsed( + Map("target-file-size-bytes" -> (32 * MB).toString), + Map("write.target-file-size-bytes" -> (64 * MB).toString)) + assert(explicit.targetFileSize == 32 * MB) + } + + test("the rolling size sits halfway between the target and the maximum") { + val o = parsed(Map("target-file-size-bytes" -> "100")) + assert(o.maxFileSize == 180L) + assert(o.writeMaxFileSize == 140L) + } + + test("every supported option is read") { + val o = parsed( + Map( + "target-file-size-bytes" -> "1000", + "min-file-size-bytes" -> "500", + "max-file-size-bytes" -> "2000", + "min-input-files" -> "2", + "rewrite-all" -> "true", + "max-file-group-size-bytes" -> "5000", + "delete-file-threshold" -> "3", + "delete-ratio-threshold" -> "0.5", + "rewrite-job-order" -> "Bytes-Desc", + "max-files-to-rewrite" -> "20", + "max-concurrent-file-group-rewrites" -> "8", + "partial-progress.enabled" -> "true", + "partial-progress.max-commits" -> "4", + "partial-progress.max-failed-commits" -> "2", + "use-starting-sequence-number" -> "false", + "remove-dangling-deletes" -> "false", + "output-spec-id" -> "3")) + assert(o.targetFileSize == 1000 && o.minFileSize == 500 && o.maxFileSize == 2000) + assert(o.minInputFiles == 2 && o.rewriteAll && o.maxGroupSize == 5000) + assert(o.deleteFileThreshold == 3 && o.deleteRatioThreshold == 0.5) + assert(o.jobOrder == RewriteJobOrder.BytesDesc && o.maxFilesToRewrite.contains(20)) + assert(o.maxConcurrentGroupRewrites == 8) + assert(o.partialProgress && o.maxCommits == 4 && o.maxFailedCommits == 2) + assert(!o.useStartingSequenceNumber) + } + + test("the failed commit limit defaults to the commit limit") { + val o = + parsed(Map("partial-progress.enabled" -> "true", "partial-progress.max-commits" -> "7")) + assert(o.maxFailedCommits == 7) + } + + test("booleans parse the way Boolean.parseBoolean does") { + assert(parsed(Map("rewrite-all" -> "TRUE")).rewriteAll) + assert(!parsed(Map("rewrite-all" -> "yes")).rewriteAll) + assert(!parsed(Map("use-starting-sequence-number" -> "1")).useStartingSequenceNumber) + } + + test("an option Comet does not reproduce is reported") { + val reason = parse(Map("shuffle-partitions-per-file" -> "2", "compression-factor" -> "1")) + assert(reason.isLeft) + assert(reason.left.toOption.get.contains("compression-factor")) + assert(reason.left.toOption.get.contains("shuffle-partitions-per-file")) + } + + test("remove-dangling-deletes is only accepted when it is off") { + assert(parse(Map("remove-dangling-deletes" -> "true")).isLeft) + assert(parse(Map("remove-dangling-deletes" -> "false")).isRight) + } + + test("an output spec other than the current one is reported") { + assert(parse(Map("output-spec-id" -> "2")).isLeft) + } + + test("values Iceberg rejects are reported instead of being corrected") { + val bad = Seq( + Map("target-file-size-bytes" -> "abc"), + Map("target-file-size-bytes" -> "0"), + Map("target-file-size-bytes" -> "100", "min-file-size-bytes" -> "100"), + Map("target-file-size-bytes" -> "100", "max-file-size-bytes" -> "100"), + Map("min-file-size-bytes" -> "-1"), + Map("min-input-files" -> "0"), + Map("max-file-group-size-bytes" -> "0"), + Map("delete-file-threshold" -> "-1"), + Map("delete-ratio-threshold" -> "0"), + Map("delete-ratio-threshold" -> "1.5"), + Map("rewrite-job-order" -> "sideways"), + Map("max-files-to-rewrite" -> "0"), + Map("max-concurrent-file-group-rewrites" -> "0"), + Map("partial-progress.enabled" -> "true", "partial-progress.max-commits" -> "0")) + bad.foreach { options => + assert(parse(options).isLeft, s"Expected $options to be reported") + } + } + + test("the commit limit is only checked when partial progress is on") { + assert(parse(Map("partial-progress.max-commits" -> "0")).isRight) + } +} diff --git a/spark/src/test/scala/org/apache/comet/iceberg/compaction/CompactionScanIcebergSuite.scala b/spark/src/test/scala/org/apache/comet/iceberg/compaction/CompactionScanIcebergSuite.scala new file mode 100644 index 00000000000..bb7e8893542 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/iceberg/compaction/CompactionScanIcebergSuite.scala @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import java.util.UUID + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.CometTestBase +import org.apache.spark.sql.comet.CometIcebergNativeScanExec + +import org.apache.comet.{CometConf, CometIcebergTestBase} +import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.iceberg.IcebergReflection + +/** + * Tests that Comet's native Iceberg scan reads a [[CompactionScan]] over a real table: one Spark + * partition per bin, and the table's rows. This is the read half of a bins job, without the write + * and the commit. + */ +class CompactionScanIcebergSuite extends CometTestBase with CometIcebergTestBase { + + private val catalog = "test_cat" + + private def withNativeIceberg(body: => Unit): Unit = { + assume(icebergAvailable, "Iceberg not available in classpath") + assume(isSpark40Plus, "Comet's compaction runs on Spark 4.0 and later only") + withHadoopCatalog(catalog) { + withSQLConf( + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true")(body) + } + } + + private def appendFile(table: String, index: Int): Unit = + spark + .range(index * 50L, (index + 1) * 50L) + .selectExpr("CAST(id AS INT) AS id", "CAST(id AS STRING) AS name") + .coalesce(1) + .write + .format("iceberg") + .mode("append") + .saveAsTable(table) + + /** The batch that reads the table's files in bins of `filesPerBin` files each. */ + private def batchOf(icebergTable: AnyRef, filesPerBin: Int): CompactionBatch = { + val snapshotId = IcebergRewriteReflection.currentSnapshotId(icebergTable).get + val specId = IcebergRewriteReflection.currentSpecId(icebergTable) + val files = IcebergRewriteReflection.planFiles(spark, icebergTable, snapshotId, None, specId) + val bins = files.grouped(filesPerBin).toSeq.map { bin => + new BinTasks(new java.util.ArrayList[AnyRef](bin.map(_.task).asJava)) + } + val schema = IcebergReflection.getSchema(icebergTable).get.asInstanceOf[AnyRef] + new CompactionBatch(icebergTable, schema, IcebergReflection.toSparkSchema(schema), bins) + } + + test("the native scan reads each bin as a Spark partition") { + withNativeIceberg { + val table = s"$catalog.db.compaction_scan" + try { + sql(s"CREATE TABLE $table (id INT, name STRING) USING iceberg") + (0 until 4).foreach(appendFile(table, _)) + val icebergTable = loadIcebergTable(spark, catalog, "db", "compaction_scan") + + val id = UUID.randomUUID().toString + CompactionBatches.register(id, batchOf(icebergTable, filesPerBin = 2)) + try { + val df = spark.read + .format(classOf[CompactionSource].getName) + .option(CompactionSource.BatchIdOption, id) + .load() + + val nativeScans = df.queryExecution.executedPlan.collect { + case scan: CometIcebergNativeScanExec => scan + } + assert(nativeScans.size == 1, df.queryExecution.executedPlan.toString) + assert(nativeScans.head.numPartitions == 2) + checkAnswer(df, sql(s"SELECT * FROM $table")) + assert(df.count() == 200) + } finally { + CompactionBatches.remove(id) + } + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } + + test("the native scan reads a bin of one file as one partition") { + withNativeIceberg { + val table = s"$catalog.db.compaction_scan_files" + try { + sql(s"CREATE TABLE $table (id INT, name STRING) USING iceberg") + (0 until 3).foreach(appendFile(table, _)) + val icebergTable = loadIcebergTable(spark, catalog, "db", "compaction_scan_files") + + val id = UUID.randomUUID().toString + CompactionBatches.register(id, batchOf(icebergTable, filesPerBin = 1)) + try { + val df = spark.read + .format(classOf[CompactionSource].getName) + .option(CompactionSource.BatchIdOption, id) + .load() + val nativeScan = df.queryExecution.executedPlan.collectFirst { + case scan: CometIcebergNativeScanExec => scan + } + assert(nativeScan.exists(_.numPartitions == 3), df.queryExecution.executedPlan.toString) + checkAnswer(df, sql(s"SELECT * FROM $table")) + } finally { + CompactionBatches.remove(id) + } + } finally { + sql(s"DROP TABLE IF EXISTS $table") + } + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/iceberg/compaction/CompactionSourceSuite.scala b/spark/src/test/scala/org/apache/comet/iceberg/compaction/CompactionSourceSuite.scala new file mode 100644 index 00000000000..bb4862bd7fb --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/iceberg/compaction/CompactionSourceSuite.scala @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.iceberg.compaction + +import java.util.UUID + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.CometTestBase +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec +import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} + +/** + * Tests the Spark side of the compaction source: how a registered batch reads through + * `DataFrameReader`, without Iceberg. What Comet's native scan makes of it is covered by + * `CometIcebergCompactionSuite`. + */ +class CompactionSourceSuite extends CometTestBase { + + private val schema = StructType( + Seq(StructField("id", IntegerType, nullable = false), StructField("name", StringType))) + + private def newBatch(bins: Seq[String]*): CompactionBatch = { + val tasks = bins.map { bin => + new BinTasks(new java.util.ArrayList[AnyRef](bin.map(_.asInstanceOf[AnyRef]).asJava)) + } + new CompactionBatch("table", "iceberg schema", schema, tasks) + } + + private def withBatch(batch: CompactionBatch)(body: String => Unit): Unit = { + val id = UUID.randomUUID().toString + CompactionBatches.register(id, batch) + try body(id) + finally CompactionBatches.remove(id) + } + + private def read(id: String) = + spark.read + .format(classOf[CompactionSource].getName) + .option(CompactionSource.BatchIdOption, id) + .load() + + test("a batch reads with its schema and one Spark partition per bin") { + withBatch(newBatch(Seq("a", "b"), Seq("c"), Seq("d", "e", "f"))) { id => + val df = read(id) + assert( + df.schema.map(f => (f.name, f.dataType, f.nullable)) == + schema.map(f => (f.name, f.dataType, f.nullable))) + + val scans = df.queryExecution.sparkPlan.collect { case scan: BatchScanExec => scan } + assert(scans.size == 1) + assert(scans.head.scan.isInstanceOf[CompactionScan]) + assert(scans.head.inputRDD.getNumPartitions == 3) + } + } + + test("a bin is the task group of its partition") { + withBatch(newBatch(Seq("a", "b"), Seq("c"))) { id => + val scan = read(id).queryExecution.sparkPlan.collect { case s: BatchScanExec => s }.head + val partitions = scan.scan.toBatch.planInputPartitions() + val bins = partitions.map(_.asInstanceOf[CompactionInputPartition].taskGroup.tasks.asScala) + assert(bins.map(_.toSeq).toSeq == Seq(Seq("a", "b"), Seq("c"))) + } + } + + test("the scan exposes the accessors Comet reads off an Iceberg scan") { + val batch = newBatch(Seq("a", "b"), Seq("c")) + val scan = new CompactionScan(batch) + assert(scan.table() == "table") + assert(scan.expectedSchema() == "iceberg schema") + assert(scan.tasks().asScala.toSeq == Seq("a", "b", "c")) + assert(scan.readSchema() == schema) + // Comet finds these by name, so a rename breaks the native scan without breaking the build. + Seq("table", "expectedSchema", "tasks").foreach { name => + assert(classOf[CompactionScan].getMethods.exists(_.getName == name), name) + } + assert(classOf[CompactionInputPartition].getMethods.exists(_.getName == "taskGroup")) + assert(classOf[BinTasks].getMethods.exists(_.getName == "tasks")) + } + + test("scans of one batch are equal and scans of two batches are not") { + val batch = newBatch(Seq("a")) + assert(new CompactionScan(batch) == new CompactionScan(batch)) + assert(new CompactionScan(batch).hashCode == new CompactionScan(batch).hashCode) + assert(new CompactionScan(batch) != new CompactionScan(newBatch(Seq("a")))) + } + + test("a batch that is not registered cannot be read") { + val error = intercept[Exception](read("no-such-batch")) + val messages = + Iterator.iterate[Throwable](error)(_.getCause).takeWhile(_ != null).map(_.getMessage) + assert(messages.exists(m => m != null && m.contains("'no-such-batch'"))) + } + + test("Spark itself cannot read a compaction scan") { + withBatch(newBatch(Seq("a"))) { id => + val scan = read(id).queryExecution.sparkPlan.collect { case s: BatchScanExec => s }.head + val factory = scan.scan.toBatch.createReaderFactory() + val partition = scan.scan.toBatch.planInputPartitions().head + intercept[UnsupportedOperationException](factory.createReader(partition)) + } + } +}