From 9961592770c6ebca7d8b04803e25b7599e4f9c15 Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Thu, 1 Oct 2026 11:15:27 +0100 Subject: [PATCH 1/5] [FLINK-40896][table] Add built-in DEDUPLICATE_KEEP_FIRST process table function --- .../docs/sql/functions/built-in-functions.md | 1 + .../queries/deduplicate-keep-first.md | 186 ++++++++ .../docs/sql/functions/built-in-functions.md | 1 + .../queries/deduplicate-keep-first.md | 186 ++++++++ .../functions/BuiltInFunctionDefinition.java | 8 + .../functions/BuiltInFunctionDefinitions.java | 28 ++ .../DeduplicateKeepFirstTypeStrategy.java | 188 ++++++++ .../SpecificInputTypeStrategies.java | 4 + .../strategies/SpecificTypeStrategies.java | 4 + .../DeduplicateKeepFirstTypeStrategyTest.java | 66 +++ .../DeduplicateKeepFirstRestoreTest.java | 65 +++ ...duplicateKeepFirstRestoreTestPrograms.java | 94 ++++ .../DeduplicateKeepFirstSemanticTests.java | 59 +++ .../DeduplicateKeepFirstTestPrograms.java | 430 ++++++++++++++++++ ...keep-first-event-time-pending-restore.json | 352 ++++++++++++++ .../savepoint/_metadata | Bin 0 -> 17098 bytes .../deduplicate-keep-first-keyed-restore.json | 144 ++++++ .../savepoint/_metadata | Bin 0 -> 16582 bytes .../ptf/BuiltInProcessTableFunction.java | 35 +- .../functions/ptf/DeduplicateKeepFirst.java | 165 +++++++ 20 files changed, 2014 insertions(+), 2 deletions(-) create mode 100644 docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md create mode 100644 docs/content/docs/sql/reference/queries/deduplicate-keep-first.md create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java create mode 100644 flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/plan/deduplicate-keep-first-event-time-pending-restore.json create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/savepoint/_metadata create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-keyed-restore/plan/deduplicate-keep-first-keyed-restore.json create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-keyed-restore/savepoint/_metadata create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java diff --git a/docs/content.zh/docs/sql/functions/built-in-functions.md b/docs/content.zh/docs/sql/functions/built-in-functions.md index 242cb4b03582e8..9b01e5858cb40f 100644 --- a/docs/content.zh/docs/sql/functions/built-in-functions.md +++ b/docs/content.zh/docs/sql/functions/built-in-functions.md @@ -289,6 +289,7 @@ Table functions can be used in two ways: as stand-alone inputs, where they are i | `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See Changelog Conversion for the full list of arguments, semantics, and usage. | | `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See Changelog Conversion for the full list of arguments, semantics, and usage. | | `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | +| `DEDUPLICATE_KEEP_FIRST(input => TABLE t [, ...])` | Keeps the first record per key as an insert-only stream (first arrival, or earliest event time with `on_time`). See [Deduplicate Keep First]({{< ref "docs/sql/reference/queries/deduplicate-keep-first" >}}#deduplicate_keep_first) for the full list of arguments, semantics, and usage. | To implement your own table functions, see [user-defined table functions]({{< ref "docs/dev/table/functions/udfs" >}}#table-functions). diff --git a/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md b/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md new file mode 100644 index 00000000000000..1e08355875975c --- /dev/null +++ b/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md @@ -0,0 +1,186 @@ +--- +title: "Deduplicate Keep First" +weight: 16 +type: docs +--- + + +# Deduplicate Keep First + +{{< label Streaming >}} + +Flink SQL provides the `DEDUPLICATE_KEEP_FIRST` process table function (PTF) for removing duplicate rows while keeping only the **first** record per key. Its output is always **insert-only**, which makes it a concise, append-in / append-out alternative to the `ROW_NUMBER()` [deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}) pattern. + +| Function | Description | +|:---------|:------------| +| [DEDUPLICATE_KEEP_FIRST](#deduplicate_keep_first) | Keeps the first record per key as an insert-only stream (first arrival, or earliest event time with `on_time`) | + +## DEDUPLICATE_KEEP_FIRST + +The `DEDUPLICATE_KEEP_FIRST` PTF keeps the first record per key and drops every later record for that key. It supports two ordering modes: + +* **Processing time (default):** without `on_time`, the first record *observed* for a key is emitted; later records for that key are dropped while its state exists. This requires no watermark or event-time attribute. +* **Event time:** with `on_time`, the record with the *smallest event time* per key is kept and emitted once the watermark makes the choice final. A record arriving later but carrying an earlier event time replaces the candidate before finalization; records that can no longer become the earliest are dropped as late. + +The input may be insert-only or updating; the output is insert-only in every case. + +### Syntax + +```sql +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE source_table [PARTITION BY key_col [, key_col2 ...]], + [on_time => DESCRIPTOR(rowtime_column),] + [state_ttl => ,] + [reset_ttl_on_duplicate => ] +) +``` + +### Parameters + +| Parameter | Required | Description | +|:----------|:---------|:------------| +| `input` | Yes | The input table (insert-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | +| `on_time` | No | A `DESCRIPTOR` naming a single rowtime attribute. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires insert-only input; combining `on_time` with an updating input is rejected at planning time. | +| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. Defaults to no TTL (state retained indefinitely). After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. `INTERVAL '0'` disables retention. | +| `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful together with `state_ttl`. | + +### Output Schema + +When `PARTITION BY` is used, the partition key columns are prepended to the output, followed by the remaining (non-key) input columns. In event-time mode a rowtime column is appended. + +``` +[partition_key_columns] + [remaining_input_columns] +``` + +### Examples + +#### Keyed keep-first (processing time) + +```sql +-- Input (append-only): +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'click'] + +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events PARTITION BY user_name +) + +-- Output (insert-only): +-- +I[user_name:'Vas', action:'login'] +``` + +The first row for `Vas` is emitted; the later `click` on the same key is dropped. + +#### Whole input, no PARTITION BY + +```sql +-- Input (append-only): +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Alice', action:'click'] +-- +I[user_name:'Bob', action:'view'] + +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events +) + +-- Output (insert-only): +-- +I[user_name:'Vas', action:'login'] +``` + +Without `PARTITION BY` the whole input is a single group at parallelism 1, so only the first row of the entire stream is emitted; every later row is dropped regardless of its content. + +#### Bounded state with state_ttl + +```sql +-- Input (append-only): +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'click'] +-- +I[user_name:'Vas', action:'view'] + +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events PARTITION BY user_name, + state_ttl => INTERVAL '5' SECOND, + reset_ttl_on_duplicate => FALSE +) + +-- Output (insert-only): +-- +I[user_name:'Vas', action:'login'] +``` + +`state_ttl` bounds how long per-key state is retained (processing time). While the state exists, duplicates are dropped and only the first row per key is emitted; after the state expires, a later record for the key starts a new deduplication period. With the default `reset_ttl_on_duplicate => TRUE`, each dropped duplicate refreshes the retention window; with `FALSE`, retention is measured from the first record for the key. + +#### Earliest by event time + +```sql +-- Source declares: WATERMARK FOR ts AS ts - INTERVAL '10' SECOND +-- Input (ts = event time): +-- +I[user_name:'Vas', action:'login', ts:3.000] +-- +I[user_name:'Vas', action:'click', ts:1.000] +-- +I[user_name:'Vas', action:'view', ts:5.000] + +SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events PARTITION BY user_name, + on_time => DESCRIPTOR(ts) +) + +-- Output (insert-only, once the watermark finalizes the earliest): +-- +I[user_name:'Vas', action:'click'] +``` + +With `on_time`, the record with the smallest event time per key is kept (here `click` at `ts 1.000`, not `login` which arrived first) and emitted once the watermark passes that timestamp. + +#### Updating input + +```sql +-- Input (updating changelog): +-- +I[user_name:'Vas', action:'login'] first record for the key -> emitted +-- +I[user_name:'Vas', action:'login'] duplicate -> swallowed +-- -U[user_name:'Vas', action:'login'] retraction -> swallowed +-- +U[user_name:'Vas', action:'click'] update -> swallowed +-- -D[user_name:'Vas', action:'click'] delete -> swallowed + +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events PARTITION BY user_name +) + +-- Output (insert-only): +-- +I[user_name:'Vas', action:'login'] +``` + +The input may be an updating changelog. `DEDUPLICATE_KEEP_FIRST` keeps the first record observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. Event-time mode (`on_time`) is not supported with updating input. + +#### Table API + +```java +Table userEvents = ...; + +// Keep the first row per key (processing time). +Table result = userEvents.partitionBy($("user_name")).process("DEDUPLICATE_KEEP_FIRST"); + +// With optional arguments. +Table result = userEvents + .partitionBy($("user_name")) + .process( + "DEDUPLICATE_KEEP_FIRST", + lit(Duration.ofSeconds(5)).asArgument("state_ttl"), + lit(false).asArgument("reset_ttl_on_duplicate")); +``` + +{{< top >}} diff --git a/docs/content/docs/sql/functions/built-in-functions.md b/docs/content/docs/sql/functions/built-in-functions.md index 40b54b2869b29c..fe0a84c8a0bb2c 100644 --- a/docs/content/docs/sql/functions/built-in-functions.md +++ b/docs/content/docs/sql/functions/built-in-functions.md @@ -292,6 +292,7 @@ Table functions can be used in two ways: as stand-alone inputs, where they are i | `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#from_changelog) for the full list of arguments, semantics, and usage. | | `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#to_changelog) for the full list of arguments, semantics, and usage. | | `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | +| `DEDUPLICATE_KEEP_FIRST(input => TABLE t [, ...])` | Keeps the first record per key as an insert-only stream (first arrival, or earliest event time with `on_time`). See [Deduplicate Keep First]({{< ref "docs/sql/reference/queries/deduplicate-keep-first" >}}#deduplicate_keep_first) for the full list of arguments, semantics, and usage. | To implement your own table functions, see [user-defined table functions]({{< ref "docs/dev/table/functions/udfs" >}}#table-functions). diff --git a/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md b/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md new file mode 100644 index 00000000000000..1e08355875975c --- /dev/null +++ b/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md @@ -0,0 +1,186 @@ +--- +title: "Deduplicate Keep First" +weight: 16 +type: docs +--- + + +# Deduplicate Keep First + +{{< label Streaming >}} + +Flink SQL provides the `DEDUPLICATE_KEEP_FIRST` process table function (PTF) for removing duplicate rows while keeping only the **first** record per key. Its output is always **insert-only**, which makes it a concise, append-in / append-out alternative to the `ROW_NUMBER()` [deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}) pattern. + +| Function | Description | +|:---------|:------------| +| [DEDUPLICATE_KEEP_FIRST](#deduplicate_keep_first) | Keeps the first record per key as an insert-only stream (first arrival, or earliest event time with `on_time`) | + +## DEDUPLICATE_KEEP_FIRST + +The `DEDUPLICATE_KEEP_FIRST` PTF keeps the first record per key and drops every later record for that key. It supports two ordering modes: + +* **Processing time (default):** without `on_time`, the first record *observed* for a key is emitted; later records for that key are dropped while its state exists. This requires no watermark or event-time attribute. +* **Event time:** with `on_time`, the record with the *smallest event time* per key is kept and emitted once the watermark makes the choice final. A record arriving later but carrying an earlier event time replaces the candidate before finalization; records that can no longer become the earliest are dropped as late. + +The input may be insert-only or updating; the output is insert-only in every case. + +### Syntax + +```sql +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE source_table [PARTITION BY key_col [, key_col2 ...]], + [on_time => DESCRIPTOR(rowtime_column),] + [state_ttl => ,] + [reset_ttl_on_duplicate => ] +) +``` + +### Parameters + +| Parameter | Required | Description | +|:----------|:---------|:------------| +| `input` | Yes | The input table (insert-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | +| `on_time` | No | A `DESCRIPTOR` naming a single rowtime attribute. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires insert-only input; combining `on_time` with an updating input is rejected at planning time. | +| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. Defaults to no TTL (state retained indefinitely). After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. `INTERVAL '0'` disables retention. | +| `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful together with `state_ttl`. | + +### Output Schema + +When `PARTITION BY` is used, the partition key columns are prepended to the output, followed by the remaining (non-key) input columns. In event-time mode a rowtime column is appended. + +``` +[partition_key_columns] + [remaining_input_columns] +``` + +### Examples + +#### Keyed keep-first (processing time) + +```sql +-- Input (append-only): +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'click'] + +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events PARTITION BY user_name +) + +-- Output (insert-only): +-- +I[user_name:'Vas', action:'login'] +``` + +The first row for `Vas` is emitted; the later `click` on the same key is dropped. + +#### Whole input, no PARTITION BY + +```sql +-- Input (append-only): +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Alice', action:'click'] +-- +I[user_name:'Bob', action:'view'] + +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events +) + +-- Output (insert-only): +-- +I[user_name:'Vas', action:'login'] +``` + +Without `PARTITION BY` the whole input is a single group at parallelism 1, so only the first row of the entire stream is emitted; every later row is dropped regardless of its content. + +#### Bounded state with state_ttl + +```sql +-- Input (append-only): +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'click'] +-- +I[user_name:'Vas', action:'view'] + +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events PARTITION BY user_name, + state_ttl => INTERVAL '5' SECOND, + reset_ttl_on_duplicate => FALSE +) + +-- Output (insert-only): +-- +I[user_name:'Vas', action:'login'] +``` + +`state_ttl` bounds how long per-key state is retained (processing time). While the state exists, duplicates are dropped and only the first row per key is emitted; after the state expires, a later record for the key starts a new deduplication period. With the default `reset_ttl_on_duplicate => TRUE`, each dropped duplicate refreshes the retention window; with `FALSE`, retention is measured from the first record for the key. + +#### Earliest by event time + +```sql +-- Source declares: WATERMARK FOR ts AS ts - INTERVAL '10' SECOND +-- Input (ts = event time): +-- +I[user_name:'Vas', action:'login', ts:3.000] +-- +I[user_name:'Vas', action:'click', ts:1.000] +-- +I[user_name:'Vas', action:'view', ts:5.000] + +SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events PARTITION BY user_name, + on_time => DESCRIPTOR(ts) +) + +-- Output (insert-only, once the watermark finalizes the earliest): +-- +I[user_name:'Vas', action:'click'] +``` + +With `on_time`, the record with the smallest event time per key is kept (here `click` at `ts 1.000`, not `login` which arrived first) and emitted once the watermark passes that timestamp. + +#### Updating input + +```sql +-- Input (updating changelog): +-- +I[user_name:'Vas', action:'login'] first record for the key -> emitted +-- +I[user_name:'Vas', action:'login'] duplicate -> swallowed +-- -U[user_name:'Vas', action:'login'] retraction -> swallowed +-- +U[user_name:'Vas', action:'click'] update -> swallowed +-- -D[user_name:'Vas', action:'click'] delete -> swallowed + +SELECT * FROM DEDUPLICATE_KEEP_FIRST( + input => TABLE user_events PARTITION BY user_name +) + +-- Output (insert-only): +-- +I[user_name:'Vas', action:'login'] +``` + +The input may be an updating changelog. `DEDUPLICATE_KEEP_FIRST` keeps the first record observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. Event-time mode (`on_time`) is not supported with updating input. + +#### Table API + +```java +Table userEvents = ...; + +// Keep the first row per key (processing time). +Table result = userEvents.partitionBy($("user_name")).process("DEDUPLICATE_KEEP_FIRST"); + +// With optional arguments. +Table result = userEvents + .partitionBy($("user_name")) + .process( + "DEDUPLICATE_KEEP_FIRST", + lit(Duration.ofSeconds(5)).asArgument("state_ttl"), + lit(false).asArgument("reset_ttl_on_duplicate")); +``` + +{{< top >}} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinition.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinition.java index 03e4780e0183d0..a81de868f34233 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinition.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinition.java @@ -23,6 +23,7 @@ import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.inference.InputTypeStrategy; +import org.apache.flink.table.types.inference.StateTypeStrategy; import org.apache.flink.table.types.inference.StaticArgument; import org.apache.flink.table.types.inference.TypeInference; import org.apache.flink.table.types.inference.TypeStrategy; @@ -31,6 +32,7 @@ import java.lang.reflect.Constructor; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.Locale; import java.util.Optional; import java.util.regex.Pattern; @@ -345,6 +347,12 @@ public Builder outputTypeStrategy(TypeStrategy outputTypeStrategy) { return this; } + public Builder stateTypeStrategies( + LinkedHashMap stateTypeStrategies) { + this.typeInferenceBuilder.stateTypeStrategies(stateTypeStrategies); + return this; + } + public Builder notDeterministic() { this.isDeterministic = false; return this; diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index 9e993e2642991d..9d1e8e1ca221cf 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -38,6 +38,7 @@ import org.apache.flink.table.types.inference.TraitCondition; import org.apache.flink.table.types.inference.TypeStrategies; import org.apache.flink.table.types.inference.strategies.ArrayOfStringArgumentTypeStrategy; +import org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy; import org.apache.flink.table.types.inference.strategies.FromChangelogTypeStrategy; import org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies; import org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies; @@ -108,6 +109,7 @@ import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.ARRAY_ELEMENT_ARG; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.ARRAY_FULLY_COMPARABLE; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.ARRAY_OF_ENTRIES_ARG; +import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.DEDUPLICATE_KEEP_FIRST_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.FROM_CHANGELOG_INPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.INDEX; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.JSON_ARGUMENT; @@ -121,6 +123,7 @@ import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.percentageArray; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.plainJsonPath; import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.ARRAY_APPEND_PREPEND; +import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.DEDUPLICATE_KEEP_FIRST_OUTPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.FROM_CHANGELOG_OUTPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.LATERAL_SNAPSHOT_OUTPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.ML_PREDICT_OUTPUT_TYPE_STRATEGY; @@ -939,6 +942,31 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) "org.apache.flink.table.runtime.functions.ptf.FromChangelogFunction") .build(); + public static final BuiltInFunctionDefinition DEDUPLICATE_KEEP_FIRST = + BuiltInFunctionDefinition.newBuilder() + .name("DEDUPLICATE_KEEP_FIRST") + .kind(PROCESS_TABLE) + .staticArguments( + StaticArgument.table( + "input", + Row.class, + false, + EnumSet.of( + StaticArgumentTrait.TABLE, + StaticArgumentTrait.SET_SEMANTIC_TABLE, + StaticArgumentTrait.OPTIONAL_PARTITION_BY, + StaticArgumentTrait.SUPPORT_UPDATES)), + StaticArgument.scalar( + "state_ttl", DataTypes.INTERVAL(DataTypes.SECOND()), true), + StaticArgument.scalar( + "reset_ttl_on_duplicate", DataTypes.BOOLEAN(), true)) + .inputTypeStrategy(DEDUPLICATE_KEEP_FIRST_TYPE_STRATEGY) + .outputTypeStrategy(DEDUPLICATE_KEEP_FIRST_OUTPUT_TYPE_STRATEGY) + .stateTypeStrategies(DeduplicateKeepFirstTypeStrategy.stateTypeStrategies()) + .runtimeClass( + "org.apache.flink.table.runtime.functions.ptf.DeduplicateKeepFirst") + .build(); + /** * Built-in proxy function for the LATERAL SNAPSHOT temporal join. * diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java new file mode 100644 index 00000000000000..4860a73a497e07 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.inference.strategies; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.DataTypes.Field; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.api.dataview.ValueView; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.InputTypeStrategy; +import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.TypeStrategy; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class DeduplicateKeepFirstTypeStrategy { + + public static final int ARG_INPUT = 0; + public static final int ARG_STATE_TTL = 1; + public static final int ARG_RESET_TTL_ON_DUPLICATE = 2; + + public static final InputTypeStrategy INPUT_TYPE_STRATEGY = + new ValidationOnlyInputTypeStrategy() { + @Override + public Optional> inferInputTypes( + final CallContext callContext, final boolean throwOnFailure) { + final Optional> stateTtlError = + validateStateTtl(callContext, throwOnFailure); + if (stateTtlError.isPresent()) { + return stateTtlError; + } + + final Optional> resetTtlError = + validateResetTtlOnDuplicate(callContext, throwOnFailure); + if (resetTtlError.isPresent()) { + return resetTtlError; + } + + return Optional.of(callContext.getArgumentDataTypes()); + } + }; + + public static final TypeStrategy OUTPUT_TYPE_STRATEGY = + callContext -> { + final TableSemantics tableSemantics = + callContext + .getTableSemantics(ARG_INPUT) + .orElseThrow( + () -> + new ValidationException( + "First argument must be a table for DEDUPLICATE_KEEP_FIRST.")); + + final List inputFields = DataType.getFields(tableSemantics.dataType()); + final List outputFields = + Arrays.stream( + ChangelogTypeStrategyUtils.computeOutputIndices( + tableSemantics)) + .mapToObj(inputFields::get) + .collect(Collectors.toList()); + return Optional.of(DataTypes.ROW(outputFields).notNull()); + }; + + private static final String SEEN_STATE_NAME = "seen"; + + public static final StateTypeStrategy SEEN_STATE_TYPE_STRATEGY = + new StateTypeStrategy() { + @Override + public Optional inferType(final CallContext callContext) { + return Optional.of( + ValueView.newValueViewDataType(DataTypes.BOOLEAN().notNull())); + } + + @Override + public Optional getTimeToLive(final CallContext callContext) { + return callContext.getArgumentValue(ARG_STATE_TTL, Duration.class); + } + }; + + private static final String CANDIDATE_STATE_NAME = "candidate"; + + public static final StateTypeStrategy CANDIDATE_STATE_TYPE_STRATEGY = + new StateTypeStrategy() { + @Override + public Optional inferType(final CallContext callContext) { + final TableSemantics tableSemantics = + callContext + .getTableSemantics(ARG_INPUT) + .orElseThrow( + () -> + new ValidationException( + "First argument must be a table for DEDUPLICATE_KEEP_FIRST.")); + final List inputFields = DataType.getFields(tableSemantics.dataType()); + final List candidateFields = + Arrays.stream( + ChangelogTypeStrategyUtils.computeOutputIndices( + tableSemantics)) + .mapToObj(inputFields::get) + .collect(Collectors.toCollection(ArrayList::new)); + // The trailing timestamp field is read positionally at runtime, so its name + // only needs to avoid clashing with a payload column of the same name. + final Set takenNames = + candidateFields.stream() + .map(Field::getName) + .collect(Collectors.toSet()); + String timestampField = "event_time"; + for (int i = 0; takenNames.contains(timestampField); i++) { + timestampField = "event_time_" + i; + } + candidateFields.add(DataTypes.FIELD(timestampField, DataTypes.BIGINT())); + return Optional.of( + ValueView.newValueViewDataType(DataTypes.ROW(candidateFields))); + } + + @Override + public Optional getTimeToLive(final CallContext callContext) { + // The candidate buffer is cleared by its timer, not by retention; disable its + // TTL so a global table.exec.state.ttl cannot expire it before the timer fires. + return Optional.of(Duration.ZERO); + } + }; + + public static LinkedHashMap stateTypeStrategies() { + final LinkedHashMap strategies = new LinkedHashMap<>(); + strategies.put(SEEN_STATE_NAME, SEEN_STATE_TYPE_STRATEGY); + strategies.put(CANDIDATE_STATE_NAME, CANDIDATE_STATE_TYPE_STRATEGY); + return strategies; + } + + private static Optional> validateStateTtl( + final CallContext callContext, final boolean throwOnFailure) { + if (callContext.isArgumentNull(ARG_STATE_TTL)) { + return Optional.empty(); + } + if (!callContext.isArgumentLiteral(ARG_STATE_TTL)) { + return callContext.fail( + throwOnFailure, + "The 'state_ttl' argument must be a constant INTERVAL literal."); + } + final Optional stateTtl = + callContext.getArgumentValue(ARG_STATE_TTL, Duration.class); + if (stateTtl.isPresent() && stateTtl.get().isNegative()) { + return callContext.fail( + throwOnFailure, + "The 'state_ttl' argument must not be negative, but was '%s'. " + + "Use INTERVAL '0' to disable state retention.", + stateTtl.get()); + } + return Optional.empty(); + } + + private static Optional> validateResetTtlOnDuplicate( + final CallContext callContext, final boolean throwOnFailure) { + if (callContext.isArgumentNull(ARG_RESET_TTL_ON_DUPLICATE)) { + return Optional.empty(); + } + if (!callContext.isArgumentLiteral(ARG_RESET_TTL_ON_DUPLICATE)) { + return callContext.fail( + throwOnFailure, + "The 'reset_ttl_on_duplicate' argument must be a constant BOOLEAN literal."); + } + return Optional.empty(); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java index e012f310953b3b..066f9b38cf7705 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java @@ -142,6 +142,10 @@ public static InputTypeStrategy plainJsonPath(final InputTypeStrategy signatures public static final InputTypeStrategy FROM_CHANGELOG_INPUT_TYPE_STRATEGY = FromChangelogTypeStrategy.INPUT_TYPE_STRATEGY; + /** Input strategy for {@link BuiltInFunctionDefinitions#DEDUPLICATE_KEEP_FIRST}. */ + public static final InputTypeStrategy DEDUPLICATE_KEEP_FIRST_TYPE_STRATEGY = + DeduplicateKeepFirstTypeStrategy.INPUT_TYPE_STRATEGY; + /** Input strategy for {@link BuiltInFunctionDefinitions#SNAPSHOT}. */ public static final InputTypeStrategy LATERAL_SNAPSHOT_INPUT_TYPE_STRATEGY = LateralSnapshotTypeStrategy.INPUT_TYPE_STRATEGY; diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificTypeStrategies.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificTypeStrategies.java index 00a82abb95d95c..83b522e3c512ae 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificTypeStrategies.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificTypeStrategies.java @@ -243,6 +243,10 @@ public final class SpecificTypeStrategies { public static final TypeStrategy FROM_CHANGELOG_OUTPUT_TYPE_STRATEGY = FromChangelogTypeStrategy.OUTPUT_TYPE_STRATEGY; + /** Type strategy specific for {@link BuiltInFunctionDefinitions#DEDUPLICATE_KEEP_FIRST}. */ + public static final TypeStrategy DEDUPLICATE_KEEP_FIRST_OUTPUT_TYPE_STRATEGY = + DeduplicateKeepFirstTypeStrategy.OUTPUT_TYPE_STRATEGY; + /** Type strategy specific for {@link BuiltInFunctionDefinitions#SNAPSHOT}. */ public static final TypeStrategy LATERAL_SNAPSHOT_OUTPUT_TYPE_STRATEGY = LateralSnapshotTypeStrategy.OUTPUT_TYPE_STRATEGY; diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java new file mode 100644 index 00000000000000..81a675c35763d0 --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.inference.strategies; + +import org.apache.flink.table.types.inference.utils.CallContextMock; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE; +import static org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy.ARG_STATE_TTL; +import static org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy.CANDIDATE_STATE_TYPE_STRATEGY; +import static org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy.SEEN_STATE_TYPE_STRATEGY; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for the state retention contract of {@link DeduplicateKeepFirstTypeStrategy}. */ +class DeduplicateKeepFirstTypeStrategyTest { + + @Test + void testCandidateStateDisablesTtl() { + // The event-time candidate buffer is cleared by its timer, so it must declare a disabled + // TTL rather than inheriting the global state TTL and expiring before the timer fires. + assertThat(CANDIDATE_STATE_TYPE_STRATEGY.getTimeToLive(callContext(Duration.ofSeconds(5)))) + .contains(Duration.ZERO); + assertThat(CANDIDATE_STATE_TYPE_STRATEGY.getTimeToLive(callContext(null))) + .contains(Duration.ZERO); + } + + @Test + void testSeenStateTtlFollowsStateTtlArgument() { + assertThat(SEEN_STATE_TYPE_STRATEGY.getTimeToLive(callContext(Duration.ofSeconds(5)))) + .contains(Duration.ofSeconds(5)); + assertThat(SEEN_STATE_TYPE_STRATEGY.getTimeToLive(callContext(null))).isEmpty(); + } + + private static CallContextMock callContext(Duration stateTtl) { + final CallContextMock callContext = new CallContextMock(); + final List> values = new ArrayList<>(); + for (int i = 0; i <= ARG_RESET_TTL_ON_DUPLICATE; i++) { + values.add(Optional.empty()); + } + values.set(ARG_STATE_TTL, Optional.ofNullable(stateTtl)); + callContext.argumentValues = values; + return callContext; + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java new file mode 100644 index 00000000000000..fa9ad95f08559e --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.planner.factories.TestValuesTableFactory; +import org.apache.flink.table.planner.plan.nodes.exec.testutils.RestoreTestBase; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** Restore tests for the built-in DEDUPLICATE_KEEP_FIRST PTF. */ +public class DeduplicateKeepFirstRestoreTest extends RestoreTestBase { + + // Bounds the wait for the pending-candidate savepoint trigger so a stuck source fails fast. + private static final long SAVEPOINT_READY_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); + + public DeduplicateKeepFirstRestoreTest() { + super(StreamExecProcessTableFunction.class); + } + + @Override + public List programs() { + return List.of( + DeduplicateKeepFirstRestoreTestPrograms.KEYED_RESTORE, + DeduplicateKeepFirstRestoreTestPrograms.EVENT_TIME_PENDING_RESTORE); + } + + @Override + protected void awaitSavepointReady(TableTestProgram program, List> futures) + throws Exception { + if (program != DeduplicateKeepFirstRestoreTestPrograms.EVENT_TIME_PENDING_RESTORE) { + super.awaitSavepointReady(program, futures); + return; + } + // The candidate is buffered but not emitted before restore, so there is no sink output to + // gate on. Instead wait until the source has emitted its before-restore rows; + // stop-with-savepoint then drains them into keyed state before snapshotting. + for (SourceTestStep source : program.getSetupSourceTestSteps()) { + final int count = source.dataBeforeRestore.size(); + if (count > 0) { + TestValuesTableFactory.awaitSourceEmitted(source.name, count) + .get(SAVEPOINT_READY_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } + } + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java new file mode 100644 index 00000000000000..401ddac82e3b0a --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + +import java.time.Instant; + +/** Restore {@link TableTestProgram}s for the built-in DEDUPLICATE_KEEP_FIRST PTF. */ +public class DeduplicateKeepFirstRestoreTestPrograms { + + private static final String[] USER_EVENTS_SCHEMA = {"user_name STRING", "action STRING"}; + + private static final String[] TIMED_EVENTS_SCHEMA = { + "user_name STRING", + "action STRING", + "ts TIMESTAMP_LTZ(3)", + "WATERMARK FOR ts AS ts - INTERVAL '0.001' SECOND" + }; + + public static final TableTestProgram KEYED_RESTORE = + TableTestProgram.of( + "deduplicate-keep-first-keyed-restore", + "the keep-first 'seen' state restores via compiled plan + savepoint: " + + "an already-seen key stays deduplicated after restore, and a " + + "key first observed after restore is emitted") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedBeforeRestore( + Row.of("Vas", "login"), Row.of("Vas", "click")) + .producedAfterRestore( + Row.of("Vas", "logout"), Row.of("Alice", "hello")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedBeforeRestore( + Row.ofKind(RowKind.INSERT, "Vas", "login")) + .consumedAfterRestore( + Row.ofKind(RowKind.INSERT, "Alice", "hello")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name)") + .build(); + + public static final TableTestProgram EVENT_TIME_PENDING_RESTORE = + TableTestProgram.of( + "deduplicate-keep-first-event-time-pending-restore", + "an event-time candidate buffered but not yet finalized (its timer has " + + "not fired) survives the savepoint: after restore, the " + + "watermark advances and the buffered candidate is emitted") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(TIMED_EVENTS_SCHEMA) + .producedBeforeRestore( + Row.of("Bob", "first", Instant.ofEpochMilli(5))) + .producedAfterRestore( + Row.of("Bob", "later", Instant.ofEpochMilli(100))) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedBeforeRestore(new Row[0]) + .consumedAfterRestore( + Row.ofKind(RowKind.INSERT, "Bob", "first")) + .build()) + .runSql( + "INSERT INTO sink SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + "on_time => DESCRIPTOR(ts))") + .build(); +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java new file mode 100644 index 00000000000000..7bbe0f0848d734 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.config.OptimizerConfigOptions; +import org.apache.flink.table.planner.plan.nodes.exec.testutils.SemanticTestBase; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.List; + +/** Semantic tests for the built-in DEDUPLICATE_KEEP_FIRST process table function. */ +public class DeduplicateKeepFirstSemanticTests extends SemanticTestBase { + + @Override + protected void applyDefaultEnvironmentOptions(TableConfig config) { + super.applyDefaultEnvironmentOptions(config); + config.set( + OptimizerConfigOptions.TABLE_OPTIMIZER_NONDETERMINISTIC_UPDATE_STRATEGY, + OptimizerConfigOptions.NonDeterministicUpdateStrategy.IGNORE); + } + + @Override + public List programs() { + return List.of( + DeduplicateKeepFirstTestPrograms.KEYED_KEEP_FIRST, + DeduplicateKeepFirstTestPrograms.KEYED_KEEP_FIRST_TABLE_API, + DeduplicateKeepFirstTestPrograms.KEYED_KEEP_FIRST_TABLE_API_WITH_ARGS, + DeduplicateKeepFirstTestPrograms.MULTI_KEY_INDEPENDENCE, + DeduplicateKeepFirstTestPrograms.MULTI_COLUMN_KEY, + DeduplicateKeepFirstTestPrograms.NULL_PARTITION_KEY, + DeduplicateKeepFirstTestPrograms.NO_PARTITION_BY, + DeduplicateKeepFirstTestPrograms.RESET_ON_TTL_OFF, + DeduplicateKeepFirstTestPrograms.RESET_TTL_ON_DUPLICATE_FALSE, + DeduplicateKeepFirstTestPrograms.INPUT_COLUMN_NAMED_EVENT_TIME, + DeduplicateKeepFirstTestPrograms.WHOLE_ROW_AS_KEY, + DeduplicateKeepFirstTestPrograms.EVENT_TIME_KEEP_EARLIEST, + DeduplicateKeepFirstTestPrograms.EVENT_TIME_LATER_ARRIVAL_EARLIER_WINS, + DeduplicateKeepFirstTestPrograms.EVENT_TIME_LATE_EVENT_DROPPED, + DeduplicateKeepFirstTestPrograms.UPDATING_INPUT_SWALLOWED, + DeduplicateKeepFirstTestPrograms.ON_TIME_WITH_UPDATING_INPUT_FAILS); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java new file mode 100644 index 00000000000000..617ef192225af2 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + +import java.time.Duration; +import java.time.Instant; + +import static org.apache.flink.table.api.Expressions.$; +import static org.apache.flink.table.api.Expressions.lit; + +/** {@link TableTestProgram} definitions for testing the built-in DEDUPLICATE_KEEP_FIRST PTF. */ +public class DeduplicateKeepFirstTestPrograms { + + private static final String[] USER_EVENTS_SCHEMA = {"user_name STRING", "action STRING"}; + + private static final String[] TIMED_EVENTS_SCHEMA = { + "user_name STRING", + "action STRING", + "ts TIMESTAMP_LTZ(3)", + "WATERMARK FOR ts AS ts - INTERVAL '10' SECOND" + }; + + public static final TableTestProgram KEYED_KEEP_FIRST = + TableTestProgram.of( + "deduplicate-keep-first-keyed-keep-first", + "only the first row per PARTITION BY key is emitted") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues(Row.of("Vas", "login"), Row.of("Vas", "click")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name)") + .build(); + + public static final TableTestProgram KEYED_KEEP_FIRST_TABLE_API = + TableTestProgram.of( + "deduplicate-keep-first-keyed-table-api", + "the generic Table API process() call invokes the PTF, matching " + + "the SQL form") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues(Row.of("Vas", "login"), Row.of("Vas", "click")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .runTableApi( + env -> + env.from("user_events") + .partitionBy($("user_name")) + .process("DEDUPLICATE_KEEP_FIRST"), + "sink") + .build(); + + public static final TableTestProgram KEYED_KEEP_FIRST_TABLE_API_WITH_ARGS = + TableTestProgram.of( + "deduplicate-keep-first-keyed-table-api-with-args", + "the Table API process() call passes the optional state_ttl and " + + "reset_ttl_on_duplicate as named arguments") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues(Row.of("Vas", "login"), Row.of("Vas", "click")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .runTableApi( + env -> + env.from("user_events") + .partitionBy($("user_name")) + .process( + "DEDUPLICATE_KEEP_FIRST", + lit(Duration.ofSeconds(5)) + .asArgument("state_ttl"), + lit(false) + .asArgument("reset_ttl_on_duplicate")), + "sink") + .build(); + + public static final TableTestProgram MULTI_KEY_INDEPENDENCE = + TableTestProgram.of( + "deduplicate-keep-first-multi-key-independence", + "each PARTITION BY key keeps its own first row, independently of " + + "interleaving with other keys") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues( + Row.of("Vas", "login"), + Row.of("Alice", "click"), + Row.of("Vas", "click"), + Row.of("Alice", "view")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues( + Row.ofKind(RowKind.INSERT, "Vas", "login"), + Row.ofKind(RowKind.INSERT, "Alice", "click")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name)") + .build(); + + public static final TableTestProgram MULTI_COLUMN_KEY = + TableTestProgram.of( + "deduplicate-keep-first-multi-column-key", + "a multi-column PARTITION BY keeps the first row per composite key; " + + "rows differing in either key column form distinct groups") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema("region STRING", "user_name STRING", "action STRING") + .producedValues( + Row.of("EU", "Vas", "login"), + Row.of("EU", "Vas", "click"), + Row.of("US", "Vas", "signup"), + Row.of("EU", "Alice", "view")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("region STRING", "user_name STRING", "action STRING") + .consumedValues( + Row.ofKind(RowKind.INSERT, "EU", "Vas", "login"), + Row.ofKind(RowKind.INSERT, "US", "Vas", "signup"), + Row.ofKind(RowKind.INSERT, "EU", "Alice", "view")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY (region, user_name))") + .build(); + + public static final TableTestProgram NULL_PARTITION_KEY = + TableTestProgram.of( + "deduplicate-keep-first-null-partition-key", + "a NULL PARTITION BY key forms its own dedup group, keeping only " + + "its first row") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues( + Row.of(null, "login"), + Row.of("Vas", "click"), + Row.of(null, "click")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues( + Row.ofKind(RowKind.INSERT, null, "login"), + Row.ofKind(RowKind.INSERT, "Vas", "click")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name)") + .build(); + + public static final TableTestProgram NO_PARTITION_BY = + TableTestProgram.of( + "deduplicate-keep-first-no-partition-by", + "without PARTITION BY the whole input is one group: only the " + + "very first row is emitted, regardless of its content") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues( + Row.of("Vas", "login"), + Row.of("Alice", "click"), + Row.of("Bob", "view")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(input => TABLE user_events)") + .build(); + + public static final TableTestProgram RESET_ON_TTL_OFF = + TableTestProgram.of( + "deduplicate-keep-first-reset-on-ttl-off", + "reset_ttl_on_duplicate=TRUE with state_ttl=INTERVAL '0' (retention " + + "disabled): only the first row per key is emitted") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues( + Row.of("Vas", "login"), + Row.of("Vas", "click"), + Row.of("Vas", "view"), + Row.of("Vas", "logout"), + Row.of("Vas", "login")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + "state_ttl => INTERVAL '0' SECOND, " + + "reset_ttl_on_duplicate => TRUE)") + .build(); + + public static final TableTestProgram RESET_TTL_ON_DUPLICATE_FALSE = + TableTestProgram.of( + "deduplicate-keep-first-reset-ttl-on-duplicate-false", + "reset_ttl_on_duplicate=FALSE still emits only the first row per key") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues( + Row.of("Vas", "login"), + Row.of("Vas", "click"), + Row.of("Vas", "view")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + "state_ttl => INTERVAL '5' SECOND, " + + "reset_ttl_on_duplicate => FALSE)") + .build(); + + public static final TableTestProgram INPUT_COLUMN_NAMED_EVENT_TIME = + TableTestProgram.of( + "deduplicate-keep-first-input-column-named-event-time", + "an input column named event_time is deduplicated like any other; " + + "only the first row per key is emitted") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema("user_name STRING", "event_time BIGINT") + .producedValues(Row.of("Vas", 100L), Row.of("Vas", 200L)) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("user_name STRING", "event_time BIGINT") + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", 100L)) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name)") + .build(); + + public static final TableTestProgram WHOLE_ROW_AS_KEY = + TableTestProgram.of( + "deduplicate-keep-first-whole-row-as-key", + "PARTITION BY over every column deduplicates exact-duplicate rows, " + + "keeping one row per distinct payload") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues(Row.of("Vas", "login"), Row.of("Vas", "login")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY (user_name, action))") + .build(); + + public static final TableTestProgram EVENT_TIME_KEEP_EARLIEST = + TableTestProgram.of( + "deduplicate-keep-first-event-time-keep-earliest", + "with on_time, the row with the smallest event time is kept per key, " + + "not the first to arrive") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(TIMED_EVENTS_SCHEMA) + .producedValues( + Row.of("Vas", "login", Instant.ofEpochMilli(3000)), + Row.of("Vas", "click", Instant.ofEpochMilli(1000)), + Row.of("Vas", "view", Instant.ofEpochMilli(5000))) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("user_name STRING", "action STRING") + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "click")) + .build()) + .runSql( + "INSERT INTO sink SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + "on_time => DESCRIPTOR(ts))") + .build(); + + public static final TableTestProgram EVENT_TIME_LATER_ARRIVAL_EARLIER_WINS = + TableTestProgram.of( + "deduplicate-keep-first-event-time-later-arrival-earlier-wins", + "a row that arrives later but carries an earlier event time replaces " + + "the candidate before it is finalized") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(TIMED_EVENTS_SCHEMA) + .producedValues( + Row.of("Vas", "first", Instant.ofEpochMilli(5000)), + Row.of("Vas", "earlier", Instant.ofEpochMilli(2000))) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("user_name STRING", "action STRING") + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "earlier")) + .build()) + .runSql( + "INSERT INTO sink SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + "on_time => DESCRIPTOR(ts))") + .build(); + + public static final TableTestProgram EVENT_TIME_LATE_EVENT_DROPPED = + TableTestProgram.of( + "deduplicate-keep-first-event-time-late-event-dropped", + "a row below the watermark is dropped, even though its event time is " + + "earlier than the already-finalized candidate") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(TIMED_EVENTS_SCHEMA) + .producedValues( + Row.of("Bob", "early", Instant.ofEpochMilli(1000)), + Row.of("Bob", "advance", Instant.ofEpochMilli(20000)), + Row.of("Bob", "late", Instant.ofEpochMilli(500))) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("user_name STRING", "action STRING") + .consumedValues(Row.ofKind(RowKind.INSERT, "Bob", "early")) + .build()) + .runSql( + "INSERT INTO sink SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + "on_time => DESCRIPTOR(ts))") + .build(); + + public static final TableTestProgram UPDATING_INPUT_SWALLOWED = + TableTestProgram.of( + "deduplicate-keep-first-updating-input-swallowed", + "an updating input is deduplicated into an insert-only result: the " + + "first record per key is emitted once and every later change " + + "(duplicate, -U, +U, -D) is swallowed") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .addOption("changelog-mode", "I,UB,UA,D") + .producedValues( + Row.ofKind(RowKind.INSERT, "Vas", "login"), + Row.ofKind(RowKind.INSERT, "Vas", "login"), + Row.ofKind(RowKind.UPDATE_BEFORE, "Vas", "login"), + Row.ofKind(RowKind.UPDATE_AFTER, "Vas", "click"), + Row.ofKind(RowKind.DELETE, "Vas", "click")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name)") + .build(); + + public static final TableTestProgram ON_TIME_WITH_UPDATING_INPUT_FAILS = + TableTestProgram.of( + "deduplicate-keep-first-on-time-with-updating-input-fails", + "event-time mode (on_time) is rejected for updating input, because a " + + "watermark asserts no earlier record will arrive while an " + + "update can invalidate that") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(TIMED_EVENTS_SCHEMA) + .addOption("changelog-mode", "I,UB,UA,D") + .producedValues( + Row.ofKind(RowKind.INSERT, "Vas", "login", null)) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("user_name STRING", "action STRING") + .consumedValues(new Row[0]) + .build()) + .runFailingSql( + "INSERT INTO sink SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + "on_time => DESCRIPTOR(ts))", + ValidationException.class, + "not supported for PTFs that consume or produce updates") + .build(); +} diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/plan/deduplicate-keep-first-event-time-pending-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/plan/deduplicate-keep-first-event-time-pending-restore.json new file mode 100644 index 00000000000000..0b6cdd7600ad32 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/plan/deduplicate-keep-first-event-time-pending-restore.json @@ -0,0 +1,352 @@ +{ + "flinkVersion" : "2.4", + "nodes" : [ { + "id" : 5, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`user_events`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "user_name", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "ts", + "dataType" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "ts", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "syntax" : "BINARY", + "internalName" : "$-$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + }, { + "kind" : "LITERAL", + "value" : "1", + "type" : "INTERVAL SECOND(6) NOT NULL" + } ], + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + }, + "serializableString" : "`ts` - INTERVAL '0.001' SECOND" + } + } ] + } + } + }, + "abilities" : [ { + "type" : "WatermarkPushDown", + "watermarkExpr" : { + "kind" : "CALL", + "syntax" : "BINARY", + "internalName" : "$-$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + }, { + "kind" : "LITERAL", + "value" : "1", + "type" : "INTERVAL SECOND(6) NOT NULL" + } ], + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + }, + "rowtimeExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + }, + "idleTimeoutMillis" : -1, + "producedType" : { + "type" : "ROW", + "nullable" : false, + "fields" : [ { + "name" : "user_name", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "ts", + "fieldType" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "watermarkParams" : { + "emitStrategy" : "ON_EVENT", + "alignGroupName" : null, + "alignMaxDrift" : "PT0S", + "alignUpdateInterval" : "PT1S", + "sourceIdleTimeout" : -1 + } + } ] + }, + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "user_name", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "ts", + "fieldType" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events, watermark=[-(ts, 1:INTERVAL SECOND)], watermarkEmitStrategy=[on-event]]], fields=[user_name, action, ts])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "user_name", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "ts", + "fieldType" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[user_name]])" + }, { + "id" : 7, + "type" : "stream-exec-process-table-function_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "user_name", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "ts", + "fieldType" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "rowtime", + "fieldType" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "nullable" : false, + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "ProcessTableFunction(invocation=[DEDUPLICATE_KEEP_FIRST(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DESCRIPTOR(_UTF-16LE'ts'), DEFAULT())], uid=[DEDUPLICATE_KEEP_FIRST], select=[user_name,action,ts,rowtime], rowType=[RecordType(VARCHAR(2147483647) user_name, VARCHAR(2147483647) action, TIMESTAMP_LTZ(3) *ROWTIME* ts, TIMESTAMP_LTZ(3) *ROWTIME* rowtime)])", + "uid" : "DEDUPLICATE_KEEP_FIRST", + "functionCall" : { + "kind" : "CALL", + "internalName" : "$DEDUPLICATE_KEEP_FIRST$1", + "operands" : [ { + "kind" : "TABLE_ARG_CALL", + "inputIndex" : 0, + "partitionKeys" : [ 0 ], + "type" : { + "type" : "ROW", + "nullable" : false, + "fields" : [ { + "name" : "user_name", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "ts", + "fieldType" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + } + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "INTERVAL SECOND(6)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "BOOLEAN" + }, { + "kind" : "CALL", + "internalName" : "$DESCRIPTOR$1", + "operands" : [ { + "kind" : "LITERAL", + "value" : "ts", + "type" : "CHAR(2) NOT NULL" + } ], + "type" : "DESCRIPTOR NOT NULL" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + } ], + "type" : { + "type" : "ROW", + "nullable" : false, + "fields" : [ { + "name" : "user_name", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "ts", + "fieldType" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "rowtime", + "fieldType" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "nullable" : false, + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + } + }, + "inputChangelogModes" : [ [ "INSERT" ] ], + "outputChangelogMode" : [ "INSERT" ], + "inputUpsertKeys" : [ [ ] ] + }, { + "id" : 8, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "VARCHAR(2147483647)" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "Calc(select=[user_name, action])" + }, { + "id" : 9, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "user_name", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "dataType" : "VARCHAR(2147483647)" + } ] + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "upsertMaterializeStrategy" : "MAP", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_name, action])" + } ], + "edges" : [ { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 8, + "target" : 9, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..f895fbd95b2e952941327e3ce5b50e0d7e0234b3 GIT binary patch literal 17098 zcmeHPZ;TsP5r4KXmt5L3m-d=ADTz+aHByv#*Z1F-Ljs@AM@)RSv7My7Q*fTw&v%=w zcir9f#mR?oRe>TE8ib&tqChJ8F-R2%so(fS_yANY6?{Sh2_(Lx2ni&lQWR$1?t0f} zJ8@`yITEjQ-u15Eym|BH&71kn?8VQm1PLMi@P~)bI*~r|R9e?0W*9kIP}o9Ml}uSv zm#5E?z4Nj8Q|UxJ8p*}-Cu6a6ej%R8=5Cgr{CMci-(S7){GXrxQTf*EHnDXi=upLL zvYmetvTmPyr=A2hQ34}-HnFo_SGSRXk9?~nE9|(qMs=~IDMh9mVkj@_t81mu#q#-4 zWqj1k>(i5Ysh}3eCPHwmRK%QkzN*S*aBOTWJQEI$PL5BHPmj(_gu>IJLDyJClT|ZF z$3q1v9Gsphg^DwwLV=A>iQ}c2@MtMC6{Ml*(cr{nI25E4lTwgTDIBKLrBTWXgp9lb zJac>KCpk@D5om=jaLH7Y1nT@hkxwd{RZG0FDRkxcYNB=eR4kv3W#W-Ud?}XUcR$<2I<3fpq?OB>>QT4? zHJGqSE3U?Q86>;@%5}^m6hG;*P)PbNI8SKmip%;)7bZTn4A>>ejTvi)cvZmDEFr_^M)iY4ej^kAp%RqH|!H+a761K)vv$sC# zP6FQ{dmH!`-oU%PK8`l5kBys1RrNXzy=|oD=WbCyx0`;};f5vR*+P1&gGZXZO=PP_ zf!$mkeFkHy8(cJnlG}K=0Y7LLOrE$yh-8|IkTaDm#!CxSf*9n|2!5}_7+$B+4k{ZX zknRM1%|dMlyu@l&1n#mt(tNAMC>;3ZXp@Ajo~(rzF}1QRAv%x)NM!%lvPV9;d}z$i zp^4{i+tO)s?i&N*)3`68XyD!rAHNN7$(`1K>tWYfJ5ZBW)tKQ8&$zB zO)_@Z2n?VvYdYEEI$m>{rZB4hW>{bU-J2i%4S=I1vTcnjRaUEXj6m5$kSRwaEje@s zV0lm;w|O^!C$cHP(_AW*h((gb6OH>wv(~nUZftRS)VAqPgVx?37-T@AYEdqNb#NxF z1)$ZlLPE32xtCUGL1s$Pa4gC{-?{#&AHVd&JS^h`d8~xboJU|#1}%evsI-`n4S-^G zvc)acp*DsT)Q&p9x~+bA;x_A8n>Lo&whbsYZnRe*v3@Qw<3)0X_#RT&JdjD90lR;g z?+~adLR3=}##6k;$SJj~73Gr5a{-`1AMbu{^k+9;8|3=dr@$RYcE|2az$LtIj;#Oc zi~l;v$@nCb?Dc4>`!m?QfVC%F$`M_smr#>yFa7qw8^5IA4`AM-&wwOL4Po}cMMSy- z&}i~*bOQ4;d|jbR!2#)%0BpTQW=OhR!DP2d9`|g{R!RVN=RS02OpR%!Y+4#oItyga$QC{t+3L>LF+{9sFOK)C9ax{;<0*g zC!~Ka{<(NE$NPPf_cnwBsq5qa>RRGB{NTYIZo?_Ji*Aa_5doZQ6>R$q=S2hxM{Y-< zE=Sxwnut7GC*n!LVVY+4)xUi6*H;hy5!UhN9WPQRgRZxktdc#vZR5^Rmt?-)(Ny7Uvc;GY@Cpd(+HmziUoy9rz^X^WmH< zdzF;w~DfDlMXa6SmCk#dp z^Wb~ymigQUr$OtIkU0%Xh6s7t`i^Bod*6ePmd}rYmwD}jg~j*b2gC5kkCLoI)JDF7 zRPk`mP|#98c4eXT==iV!pUX36N&ie9FW;H>&%XQNf$t8#fAH2f4uAOu{2Zrb9h>0= z9zNt806YABP-mtN9x9xTsseK5$D{Tm_tvJ@Wrdj(o|>2`Ks6Sd3{FfHCZT+5q7a-K zo0=}MvC>qLje_s+2Jj04Za*3vw2us@(y>e=m&({RH}mmqE)$ zd^MM^KS1{r3UykRZZtzW_Xmi4z#)MB&Grf`6fdjeS_+oNPx>G}s z38hf8%Vf9xJgphB6~jHT>QVi0GAHp4AK8|+tv;E;zg0I*bSQckD)Je?+B zS+U{?YOG@S4>T~bDX{7T1x)yPAn4zGVB(o3U}6BP*768k+*52~mxWCXw!kL#d*Bd^ zNe}w)a~KOXdAz)jhnG*o;&#RKN#kWt3~EKk*Z%%?`10&e=XeatJM^%@zaK`fj%<9i ziIJ+m7pZn5Qtd1!btnAo;fz<>6jgV>ez?o!SU%#K!FqhL(f#fDkm&#XU#5|IKIB$U zZuNY~yG?Gn&e0|(w|YLL(DNZTy!D2lq5DRfC$(UQ=#WC~MnTV}ze(0}$?vI4?g73Z zOdrx#|N3crlq%A47=YSv9y0FIi~m0NhpF%vzW0-&_*QxP=3DSXbTikn0baB^_GY!J zF0tdoJaXp=Y-4%yEZH%cH~14_=E&Dq2bP0NKhICZUi-yofJmJvu{Ai{gsM6CKfKT? zUEpuZ3eox{yzvh&qM0(Yp<&|plO0QBuqYc6)r(Q8N({", + "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events]], fields=[user_name, action])" + }, { + "id" : 2, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "Exchange(distribution=[hash[user_name]])" + }, { + "id" : 3, + "type" : "stream-exec-process-table-function_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "ProcessTableFunction(invocation=[DEDUPLICATE_KEEP_FIRST(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE_KEEP_FIRST], select=[user_name,action], rowType=[RecordType(VARCHAR(2147483647) user_name, VARCHAR(2147483647) action)])", + "uid" : "DEDUPLICATE_KEEP_FIRST", + "functionCall" : { + "kind" : "CALL", + "internalName" : "$DEDUPLICATE_KEEP_FIRST$1", + "operands" : [ { + "kind" : "TABLE_ARG_CALL", + "inputIndex" : 0, + "partitionKeys" : [ 0 ], + "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "INTERVAL SECOND(6)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "BOOLEAN" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + } ], + "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" + }, + "inputChangelogModes" : [ [ "INSERT" ] ], + "outputChangelogMode" : [ "INSERT" ], + "inputUpsertKeys" : [ [ ] ] + }, { + "id" : 4, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "user_name", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "dataType" : "VARCHAR(2147483647)" + } ] + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "upsertMaterializeStrategy" : "ADAPTIVE", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_name, action])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 4, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-keyed-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-keyed-restore/savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..c59dd5545a963112b75cff4ca18dd45eca44217f GIT binary patch literal 16582 zcmeHOZ)_V$6`ys|wCS}?dYAUDz4qjzX{74TyZ)EhX};hj4)xV_EZfap6BRSw9Vgqa zcg^lPP5PnqbmEV65`_~Woj9Oy2VW=x2_fziAA3l^2dLo7NgNVLeBeL`Do#2bgg4_| zd!4mgS4mu`cBMGGJF_!y-kUe`=DpubeQG&?5E_6#dU5rCs1NPW%j)IRegl4&q8HIX zv`BAHAO7LLe*OBfyFdN92Ok=t^#eLFbu60j->503;ixzA zCfa9JaTUG8q1Vc?LQV^-SQpBgA`#sXf<-}JSuF>zR4#>-NXRVevGJlAk=S@N7>`GSv58o4d^|c46AZjcs+z1qX(<#ECj-HFXfhHc!BDI`Az&$7 zj!30oAc9MgKr|AH2XH(#8HkclL<$E=IG;_Y3eAEf`)+t4kv^X+=99TpBAr@H=HRUlEok~O zkE>W*A$(Z@uJR@>DTLQ+swr0pZ?0EK&6E{`Pp+9nSFw^Ox{MY1D$z&XuktFc8Y`L! zZG)KqOr^!$`YYOH=;*l)J<%QveoiKeWXxg{7vTBsk8U0N!OPz&`Vf~!2gx(c$FHFZY81~h!I-mrqz zWr4k#N~6aVZCMtvLdh~wI1T*>b$$tqUjpNoz(yfhb%k{Eo2j+xs*aAj$z#LLX;*+n z^y>4We&~&jb$UI6E704JW1#17b!rI>S>;*HkWC=mL{BU^h@LN9BBBXY_g|J($wdA( zwcH zB$s72D45qD4V?`Av)~&*V@a!IsW=P_#1tQOQC)PR3F#{i^mnV7{hqRyW*QQCr07_9M$NiJo{kc zOT-X$xmuspk2~gc-qdAvna*hyl!9{LX}G5nr}zD9Rp9Gj%^-S_mIF22O{@j1&#Yj5 z(=1J)0fnf`=E@pvd_Q-M{u?&Y?uA5d=1d~@nT?GNHiM5suLH38P{GkU&v5kDPs`b- zN7&n^TG({&6)<6H6ZrEi^yU33O}$t~ZVyfK3l;_1_+$bSvDRtr}b} zRt@Lm^VrFDW8NX*2VI1n>~!dX4=r}B2X*z^qzdg`I<%p0r`zWo%b>kBxy)eChNSk7 zXG2QU>z)ni*^vA_jnJlX=-H6DiFmdXVoqJ0E!)5^aDB z*T~>(^#bQd=-VHpj)UIQvn{(8Fmt`WLRUo-mS-hN0tM<5Bf}oW6>y{3`pTn8A3hLa-f7KgMoOt9E-(c zQaLmM9#B7BIw9yoLoqw)hO>h(o*S+inb{`45x4Vx9bY+!cB3N@Ut|%u0CjXr0UDpv zjX(ucXFR<)G<-IBAvtRW1`8LmNz`{Pk%b~3Y9IdlxXa~*t1>TYm5Qc1Vt*xU5FSE6 zO~-GGW6KKk_0$l`J^h_pB21l|I*h3MkLJe2fkf!b;B`DlxhkX53Zo_;j&8V}}} z^5STYh3yg1kLOzSWAi3bQ=LWwY#-h>qVV_g-+F4&do^(b(zege8f#Gz#}?vg2X(wL zgTMRehNm5mmUi?Tgn<8+UG0vNyV!99iaQ-9&$y0ct*UF93STglJUuVX0d`8$*GK7d z15kS36UfdS`rJ#-!@-90@&pH3z+t*AZstnKW&vDn$vv{Bca7TtR| zZjeC{tCB3i>R=14jYG>@g*+twR*?87T$@yVv4?4gDQumB`mE_k|2@)}_nvsAJj{EQ zY#sCdqzm(IqtCnD6YqD}rAYC?I^ca@8^HTv0KD(5ADO7L8g_D_gC@1QqNe5Llxl9? z$Xh!bm`;5nl_@Zw`ymEa9|6^C<`--gI6u~bQl|4w*lvs3QwpDEw44O!y(6uIMh~W5 z{nM9zdE>-;VBEZ58#gWZ{7kJJ1<(-Gx@5n)sYoO{K3n6+awz30}B_Q-Y<88>Xgyrk-ci?B( zUdL=;7JVLsfZ@$Mb7#l6jSagT>gTnZE|SxJmOsT-cVj7Z5$y>T4OVjJAIH9b`#(eP zt^WLHUtW6mW3+h&w1PBLDN+uHL|hUl$|P8p%CQMCULupB@sd;uhjAFH?z5`f9oo<= zmc`4VgZ18X1Vaig^9rzvJ%?(=p9u>oZs2IYbiEtzmiiUu) zyYL54jSh6vv2Ay9uI}2mIv?>c&DP_m*G-buqp;=p=K%aMi8Ow! zH8Ad?%XaiY8nJkpWkT~ajavZsLCo2d3Bw6OV?H0+vxtTz*$}ZV&0tj|kOe6%qW#vZ zISdKf@H}Kc%c^XwIN#)JC8Gu?h-NeSno%XHL|+Y*HJH_KGYC*Z1%tUFL85DuHfoKu z&E>U*AvCKBHe={FG)@}N0NH9#LiO#sYYBy>p-J_5x;%W(a z0l9Ax7ZWJme1mSj4c&}|@Y7Ag?BGlyGoJzYXBx7zEeZP@Z2|$nj!pUNZKh)511g2K TqjA#2Y*Vm+^XK!~WM=livek^s literal 0 HcmV?d00001 diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java index 1ac2e2e03e71d4..2fb174cc37a8f9 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/BuiltInProcessTableFunction.java @@ -19,16 +19,21 @@ package org.apache.flink.table.runtime.functions.ptf; import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.dataview.ListView; +import org.apache.flink.table.api.dataview.MapView; +import org.apache.flink.table.api.dataview.ValueView; import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.functions.BuiltInFunctionDefinition; import org.apache.flink.table.functions.FunctionRequirement; import org.apache.flink.table.functions.ProcessTableFunction; import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.table.runtime.dataview.DataViewUtils; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.inference.CallContext; import org.apache.flink.table.types.inference.StateTypeStrategy; import org.apache.flink.table.types.inference.TypeInference; import org.apache.flink.table.types.inference.TypeStrategies; +import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.utils.DataTypeUtils; import java.util.LinkedHashMap; @@ -109,12 +114,38 @@ private static LinkedHashMap toInternalStateStrategie ctx -> strategy.inferType(ctx) .map( - DataTypeUtils - ::toInternalDataType), + BuiltInProcessTableFunction + ::toInternalStateType), strategy.getTimeToLive(callContext).orElse(null)))); return result; } + /** + * Converts a state type to its internal representation. For {@link ValueView}/{@link + * MapView}/{@link ListView} only the value/key/element types are internalized; the view wrapper + * is preserved, unlike {@link DataTypeUtils#toInternalDataType} which would rewrite the whole + * view to {@code RowData}. + */ + private static DataType toInternalStateType(final DataType stateType) { + final LogicalType logicalType = stateType.getLogicalType(); + if (DataViewUtils.isDataView(logicalType, ValueView.class)) { + final DataType valueType = stateType.getChildren().get(0); + return ValueView.newValueViewDataType(DataTypeUtils.toInternalDataType(valueType)); + } + if (DataViewUtils.isDataView(logicalType, MapView.class)) { + final DataType mapType = stateType.getChildren().get(0); + return MapView.newMapViewDataType( + DataTypeUtils.toInternalDataType(mapType.getChildren().get(0)), + DataTypeUtils.toInternalDataType(mapType.getChildren().get(1))); + } + if (DataViewUtils.isDataView(logicalType, ListView.class)) { + final DataType arrayType = stateType.getChildren().get(0); + return ListView.newListViewDataType( + DataTypeUtils.toInternalDataType(arrayType.getChildren().get(0))); + } + return DataTypeUtils.toInternalDataType(stateType); + } + @Override public Set getRequirements() { if (definition != null) { diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java new file mode 100644 index 00000000000000..9ca42badc904aa --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.ptf; + +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.table.api.dataview.ValueView; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.utils.ProjectedRowData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; +import org.apache.flink.table.functions.TableSemantics; +import org.apache.flink.table.runtime.typeutils.InternalSerializers; +import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils; +import org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.types.RowKind; + +import javax.annotation.Nullable; + +import java.util.stream.IntStream; + +public class DeduplicateKeepFirst extends BuiltInProcessTableFunction { + + private static final long serialVersionUID = 1L; + + private final int[] outputIndices; + + private final int eventTimeIndex; + + private final RowType inputRowType; + + private transient ProjectedRowData projectedOutput; + + private transient ProjectedRowData emitProjection; + + private transient RowData.FieldGetter[] payloadGetters; + + private transient TypeSerializer candidateSerializer; + + public DeduplicateKeepFirst(final SpecializedContext context) { + super(BuiltInFunctionDefinitions.DEDUPLICATE_KEEP_FIRST, context); + final CallContext callContext = context.getCallContext(); + final TableSemantics tableSemantics = + callContext + .getTableSemantics(DeduplicateKeepFirstTypeStrategy.ARG_INPUT) + .orElseThrow(() -> new IllegalStateException("Table argument expected.")); + this.outputIndices = ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics); + this.eventTimeIndex = outputIndices.length; + this.inputRowType = (RowType) tableSemantics.dataType().getLogicalType(); + } + + @Override + public void open(final FunctionContext context) throws Exception { + super.open(context); + projectedOutput = ProjectedRowData.from(outputIndices); + emitProjection = ProjectedRowData.from(IntStream.range(0, eventTimeIndex).toArray()); + payloadGetters = new RowData.FieldGetter[outputIndices.length]; + for (int i = 0; i < outputIndices.length; i++) { + payloadGetters[i] = + RowData.createFieldGetter( + inputRowType.getTypeAt(outputIndices[i]), outputIndices[i]); + } + + final LogicalType[] candidateTypes = new LogicalType[eventTimeIndex + 1]; + for (int i = 0; i < eventTimeIndex; i++) { + candidateTypes[i] = inputRowType.getTypeAt(outputIndices[i]); + } + candidateTypes[eventTimeIndex] = new BigIntType(); + candidateSerializer = InternalSerializers.create(RowType.of(candidateTypes)); + } + + public void eval( + final Context ctx, + final ValueView seen, + final ValueView candidate, + final RowData input, + @Nullable final Long stateTtl, + @Nullable final Boolean resetTtlOnDuplicate) + throws Exception { + // state_ttl is applied at plan time via getTimeToLive(); the PTF codegen still requires + // a parameter for every declared argument, so it is unused here. + final Long rowtime = ctx.timeContext(Long.class).time(); + + if (rowtime != null) { // watermark mode + final Long watermark = ctx.timeContext(Long.class).tableWatermark(); + if (watermark != null && rowtime < watermark) { + return; + } + + if (seen.getValue() == null) { // not yet emitted → buffer the earliest candidate + final RowData currLowest = candidate.getValue(); + if (currLowest == null || rowtime < currLowest.getLong(eventTimeIndex)) { + if (currLowest != null) { + ctx.timeContext(Long.class).clearTimer(currLowest.getLong(eventTimeIndex)); + } + // object reuse may overwrite the input's memory before the timer fires + candidate.setValue( + candidateSerializer.copy(materializeCandidate(input, rowtime))); + ctx.timeContext(Long.class).registerOnTime(rowtime); + } + } else { // already emitted → drop, optionally refresh the TTL + if (resetTtlOnDuplicate == null || resetTtlOnDuplicate) { + seen.setValue(true); + } + } + + } else { // watermarkless mode + if (seen.getValue() == null) { + projectedOutput.replaceRow(input); + projectedOutput.setRowKind(RowKind.INSERT); + collect(projectedOutput); + seen.setValue(true); + } else { + if (resetTtlOnDuplicate == null || resetTtlOnDuplicate) { + seen.setValue(true); + } + } + } + } + + public void onTimer( + final OnTimerContext ctx, + final ValueView seen, + final ValueView candidate) + throws Exception { + final RowData currLowest = candidate.getValue(); + if (currLowest == null) { + return; + } + emitProjection.replaceRow(currLowest); + collect(emitProjection); + candidate.clear(); + seen.setValue(true); + } + + private GenericRowData materializeCandidate(final RowData input, final long rowtime) { + final GenericRowData row = new GenericRowData(eventTimeIndex + 1); + for (int i = 0; i < eventTimeIndex; i++) { + row.setField(i, payloadGetters[i].getFieldOrNull(input)); + } + row.setField(eventTimeIndex, rowtime); + return row; + } +} From 3493371fd2aa9b0310c9322dd77bdd6bae943a7c Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Tue, 6 Oct 2026 17:41:22 +0100 Subject: [PATCH 2/5] [FLINK-40896][table] Added test coverage, change md as flip changed, addressed -u -d issue, @Internal etc --- .../queries/deduplicate-keep-first.md | 7 +- .../queries/deduplicate-keep-first.md | 7 +- .../functions/BuiltInFunctionDefinitions.java | 4 +- .../DeduplicateKeepFirstTypeStrategy.java | 13 +- .../SpecificInputTypeStrategies.java | 2 +- .../DeduplicateKeepFirstRestoreTest.java | 4 +- ...duplicateKeepFirstRestoreTestPrograms.java | 57 +++++ .../DeduplicateKeepFirstSemanticTests.java | 32 +-- .../DeduplicateKeepFirstTestPrograms.java | 195 ++++++++++++++---- ...keep-first-global-ttl-expired-restore.json | 144 +++++++++++++ .../savepoint/_metadata | Bin 0 -> 16398 bytes ...licate-keep-first-ttl-expired-restore.json | 142 +++++++++++++ .../savepoint/_metadata | Bin 0 -> 16395 bytes .../functions/ptf/DeduplicateKeepFirst.java | 23 ++- 14 files changed, 554 insertions(+), 76 deletions(-) create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/plan/deduplicate-keep-first-global-ttl-expired-restore.json create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/savepoint/_metadata create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/plan/deduplicate-keep-first-ttl-expired-restore.json create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/savepoint/_metadata diff --git a/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md b/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md index 1e08355875975c..c7bb5464fa38f5 100644 --- a/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md +++ b/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md @@ -58,8 +58,8 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( |:----------|:---------|:------------| | `input` | Yes | The input table (insert-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | | `on_time` | No | A `DESCRIPTOR` naming a single rowtime attribute. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires insert-only input; combining `on_time` with an updating input is rejected at planning time. | -| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. Defaults to no TTL (state retained indefinitely). After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. `INTERVAL '0'` disables retention. | -| `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful together with `state_ttl`. | +| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. If omitted, falls back to `table.exec.state.ttl`, which retains state indefinitely at its default of 0. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. `INTERVAL '0'` disables retention. | +| `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful when a TTL applies, whether set through `state_ttl` or inherited from `table.exec.state.ttl`. | ### Output Schema @@ -150,6 +150,7 @@ With `on_time`, the record with the smallest event time per key is kept (here `c ```sql -- Input (updating changelog): +-- -D[user_name:'Vas', action:'logout'] key not seen yet -> ignored -- +I[user_name:'Vas', action:'login'] first record for the key -> emitted -- +I[user_name:'Vas', action:'login'] duplicate -> swallowed -- -U[user_name:'Vas', action:'login'] retraction -> swallowed @@ -164,7 +165,7 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( -- +I[user_name:'Vas', action:'login'] ``` -The input may be an updating changelog. `DEDUPLICATE_KEEP_FIRST` keeps the first record observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. Event-time mode (`on_time`) is not supported with updating input. +The input may be an updating changelog. `DEDUPLICATE_KEEP_FIRST` keeps the first `+I` or `+U` observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. A `-U` or `-D` for a key that has not been seen yet is ignored and does not mark the key as seen, so the next `+I` or `+U` for that key is still emitted. Event-time mode (`on_time`) is not supported with updating input. #### Table API diff --git a/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md b/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md index 1e08355875975c..c7bb5464fa38f5 100644 --- a/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md +++ b/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md @@ -58,8 +58,8 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( |:----------|:---------|:------------| | `input` | Yes | The input table (insert-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | | `on_time` | No | A `DESCRIPTOR` naming a single rowtime attribute. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires insert-only input; combining `on_time` with an updating input is rejected at planning time. | -| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. Defaults to no TTL (state retained indefinitely). After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. `INTERVAL '0'` disables retention. | -| `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful together with `state_ttl`. | +| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. If omitted, falls back to `table.exec.state.ttl`, which retains state indefinitely at its default of 0. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. `INTERVAL '0'` disables retention. | +| `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful when a TTL applies, whether set through `state_ttl` or inherited from `table.exec.state.ttl`. | ### Output Schema @@ -150,6 +150,7 @@ With `on_time`, the record with the smallest event time per key is kept (here `c ```sql -- Input (updating changelog): +-- -D[user_name:'Vas', action:'logout'] key not seen yet -> ignored -- +I[user_name:'Vas', action:'login'] first record for the key -> emitted -- +I[user_name:'Vas', action:'login'] duplicate -> swallowed -- -U[user_name:'Vas', action:'login'] retraction -> swallowed @@ -164,7 +165,7 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( -- +I[user_name:'Vas', action:'login'] ``` -The input may be an updating changelog. `DEDUPLICATE_KEEP_FIRST` keeps the first record observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. Event-time mode (`on_time`) is not supported with updating input. +The input may be an updating changelog. `DEDUPLICATE_KEEP_FIRST` keeps the first `+I` or `+U` observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. A `-U` or `-D` for a key that has not been seen yet is ignored and does not mark the key as seen, so the next `+I` or `+U` for that key is still emitted. Event-time mode (`on_time`) is not supported with updating input. #### Table API diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index 9d1e8e1ca221cf..cd28a5c5e45144 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -109,7 +109,7 @@ import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.ARRAY_ELEMENT_ARG; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.ARRAY_FULLY_COMPARABLE; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.ARRAY_OF_ENTRIES_ARG; -import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.DEDUPLICATE_KEEP_FIRST_TYPE_STRATEGY; +import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.DEDUPLICATE_KEEP_FIRST_INPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.FROM_CHANGELOG_INPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.INDEX; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.JSON_ARGUMENT; @@ -960,7 +960,7 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) "state_ttl", DataTypes.INTERVAL(DataTypes.SECOND()), true), StaticArgument.scalar( "reset_ttl_on_duplicate", DataTypes.BOOLEAN(), true)) - .inputTypeStrategy(DEDUPLICATE_KEEP_FIRST_TYPE_STRATEGY) + .inputTypeStrategy(DEDUPLICATE_KEEP_FIRST_INPUT_TYPE_STRATEGY) .outputTypeStrategy(DEDUPLICATE_KEEP_FIRST_OUTPUT_TYPE_STRATEGY) .stateTypeStrategies(DeduplicateKeepFirstTypeStrategy.stateTypeStrategies()) .runtimeClass( diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java index 4860a73a497e07..4a05b90a6807f8 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java @@ -18,6 +18,7 @@ package org.apache.flink.table.types.inference.strategies; +import org.apache.flink.annotation.Internal; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.DataTypes.Field; import org.apache.flink.table.api.ValidationException; @@ -38,11 +39,15 @@ import java.util.Set; import java.util.stream.Collectors; -public class DeduplicateKeepFirstTypeStrategy { +/** Type strategies for the {@code DEDUPLICATE_KEEP_FIRST} process table function. */ +@Internal +public final class DeduplicateKeepFirstTypeStrategy { public static final int ARG_INPUT = 0; public static final int ARG_STATE_TTL = 1; public static final int ARG_RESET_TTL_ON_DUPLICATE = 2; + private static final String SEEN_STATE_NAME = "seen"; + private static final String CANDIDATE_STATE_NAME = "candidate"; public static final InputTypeStrategy INPUT_TYPE_STRATEGY = new ValidationOnlyInputTypeStrategy() { @@ -85,8 +90,6 @@ public Optional> inferInputTypes( return Optional.of(DataTypes.ROW(outputFields).notNull()); }; - private static final String SEEN_STATE_NAME = "seen"; - public static final StateTypeStrategy SEEN_STATE_TYPE_STRATEGY = new StateTypeStrategy() { @Override @@ -101,8 +104,6 @@ public Optional getTimeToLive(final CallContext callContext) { } }; - private static final String CANDIDATE_STATE_NAME = "candidate"; - public static final StateTypeStrategy CANDIDATE_STATE_TYPE_STRATEGY = new StateTypeStrategy() { @Override @@ -185,4 +186,6 @@ private static Optional> validateResetTtlOnDuplicate( } return Optional.empty(); } + + private DeduplicateKeepFirstTypeStrategy() {} } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java index 066f9b38cf7705..40c0e2ae70da50 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java @@ -143,7 +143,7 @@ public static InputTypeStrategy plainJsonPath(final InputTypeStrategy signatures FromChangelogTypeStrategy.INPUT_TYPE_STRATEGY; /** Input strategy for {@link BuiltInFunctionDefinitions#DEDUPLICATE_KEEP_FIRST}. */ - public static final InputTypeStrategy DEDUPLICATE_KEEP_FIRST_TYPE_STRATEGY = + public static final InputTypeStrategy DEDUPLICATE_KEEP_FIRST_INPUT_TYPE_STRATEGY = DeduplicateKeepFirstTypeStrategy.INPUT_TYPE_STRATEGY; /** Input strategy for {@link BuiltInFunctionDefinitions#SNAPSHOT}. */ diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java index fa9ad95f08559e..d4430c64c6e674 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java @@ -41,7 +41,9 @@ public DeduplicateKeepFirstRestoreTest() { public List programs() { return List.of( DeduplicateKeepFirstRestoreTestPrograms.KEYED_RESTORE, - DeduplicateKeepFirstRestoreTestPrograms.EVENT_TIME_PENDING_RESTORE); + DeduplicateKeepFirstRestoreTestPrograms.EVENT_TIME_PENDING_RESTORE, + DeduplicateKeepFirstRestoreTestPrograms.TTL_EXPIRED_RESTORE, + DeduplicateKeepFirstRestoreTestPrograms.GLOBAL_TTL_EXPIRED_RESTORE); } @Override diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java index 401ddac82e3b0a..ad21aad63e1881 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java @@ -18,12 +18,14 @@ package org.apache.flink.table.planner.plan.nodes.exec.stream; +import org.apache.flink.table.api.config.ExecutionConfigOptions; import org.apache.flink.table.test.program.SinkTestStep; import org.apache.flink.table.test.program.SourceTestStep; import org.apache.flink.table.test.program.TableTestProgram; import org.apache.flink.types.Row; import org.apache.flink.types.RowKind; +import java.time.Duration; import java.time.Instant; /** Restore {@link TableTestProgram}s for the built-in DEDUPLICATE_KEEP_FIRST PTF. */ @@ -91,4 +93,59 @@ public class DeduplicateKeepFirstRestoreTestPrograms { + "input => TABLE user_events PARTITION BY user_name, " + "on_time => DESCRIPTOR(ts))") .build(); + + public static final TableTestProgram TTL_EXPIRED_RESTORE = + TableTestProgram.of( + "deduplicate-keep-first-ttl-expired-restore", + "the 'seen' state in the savepoint has outlived state_ttl by the time " + + "the test restores it, so an already-seen key is emitted " + + "again in a new deduplication period") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedBeforeRestore( + Row.of("Vas", "login"), Row.of("Vas", "click")) + .producedAfterRestore(Row.of("Vas", "logout")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedBeforeRestore( + Row.ofKind(RowKind.INSERT, "Vas", "login")) + .consumedAfterRestore( + Row.ofKind(RowKind.INSERT, "Vas", "logout")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + "state_ttl => INTERVAL '10' SECOND)") + .build(); + + public static final TableTestProgram GLOBAL_TTL_EXPIRED_RESTORE = + TableTestProgram.of( + "deduplicate-keep-first-global-ttl-expired-restore", + "without state_ttl, the 'seen' state falls back to " + + "table.exec.state.ttl and has expired by the time the test " + + "restores it, so an already-seen key is emitted again") + .setupConfig( + ExecutionConfigOptions.IDLE_STATE_RETENTION, Duration.ofSeconds(10)) + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedBeforeRestore( + Row.of("Vas", "login"), Row.of("Vas", "click")) + .producedAfterRestore(Row.of("Vas", "logout")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedBeforeRestore( + Row.ofKind(RowKind.INSERT, "Vas", "login")) + .consumedAfterRestore( + Row.ofKind(RowKind.INSERT, "Vas", "logout")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name)") + .build(); } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java index 7bbe0f0848d734..fc3e699597d4e3 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java @@ -39,21 +39,25 @@ protected void applyDefaultEnvironmentOptions(TableConfig config) { @Override public List programs() { return List.of( - DeduplicateKeepFirstTestPrograms.KEYED_KEEP_FIRST, - DeduplicateKeepFirstTestPrograms.KEYED_KEEP_FIRST_TABLE_API, - DeduplicateKeepFirstTestPrograms.KEYED_KEEP_FIRST_TABLE_API_WITH_ARGS, - DeduplicateKeepFirstTestPrograms.MULTI_KEY_INDEPENDENCE, + DeduplicateKeepFirstTestPrograms.KEYED, + DeduplicateKeepFirstTestPrograms.TABLE_API, + DeduplicateKeepFirstTestPrograms.TABLE_API_WITH_ARGS, + DeduplicateKeepFirstTestPrograms.MULTI_KEY, DeduplicateKeepFirstTestPrograms.MULTI_COLUMN_KEY, - DeduplicateKeepFirstTestPrograms.NULL_PARTITION_KEY, + DeduplicateKeepFirstTestPrograms.NULL_KEY, DeduplicateKeepFirstTestPrograms.NO_PARTITION_BY, - DeduplicateKeepFirstTestPrograms.RESET_ON_TTL_OFF, - DeduplicateKeepFirstTestPrograms.RESET_TTL_ON_DUPLICATE_FALSE, - DeduplicateKeepFirstTestPrograms.INPUT_COLUMN_NAMED_EVENT_TIME, - DeduplicateKeepFirstTestPrograms.WHOLE_ROW_AS_KEY, - DeduplicateKeepFirstTestPrograms.EVENT_TIME_KEEP_EARLIEST, - DeduplicateKeepFirstTestPrograms.EVENT_TIME_LATER_ARRIVAL_EARLIER_WINS, - DeduplicateKeepFirstTestPrograms.EVENT_TIME_LATE_EVENT_DROPPED, - DeduplicateKeepFirstTestPrograms.UPDATING_INPUT_SWALLOWED, - DeduplicateKeepFirstTestPrograms.ON_TIME_WITH_UPDATING_INPUT_FAILS); + DeduplicateKeepFirstTestPrograms.RESET_WITH_ZERO_TTL, + DeduplicateKeepFirstTestPrograms.NO_TTL_RESET, + DeduplicateKeepFirstTestPrograms.EVENT_TIME_COLUMN_CLASH, + DeduplicateKeepFirstTestPrograms.WHOLE_ROW_KEY, + DeduplicateKeepFirstTestPrograms.EVENT_TIME, + DeduplicateKeepFirstTestPrograms.EVENT_TIME_MULTI_KEY, + DeduplicateKeepFirstTestPrograms.EVENT_TIME_LATE_DROPPED, + DeduplicateKeepFirstTestPrograms.UPDATING_INPUT, + DeduplicateKeepFirstTestPrograms.UNSEEN_RETRACTION_IGNORED, + DeduplicateKeepFirstTestPrograms.UPDATING_EVENT_TIME_FAILS, + DeduplicateKeepFirstTestPrograms.NEGATIVE_TTL_FAILS, + DeduplicateKeepFirstTestPrograms.NON_LITERAL_TTL_FAILS, + DeduplicateKeepFirstTestPrograms.NON_LITERAL_RESET_FAILS); } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java index 617ef192225af2..6997ae957ff56a 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java @@ -43,9 +43,9 @@ public class DeduplicateKeepFirstTestPrograms { "WATERMARK FOR ts AS ts - INTERVAL '10' SECOND" }; - public static final TableTestProgram KEYED_KEEP_FIRST = + public static final TableTestProgram KEYED = TableTestProgram.of( - "deduplicate-keep-first-keyed-keep-first", + "deduplicate-keep-first-keyed", "only the first row per PARTITION BY key is emitted") .setupTableSource( SourceTestStep.newBuilder("user_events") @@ -62,9 +62,9 @@ public class DeduplicateKeepFirstTestPrograms { + "input => TABLE user_events PARTITION BY user_name)") .build(); - public static final TableTestProgram KEYED_KEEP_FIRST_TABLE_API = + public static final TableTestProgram TABLE_API = TableTestProgram.of( - "deduplicate-keep-first-keyed-table-api", + "deduplicate-keep-first-table-api", "the generic Table API process() call invokes the PTF, matching " + "the SQL form") .setupTableSource( @@ -85,9 +85,9 @@ public class DeduplicateKeepFirstTestPrograms { "sink") .build(); - public static final TableTestProgram KEYED_KEEP_FIRST_TABLE_API_WITH_ARGS = + public static final TableTestProgram TABLE_API_WITH_ARGS = TableTestProgram.of( - "deduplicate-keep-first-keyed-table-api-with-args", + "deduplicate-keep-first-table-api-with-args", "the Table API process() call passes the optional state_ttl and " + "reset_ttl_on_duplicate as named arguments") .setupTableSource( @@ -113,9 +113,9 @@ public class DeduplicateKeepFirstTestPrograms { "sink") .build(); - public static final TableTestProgram MULTI_KEY_INDEPENDENCE = + public static final TableTestProgram MULTI_KEY = TableTestProgram.of( - "deduplicate-keep-first-multi-key-independence", + "deduplicate-keep-first-multi-key", "each PARTITION BY key keeps its own first row, independently of " + "interleaving with other keys") .setupTableSource( @@ -166,9 +166,9 @@ public class DeduplicateKeepFirstTestPrograms { + "input => TABLE user_events PARTITION BY (region, user_name))") .build(); - public static final TableTestProgram NULL_PARTITION_KEY = + public static final TableTestProgram NULL_KEY = TableTestProgram.of( - "deduplicate-keep-first-null-partition-key", + "deduplicate-keep-first-null-key", "a NULL PARTITION BY key forms its own dedup group, keeping only " + "its first row") .setupTableSource( @@ -213,9 +213,9 @@ public class DeduplicateKeepFirstTestPrograms { "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(input => TABLE user_events)") .build(); - public static final TableTestProgram RESET_ON_TTL_OFF = + public static final TableTestProgram RESET_WITH_ZERO_TTL = TableTestProgram.of( - "deduplicate-keep-first-reset-on-ttl-off", + "deduplicate-keep-first-reset-with-zero-ttl", "reset_ttl_on_duplicate=TRUE with state_ttl=INTERVAL '0' (retention " + "disabled): only the first row per key is emitted") .setupTableSource( @@ -240,9 +240,9 @@ public class DeduplicateKeepFirstTestPrograms { + "reset_ttl_on_duplicate => TRUE)") .build(); - public static final TableTestProgram RESET_TTL_ON_DUPLICATE_FALSE = + public static final TableTestProgram NO_TTL_RESET = TableTestProgram.of( - "deduplicate-keep-first-reset-ttl-on-duplicate-false", + "deduplicate-keep-first-no-ttl-reset", "reset_ttl_on_duplicate=FALSE still emits only the first row per key") .setupTableSource( SourceTestStep.newBuilder("user_events") @@ -264,9 +264,9 @@ public class DeduplicateKeepFirstTestPrograms { + "reset_ttl_on_duplicate => FALSE)") .build(); - public static final TableTestProgram INPUT_COLUMN_NAMED_EVENT_TIME = + public static final TableTestProgram EVENT_TIME_COLUMN_CLASH = TableTestProgram.of( - "deduplicate-keep-first-input-column-named-event-time", + "deduplicate-keep-first-event-time-column-clash", "an input column named event_time is deduplicated like any other; " + "only the first row per key is emitted") .setupTableSource( @@ -284,9 +284,9 @@ public class DeduplicateKeepFirstTestPrograms { + "input => TABLE user_events PARTITION BY user_name)") .build(); - public static final TableTestProgram WHOLE_ROW_AS_KEY = + public static final TableTestProgram WHOLE_ROW_KEY = TableTestProgram.of( - "deduplicate-keep-first-whole-row-as-key", + "deduplicate-keep-first-whole-row-key", "PARTITION BY over every column deduplicates exact-duplicate rows, " + "keeping one row per distinct payload") .setupTableSource( @@ -304,9 +304,9 @@ public class DeduplicateKeepFirstTestPrograms { + "input => TABLE user_events PARTITION BY (user_name, action))") .build(); - public static final TableTestProgram EVENT_TIME_KEEP_EARLIEST = + public static final TableTestProgram EVENT_TIME = TableTestProgram.of( - "deduplicate-keep-first-event-time-keep-earliest", + "deduplicate-keep-first-event-time", "with on_time, the row with the smallest event time is kept per key, " + "not the first to arrive") .setupTableSource( @@ -328,46 +328,85 @@ public class DeduplicateKeepFirstTestPrograms { + "on_time => DESCRIPTOR(ts))") .build(); - public static final TableTestProgram EVENT_TIME_LATER_ARRIVAL_EARLIER_WINS = + public static final TableTestProgram EVENT_TIME_MULTI_KEY = TableTestProgram.of( - "deduplicate-keep-first-event-time-later-arrival-earlier-wins", - "a row that arrives later but carries an earlier event time replaces " - + "the candidate before it is finalized") + "deduplicate-keep-first-event-time-multi-key", + "each key emits its earliest row with that row's event time as " + + "rowtime; replacing one key's candidate leaves other keys' " + + "timers intact, and on equal event times the first arrival " + + "wins") .setupTableSource( SourceTestStep.newBuilder("user_events") .addSchema(TIMED_EVENTS_SCHEMA) .producedValues( - Row.of("Vas", "first", Instant.ofEpochMilli(5000)), - Row.of("Vas", "earlier", Instant.ofEpochMilli(2000))) + Row.of("Alice", "Hi", Instant.ofEpochMilli(1)), + Row.of("Alice", "Hello", Instant.ofEpochMilli(3)), + Row.of("Alice", "Hello world", Instant.ofEpochMilli(2)), + Row.of("Bob", "I am fine.", Instant.ofEpochMilli(3)), + Row.of("Bob", "Comment#1", Instant.ofEpochMilli(6)), + Row.of("Carol", "Comment#2", Instant.ofEpochMilli(3)), + Row.of("Carol", "Comment#3", Instant.ofEpochMilli(2)), + Row.of("Dave", "Comment#4", Instant.ofEpochMilli(4)), + Row.of("Dave", "Comment#5", Instant.ofEpochMilli(4))) .build()) .setupTableSink( SinkTestStep.newBuilder("sink") - .addSchema("user_name STRING", "action STRING") - .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "earlier")) + .addSchema( + "user_name STRING", + "action STRING", + "rowtime TIMESTAMP_LTZ(3)") + .consumedValues( + Row.ofKind( + RowKind.INSERT, + "Alice", + "Hi", + Instant.ofEpochMilli(1)), + Row.ofKind( + RowKind.INSERT, + "Bob", + "I am fine.", + Instant.ofEpochMilli(3)), + Row.ofKind( + RowKind.INSERT, + "Carol", + "Comment#3", + Instant.ofEpochMilli(2)), + Row.ofKind( + RowKind.INSERT, + "Dave", + "Comment#4", + Instant.ofEpochMilli(4))) .build()) .runSql( - "INSERT INTO sink SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST(" + "INSERT INTO sink SELECT user_name, action, rowtime FROM " + + "DEDUPLICATE_KEEP_FIRST(" + "input => TABLE user_events PARTITION BY user_name, " + "on_time => DESCRIPTOR(ts))") .build(); - public static final TableTestProgram EVENT_TIME_LATE_EVENT_DROPPED = + public static final TableTestProgram EVENT_TIME_LATE_DROPPED = TableTestProgram.of( - "deduplicate-keep-first-event-time-late-event-dropped", + "deduplicate-keep-first-event-time-late-dropped", "a row below the watermark is dropped, even though its event time is " - + "earlier than the already-finalized candidate") + + "earlier than the already-finalized candidate; a late first " + + "row for an unseen key is dropped without marking the key " + + "as seen") .setupTableSource( SourceTestStep.newBuilder("user_events") .addSchema(TIMED_EVENTS_SCHEMA) .producedValues( Row.of("Bob", "early", Instant.ofEpochMilli(1000)), Row.of("Bob", "advance", Instant.ofEpochMilli(20000)), - Row.of("Bob", "late", Instant.ofEpochMilli(500))) + Row.of("Bob", "late", Instant.ofEpochMilli(500)), + Row.of("Carol", "late", Instant.ofEpochMilli(500)), + Row.of("Carol", "on-time", Instant.ofEpochMilli(30000))) .build()) .setupTableSink( SinkTestStep.newBuilder("sink") .addSchema("user_name STRING", "action STRING") - .consumedValues(Row.ofKind(RowKind.INSERT, "Bob", "early")) + .consumedValues( + Row.ofKind(RowKind.INSERT, "Bob", "early"), + Row.ofKind(RowKind.INSERT, "Carol", "on-time")) .build()) .runSql( "INSERT INTO sink SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST(" @@ -375,17 +414,19 @@ public class DeduplicateKeepFirstTestPrograms { + "on_time => DESCRIPTOR(ts))") .build(); - public static final TableTestProgram UPDATING_INPUT_SWALLOWED = + public static final TableTestProgram UPDATING_INPUT = TableTestProgram.of( - "deduplicate-keep-first-updating-input-swallowed", - "an updating input is deduplicated into an insert-only result: the " - + "first record per key is emitted once and every later change " - + "(duplicate, -U, +U, -D) is swallowed") + "deduplicate-keep-first-updating-input", + "an updating input is deduplicated into an insert-only result: a -D " + + "before the key is seen is ignored, the first +I is emitted " + + "once and every later change (duplicate, -U, +U, -D) is " + + "swallowed") .setupTableSource( SourceTestStep.newBuilder("user_events") .addSchema(USER_EVENTS_SCHEMA) .addOption("changelog-mode", "I,UB,UA,D") .producedValues( + Row.ofKind(RowKind.DELETE, "Vas", "logout"), Row.ofKind(RowKind.INSERT, "Vas", "login"), Row.ofKind(RowKind.INSERT, "Vas", "login"), Row.ofKind(RowKind.UPDATE_BEFORE, "Vas", "login"), @@ -402,9 +443,37 @@ public class DeduplicateKeepFirstTestPrograms { + "input => TABLE user_events PARTITION BY user_name)") .build(); - public static final TableTestProgram ON_TIME_WITH_UPDATING_INPUT_FAILS = + public static final TableTestProgram UNSEEN_RETRACTION_IGNORED = TableTestProgram.of( - "deduplicate-keep-first-on-time-with-updating-input-fails", + "deduplicate-keep-first-unseen-retraction-ignored", + "a -U or -D for a key that has not been seen is ignored and does not " + + "mark the key as seen, so the next +I or +U for that key is " + + "emitted as an insert") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .addOption("changelog-mode", "I,UB,UA,D") + .producedValues( + Row.ofKind(RowKind.DELETE, "Vas", "logout"), + Row.ofKind(RowKind.UPDATE_BEFORE, "Bob", "login"), + Row.ofKind(RowKind.UPDATE_AFTER, "Bob", "click"), + Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues( + Row.ofKind(RowKind.INSERT, "Bob", "click"), + Row.ofKind(RowKind.INSERT, "Vas", "login")) + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name)") + .build(); + + public static final TableTestProgram UPDATING_EVENT_TIME_FAILS = + TableTestProgram.of( + "deduplicate-keep-first-updating-event-time-fails", "event-time mode (on_time) is rejected for updating input, because a " + "watermark asserts no earlier record will arrive while an " + "update can invalidate that") @@ -427,4 +496,48 @@ public class DeduplicateKeepFirstTestPrograms { ValidationException.class, "not supported for PTFs that consume or produce updates") .build(); + + public static final TableTestProgram NEGATIVE_TTL_FAILS = + invalidArgumentProgram( + "deduplicate-keep-first-negative-ttl-fails", + "a negative state_ttl is rejected at planning time", + "state_ttl => INTERVAL '-1' SECOND", + "The 'state_ttl' argument must not be negative"); + + public static final TableTestProgram NON_LITERAL_TTL_FAILS = + invalidArgumentProgram( + "deduplicate-keep-first-non-literal-ttl-fails", + "a state_ttl that is not a literal is rejected at planning time", + "state_ttl => INTERVAL '1' SECOND * 2", + "The 'state_ttl' argument must be a constant INTERVAL literal."); + + public static final TableTestProgram NON_LITERAL_RESET_FAILS = + invalidArgumentProgram( + "deduplicate-keep-first-non-literal-reset-fails", + "a reset_ttl_on_duplicate that is not a literal is rejected at planning time", + "reset_ttl_on_duplicate => 1 = 1", + "The 'reset_ttl_on_duplicate' argument must be a constant BOOLEAN literal."); + + private static TableTestProgram invalidArgumentProgram( + final String id, final String description, final String argument, final String error) { + return TableTestProgram.of(id, description) + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema(USER_EVENTS_SCHEMA) + .producedValues(Row.of("Vas", "login")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(USER_EVENTS_SCHEMA) + .consumedValues(new Row[0]) + .build()) + .runFailingSql( + "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + argument + + ")", + ValidationException.class, + error) + .build(); + } } diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/plan/deduplicate-keep-first-global-ttl-expired-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/plan/deduplicate-keep-first-global-ttl-expired-restore.json new file mode 100644 index 00000000000000..79cc4c4d53e63c --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/plan/deduplicate-keep-first-global-ttl-expired-restore.json @@ -0,0 +1,144 @@ +{ + "flinkVersion" : "2.4", + "nodes" : [ { + "id" : 5, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`user_events`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "user_name", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "dataType" : "VARCHAR(2147483647)" + } ] + } + } + } + }, + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events]], fields=[user_name, action])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "Exchange(distribution=[hash[user_name]])" + }, { + "id" : 7, + "type" : "stream-exec-process-table-function_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "ProcessTableFunction(invocation=[DEDUPLICATE_KEEP_FIRST(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE_KEEP_FIRST], select=[user_name,action], rowType=[RecordType(VARCHAR(2147483647) user_name, VARCHAR(2147483647) action)])", + "uid" : "DEDUPLICATE_KEEP_FIRST", + "functionCall" : { + "kind" : "CALL", + "internalName" : "$DEDUPLICATE_KEEP_FIRST$1", + "operands" : [ { + "kind" : "TABLE_ARG_CALL", + "inputIndex" : 0, + "partitionKeys" : [ 0 ], + "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "INTERVAL SECOND(6)" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "BOOLEAN" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + } ], + "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" + }, + "inputChangelogModes" : [ [ "INSERT" ] ], + "outputChangelogMode" : [ "INSERT" ], + "inputUpsertKeys" : [ [ ] ] + }, { + "id" : 8, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "user_name", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "dataType" : "VARCHAR(2147483647)" + } ] + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "upsertMaterializeStrategy" : "MAP", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_name, action])" + } ], + "edges" : [ { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..67bae9c1ae41ba77632ccd54f2f24eed9c68d657 GIT binary patch literal 16398 zcmeHOUu@gP89&O78>ji>v~F6bOXDF5*-R)> zBPmA?hRj8=tw4sR7`6cex~*Niw+t9C3@EUNJ*|D%_R?W{*~5lyz#iHa16H6|(INZp zNJ^9`HO?A4PD%nIDe~^SkKcXY-S_>zyUdrD!w8`P_|b#2FQ^Yanm2Tbm}UuAR5I7l zBumk?rT7JOXf{22E}zZJq)O@XnRGf|p34-9rFW~3d^G&>AFsdp?B5=Ht#it3 z2$ghWna6c3tq{JdD%vI9qQfC?G&D=85#Cy>lZK_JCSTAm&teM~iJ@Rsxk8LF-;<(- z>*k7X0p))9aX0>nc9{km@6iUOZStzVtVmep7qBt2f{hY9gu3+LLWs+t0hMUW)=G1; zbh0+RPCB7Qv_~;>4OOLImqoj2+XY>>EHs$~_5@o50uzD2L?AHs1cyDrLzdQ?YSy`{ z28wO!(>Bz#zYd{?5Q4ATe-W=@Ud7rnpVk^R@5{P{9#}{fW=^LHU%PqpCK`U`zf>&j zx7ed)f5#OqL(!Hu^j)`yp6=`8P^dZ%Gua5J8Zk7ihBaIxa+oTGVTNnC?l8)sp)=|8 z>Dh9zM5oKr`FtAnolWK8bqEbDq_XGImKDc^~VzuJDpxtLd6Y;VRf9 z{8?OYYn+!swC^8G$2>v_p)D2)%HIa(SzTLhvp&}Ox8qBObE&iGVm>wFlhlH*J-2&J z3tNO=&=olc{bGUv6HjA3o_Dl*vdsFj%qR(U+ZO^wDK|`Fbo4fKT3@@jH`%48We{8g zdN8^uyj~{;wsgbfOH_f{tah+ucWJ1jOM?L_7@*u4$Ez&%0JYTyKh|L+WR+-EU>_|f zfz6P?1iphO@U6Cwr#kFo<0R71yiP+fjRbz~4)t@l($6~4VIrO>B(M$^X$F(XPLBe& z(mMK0LbP^p;R_{qv2X(l^a>_V-y%e^ER`==YLUiEb6A2HWNnQ8Z^0N|r_u>3J0p;- z3Htm(?FGD~n|1`=W_is2sLNNl@XJ#^3E3^#hL^JRnj%qjpahV};TMW0KfH2mBE+DH z;~!hot@Gyfe-HmK^n5LZLRoY`(zR7U2mnf-QL$-S=wO!mM*;;n1omhuiym}P722eQ zCT<&n0rX|nKnL57*R-yy1Z%$=HP-*|@`vvMaI}batzxx7n)OX1P!18wl&g`p95x4F zc~G8q^{oJ&(6#_i)ARG$bSj5D(Kv_vTH71Cam49STi=rot-U)i$e@HZS&?CMa67FF zpmobp$S;?>W6g64QDxJ$%=dqG<1;^f?nh-Xhb-D(rSIHVV3$L9UjF;m;!INR3DE!p5 zzrXa`>&N~K>h(p}Nifh*+nLBUY87I99NY|elHQ~IJxPS%{WqP}I z;!jVOC*Qa>GV`CGaDUzV9u%hu<8@k%-q3?%^B@|g<--QC4494JYBp4hHR_Zbehuw) ziuhi=we4#g#Hc98Dsm-U8IMN8u~;G!#*uh59Er#z7Q=F)Dn(#`eh0n});2&0-{b4- z^!ey~K3zza<_k{VdNxxm6*AN3N}2gw`Rx2`8Vwa;30SGzdXc7r>ygn8-2ZI}_z(MC z0r0BW(iE!JaP);xKlo~byKnZw}+QJ4%~d@B6zse zfxDz=vV}&w;1jf60UX0KPy$NsuREmNMn^o2X2Hyed>Wf8wE6owDA^WhA_z+ALAvszOoNs?8kg*|U70S0^{V3c`VVb|qa9UER^>YFnd&End>Vq4qG?3K=%K z!GRA+{hxahLJoY$oepdVKIAX2Rx#wSWdK~;ywUV@*F|K`X9(@0f-4wW2as4M3Jw9>71@jsitgr1TK$!8 z{(krC&kn!ARs-w-9&4Co$}J@N@nT6gh(MY|f;AMnCbS66wW?9LnL=+p))EW@z-+ZU zA;p#tSm?CBscv8G<$DZJZyzO)1uQCy`h-CF=39L8UHEpjX)_tw!I@NUJ_q~iv#>1C o>EY29O<(}Tv0*oA7gcOkpuN!bYrH&B=xz|8$KttSKAoHWAE&TMk^lez literal 0 HcmV?d00001 diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/plan/deduplicate-keep-first-ttl-expired-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/plan/deduplicate-keep-first-ttl-expired-restore.json new file mode 100644 index 00000000000000..eba9144495ddfb --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/plan/deduplicate-keep-first-ttl-expired-restore.json @@ -0,0 +1,142 @@ +{ + "flinkVersion" : "2.4", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`user_events`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "user_name", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "dataType" : "VARCHAR(2147483647)" + } ] + } + } + } + }, + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events]], fields=[user_name, action])" + }, { + "id" : 2, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "Exchange(distribution=[hash[user_name]])" + }, { + "id" : 3, + "type" : "stream-exec-process-table-function_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "ProcessTableFunction(invocation=[DEDUPLICATE_KEEP_FIRST(TABLE(#0) PARTITION BY($0), 10000:INTERVAL SECOND, DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE_KEEP_FIRST], select=[user_name,action], rowType=[RecordType(VARCHAR(2147483647) user_name, VARCHAR(2147483647) action)])", + "uid" : "DEDUPLICATE_KEEP_FIRST", + "functionCall" : { + "kind" : "CALL", + "internalName" : "$DEDUPLICATE_KEEP_FIRST$1", + "operands" : [ { + "kind" : "TABLE_ARG_CALL", + "inputIndex" : 0, + "partitionKeys" : [ 0 ], + "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" + }, { + "kind" : "LITERAL", + "value" : "10000", + "type" : "INTERVAL SECOND(6) NOT NULL" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "BOOLEAN" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "DESCRIPTOR" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$DEFAULT$1", + "type" : "VARCHAR(2147483647)" + } ], + "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" + }, + "inputChangelogModes" : [ [ "INSERT" ] ], + "outputChangelogMode" : [ "INSERT" ], + "inputUpsertKeys" : [ [ ] ] + }, { + "id" : 4, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "user_name", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "action", + "dataType" : "VARCHAR(2147483647)" + } ] + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_name, action])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 4, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..750f297d9fbe871d084a15de8a1aafcdef591d75 GIT binary patch literal 16395 zcmeHOU2Gd!6~5!7Y13_+bT{p8yX|hEX{Bn{Gx1++H!nDeLrq=Bvfb=9Q89OBu9IoU zGiGL-CVePft@x2vqO1f+E0+CX7x6$5NC<(1#2Z~C-~m?fvJ#5~5)Zo|1Qjc-7Q(sL z9($az8&^$Sh-RcX^K*aBJ?EZ#&pqcmsn0C=5kdp-M=!4S5A~t_c}2T&X4Hh=#qcFG z5H8Z&Gl#$buiw0M?9PwB@y@rae>jw`SGhr}*>$;oc<`YSTHbFE%fONa|IMmunT~QJ zZ=ikkBCbzwap=X8qLMSh3O0n2uFAwPh4G?bEU%Qtua++d)KI`G8qr8m5;Zv(9*1kG zA{2znRZX${6R}7rG7$=fBcX}dcsvpf3MO756^S+1Cl=+iA96a&_pOAh6Gs( zlTu71emPc(`NI=p+#ip^ofwe=A{NQGD8>;Qc@O9x;O;Uw$^#CK&SjIiL}4yh%oh@c zWN{{yFXU2F7YeDlO!55OOcD*}v*}c!`60>b4bLai7m~$%GM7rEQwzx)y!D}Z-B{vr z1xw3>FR8#(-om0vc%!OWN}2H1T7^_CMK$^4sznS9t9fE5SXHhOW6b?3ui=WhtXt4F zNck^RSlq3@qFttep6bvO?ZM<{6{5=K47P9qp6~qN*0JwB_iE9HxHLLYqTlRKriF&n zm+=Y~C|N>UF|El}1D(8U>vrYaH~%^EitqWd5Bbt)NYb?x5WS*n(<(Nh;e)k?Rje%u z?A2r%J*4VOiiA~4mW6_8=trpWOJMvG7{3HI3ZY(BNH^37S5*)hh|(Oppp|u5 zDJg7FFt0xvIvM!U`(GYr-0N52jm^8KH*cWX@R2EWu zY&l^Vc#Z1U>T|z6{?c3cn|+8&p?(uoAJ_$p$5pl#!+0WH@vS~5#^C=f)##=?`(WbB z#FPxBQk&EdJLYuWG8Aoz&S?#lf^y(-xTg}Q_x-CC;Ok)3Bu0^b2Wq;TSo7GJUdG0T zS(-uvD$$m#+9=m1|Nl92Vn1^f}>@g;OOrt{m!Z$ zVQ(L6Vbk4Lz=Wlj70F(yj(#bB`on7{fR@+eu9mMC}13+|?K*tu~dgIsu*z~Bc{#zrIZiU;kRfFrr zs^Oe`9y{4?%sV7}uZysgoen+lp@pvXpr(EsRH5BVhc@(Wcl(@U8MN0Xmlf>UkktP1 zY)E-(&9fmr88hw9^m%S{^D?!Xjo(MJhAt7)EKt|SY*l0D z>c!9{ba*B?a{;`W)6~;>E}6_0XH&U+;r-H~k49ep{mt9Y{^_wd%IjZ;hNxkCgEmNS z8rSan-HpZ@@5aG_C$Bpmyjzb0JC*~zj`>7FvB-oRlKrw6rZJy!(I1y$QNI)j#3KPQ z6b*zS=JVuFf#DGIp^=#F^upOO7|#pW49ski+lbqKw~ntgj&`FX5ME>fI6w7s%YGW3 zGfaOOQ(rv2I5d1Nc`-Rtj|&zqW|OGzd?E{f-r*&GgDx$+E~=5-K%Q2x1;Z8r5i#Hi)v1iW)AEcH4liTcoCC5L|}#AkM4s zdW9I+(hZX@&@4l*HwgjvExXzsC3mso27K;xm^|w`lC`R?WvP6@QuFk@Gz-uv-B=r= z&ka23rYDf?IrO=goCgDe?F<0>pNF>|Y$DU9bX_G_`}L@C<99EA_%{F@E}&g2SgitR zuxoH$y^&UM9)(wS=fHA+gX{?nh~NKL(CF@|xw&*Qk!c0(xo2%>*G-+BO-&nfy=l?C z2jd1Al&~f%GOP}^(Aqe(HeH3>e0eb1%pk5i7tnsfwb<9#4C2ES{Y|lPburR^I~qFP z6Yq`(9j}o4hmJq$f{xoD?r!&%%?^zea6MSVjqhuN8$S%V@x8U92X*?wjwp1{q*j;C zl(Ll4tc@FDYi9#fsZXXd1%?+t#4z6@pn6T1{{7<5Pj#S_>3kFY+M@Q9!p9jcCjr6k z$c~_1_{kUk{FPtbJn=3V37@i!gcg{4rdk$>@r7GoJO0pD|M&ufxzh$e<^kBZD($5w zYnKyoT?eq=&#doOReNRz_JoOb?RFeGw;!>-Ee9^H0Zw-;gy@0ToDN`Y7>TWIuE}=bvqRGCz z{f}ecz5Sn|cUOM#vlkcN`8aK!;gukpQx26NVHAtfM2U=-hCXAR@|QFo=Ut zbYH!=-Jy+eV!5y!I#}yHM=&I^vePvy*lWZ?ASjCwQ4IN`O>jW-(+r!}E~+tZ0h4?0l22ie?qS1kGmhRkK1gnZ6n*=`g9`W)J{*Dh6vs zhIG*;ZPXZPn-NidXtE`}I0V*Dpc^LS?euC|{k1Q?z30tmM}Fc%kT$p% zXe?u97g;Fc&i#~hg9v1mNPu(KHKEbaY@?c;SZJZPNrxkl9&0s)kYLC}3!Qa$)o#>I zy2l{j!qpP+0y4-#E+$aA`3Bv58@d?_;isFl)4}OPW-bFD$~5FS*CjmKXcGtkc5KK3 cZ!;Ae6{r;2j>bt7Gflw&&R@u9lbM { private static final long serialVersionUID = 1L; @@ -100,24 +103,32 @@ public void eval( throws Exception { // state_ttl is applied at plan time via getTimeToLive(); the PTF codegen still requires // a parameter for every declared argument, so it is unused here. - final Long rowtime = ctx.timeContext(Long.class).time(); + final RowKind kind = input.getRowKind(); + if (kind == RowKind.UPDATE_BEFORE || kind == RowKind.DELETE) { + return; + } + + final TimeContext context = ctx.timeContext(Long.class); + final Long rowtime = context.time(); if (rowtime != null) { // watermark mode - final Long watermark = ctx.timeContext(Long.class).tableWatermark(); + final Long watermark = context.tableWatermark(); if (watermark != null && rowtime < watermark) { return; } if (seen.getValue() == null) { // not yet emitted → buffer the earliest candidate final RowData currLowest = candidate.getValue(); - if (currLowest == null || rowtime < currLowest.getLong(eventTimeIndex)) { - if (currLowest != null) { - ctx.timeContext(Long.class).clearTimer(currLowest.getLong(eventTimeIndex)); + final Long currentLowestEventTime = + currLowest == null ? null : currLowest.getLong(eventTimeIndex); + if (currentLowestEventTime == null || rowtime < currentLowestEventTime) { + if (currentLowestEventTime != null) { + context.clearTimer(currentLowestEventTime); } // object reuse may overwrite the input's memory before the timer fires candidate.setValue( candidateSerializer.copy(materializeCandidate(input, rowtime))); - ctx.timeContext(Long.class).registerOnTime(rowtime); + context.registerOnTime(rowtime); } } else { // already emitted → drop, optionally refresh the TTL if (resetTtlOnDuplicate == null || resetTtlOnDuplicate) { From 00d6ba2e4302428fd9f790653ec43fc4c9fdc74d Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Mon, 12 Oct 2026 12:07:57 +0100 Subject: [PATCH 3/5] [FLINK-40896][table] Nest dedup candidate state, clarify TTL and uid docs --- .../docs/sql/functions/built-in-functions.md | 2 +- .../queries/deduplicate-keep-first.md | 39 ++++++----- .../docs/sql/functions/built-in-functions.md | 2 +- .../queries/deduplicate-keep-first.md | 39 ++++++----- .../functions/BuiltInFunctionDefinitions.java | 6 +- .../DeduplicateKeepFirstTypeStrategy.java | 32 +++++---- .../SpecificInputTypeStrategies.java | 4 -- .../strategies/SpecificTypeStrategies.java | 4 -- .../DeduplicateKeepFirstTypeStrategyTest.java | 46 ++++++++++++- .../DeduplicateKeepFirstSemanticTests.java | 2 +- .../DeduplicateKeepFirstTestPrograms.java | 63 +++++++++++------- ...keep-first-event-time-pending-restore.json | 2 +- .../savepoint/_metadata | Bin 17098 -> 18262 bytes ...keep-first-global-ttl-expired-restore.json | 21 +++--- .../savepoint/_metadata | Bin 16398 -> 14647 bytes .../savepoint/_metadata | Bin 16582 -> 14699 bytes ...licate-keep-first-ttl-expired-restore.json | 21 +++--- .../savepoint/_metadata | Bin 16395 -> 14647 bytes .../functions/ptf/DeduplicateKeepFirst.java | 41 +++++------- 19 files changed, 186 insertions(+), 138 deletions(-) diff --git a/docs/content.zh/docs/sql/functions/built-in-functions.md b/docs/content.zh/docs/sql/functions/built-in-functions.md index 9b01e5858cb40f..79115623e0aa74 100644 --- a/docs/content.zh/docs/sql/functions/built-in-functions.md +++ b/docs/content.zh/docs/sql/functions/built-in-functions.md @@ -289,7 +289,7 @@ Table functions can be used in two ways: as stand-alone inputs, where they are i | `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See Changelog Conversion for the full list of arguments, semantics, and usage. | | `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See Changelog Conversion for the full list of arguments, semantics, and usage. | | `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | -| `DEDUPLICATE_KEEP_FIRST(input => TABLE t [, ...])` | Keeps the first record per key as an insert-only stream (first arrival, or earliest event time with `on_time`). See [Deduplicate Keep First]({{< ref "docs/sql/reference/queries/deduplicate-keep-first" >}}#deduplicate_keep_first) for the full list of arguments, semantics, and usage. | +| `DEDUPLICATE_KEEP_FIRST(input => TABLE t [, ...])` | Keeps the first record per key as an append-only table (first arrival, or earliest event time with `on_time`). See [Deduplicate Keep First]({{< ref "docs/sql/reference/queries/deduplicate-keep-first" >}}#deduplicate_keep_first) for the full list of arguments, semantics, and usage. | To implement your own table functions, see [user-defined table functions]({{< ref "docs/dev/table/functions/udfs" >}}#table-functions). diff --git a/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md b/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md index c7bb5464fa38f5..10ab2f787509e6 100644 --- a/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md +++ b/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md @@ -26,11 +26,11 @@ under the License. {{< label Streaming >}} -Flink SQL provides the `DEDUPLICATE_KEEP_FIRST` process table function (PTF) for removing duplicate rows while keeping only the **first** record per key. Its output is always **insert-only**, which makes it a concise, append-in / append-out alternative to the `ROW_NUMBER()` [deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}) pattern. +Flink SQL provides the `DEDUPLICATE_KEEP_FIRST` process table function (PTF) for removing duplicate rows while keeping only the **first** record per key. Its output is always **insert-only**, which makes it a concise alternative to the `ROW_NUMBER()` [deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}) pattern with more configuration options. | Function | Description | |:---------|:------------| -| [DEDUPLICATE_KEEP_FIRST](#deduplicate_keep_first) | Keeps the first record per key as an insert-only stream (first arrival, or earliest event time with `on_time`) | +| [DEDUPLICATE_KEEP_FIRST](#deduplicate_keep_first) | Keeps the first record per key as an append-only table (first arrival, or earliest event time with `on_time`) | ## DEDUPLICATE_KEEP_FIRST @@ -39,31 +39,35 @@ The `DEDUPLICATE_KEEP_FIRST` PTF keeps the first record per key and drops every * **Processing time (default):** without `on_time`, the first record *observed* for a key is emitted; later records for that key are dropped while its state exists. This requires no watermark or event-time attribute. * **Event time:** with `on_time`, the record with the *smallest event time* per key is kept and emitted once the watermark makes the choice final. A record arriving later but carrying an earlier event time replaces the candidate before finalization; records that can no longer become the earliest are dropped as late. -The input may be insert-only or updating; the output is insert-only in every case. +The input may be an append-only or updating table; the output is append-only in every case. ### Syntax ```sql SELECT * FROM DEDUPLICATE_KEEP_FIRST( - input => TABLE source_table [PARTITION BY key_col [, key_col2 ...]], - [on_time => DESCRIPTOR(rowtime_column),] + input => TABLE source_table [PARTITION BY (key_col [, key_col2 ...])], [state_ttl => ,] - [reset_ttl_on_duplicate => ] + [reset_ttl_on_duplicate => ,] + [on_time => DESCRIPTOR(rowtime_column),] + [uid => ] ) ``` +`on_time` and `uid` are system arguments and always come last in the signature, which matters when arguments are passed by position. + ### Parameters | Parameter | Required | Description | |:----------|:---------|:------------| -| `input` | Yes | The input table (insert-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | -| `on_time` | No | A `DESCRIPTOR` naming a single rowtime attribute. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires insert-only input; combining `on_time` with an updating input is rejected at planning time. | -| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. If omitted, falls back to `table.exec.state.ttl`, which retains state indefinitely at its default of 0. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. `INTERVAL '0'` disables retention. | +| `input` | Yes | The input table (append-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | +| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. If omitted, falls back to `table.exec.state.ttl`; at its default of 0, state is retained indefinitely, but a retention can still be introduced later. `INTERVAL '0'` also retains state indefinitely, but excludes the TTL from the state layout, so a retention cannot be introduced later without losing the state. See [State TTL]({{< ref "docs/dev/table/functions/ptfs" >}}#state-ttl). | | `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful when a TTL applies, whether set through `state_ttl` or inherited from `table.exec.state.ttl`. | +| `on_time` | No | A `DESCRIPTOR` naming a single watermarked column. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires append-only input; combining `on_time` with an updating input is rejected at planning time. | +| `uid` | No | A unique identifier for the function's state in checkpoints and savepoints. Defaults to the function name, so it is required when `DEDUPLICATE_KEEP_FIRST` is called more than once in the same job. See [Query Evolution with UIDs]({{< ref "docs/dev/table/functions/ptfs" >}}#query-evolution-with-uids). | ### Output Schema -When `PARTITION BY` is used, the partition key columns are prepended to the output, followed by the remaining (non-key) input columns. In event-time mode a rowtime column is appended. +When `PARTITION BY` is used, the partition key columns are prepended to the output, followed by the remaining (non-key) input columns. In event-time mode a rowtime column is appended for (potentially following) time-based operations. ``` [partition_key_columns] + [remaining_input_columns] @@ -88,23 +92,24 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( The first row for `Vas` is emitted; the later `click` on the same key is dropped. -#### Whole input, no PARTITION BY +#### Exact duplicates (all columns as key) ```sql -- Input (append-only): --- +I[user_name:'Vas', action:'login'] --- +I[user_name:'Alice', action:'click'] --- +I[user_name:'Bob', action:'view'] +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'click'] SELECT * FROM DEDUPLICATE_KEEP_FIRST( - input => TABLE user_events + input => TABLE user_events PARTITION BY (user_name, action) ) -- Output (insert-only): -- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'click'] ``` -Without `PARTITION BY` the whole input is a single group at parallelism 1, so only the first row of the entire stream is emitted; every later row is dropped regardless of its content. +Partitioning by every column drops only exact duplicate rows while still distributing the work across parallel instances. #### Bounded state with state_ttl @@ -165,7 +170,7 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( -- +I[user_name:'Vas', action:'login'] ``` -The input may be an updating changelog. `DEDUPLICATE_KEEP_FIRST` keeps the first `+I` or `+U` observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. A `-U` or `-D` for a key that has not been seen yet is ignored and does not mark the key as seen, so the next `+I` or `+U` for that key is still emitted. Event-time mode (`on_time`) is not supported with updating input. +The input may be an [updating changelog]({{< ref "docs/dev/table/functions/ptfs" >}}#change-data-capture-basics) containing inserts (`+I`), update-befores (`-U`), update-afters (`+U`), and deletes (`-D`). `DEDUPLICATE_KEEP_FIRST` keeps the first `+I` or `+U` observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. A `-U` or `-D` for a key that has not been seen yet is ignored and does not mark the key as seen, so the next `+I` or `+U` for that key is still emitted. Event-time mode (`on_time`) is not supported with updating input. #### Table API diff --git a/docs/content/docs/sql/functions/built-in-functions.md b/docs/content/docs/sql/functions/built-in-functions.md index fe0a84c8a0bb2c..01bf207057959f 100644 --- a/docs/content/docs/sql/functions/built-in-functions.md +++ b/docs/content/docs/sql/functions/built-in-functions.md @@ -292,7 +292,7 @@ Table functions can be used in two ways: as stand-alone inputs, where they are i | `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#from_changelog) for the full list of arguments, semantics, and usage. | | `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#to_changelog) for the full list of arguments, semantics, and usage. | | `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | -| `DEDUPLICATE_KEEP_FIRST(input => TABLE t [, ...])` | Keeps the first record per key as an insert-only stream (first arrival, or earliest event time with `on_time`). See [Deduplicate Keep First]({{< ref "docs/sql/reference/queries/deduplicate-keep-first" >}}#deduplicate_keep_first) for the full list of arguments, semantics, and usage. | +| `DEDUPLICATE_KEEP_FIRST(input => TABLE t [, ...])` | Keeps the first record per key as an append-only table (first arrival, or earliest event time with `on_time`). See [Deduplicate Keep First]({{< ref "docs/sql/reference/queries/deduplicate-keep-first" >}}#deduplicate_keep_first) for the full list of arguments, semantics, and usage. | To implement your own table functions, see [user-defined table functions]({{< ref "docs/dev/table/functions/udfs" >}}#table-functions). diff --git a/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md b/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md index c7bb5464fa38f5..10ab2f787509e6 100644 --- a/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md +++ b/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md @@ -26,11 +26,11 @@ under the License. {{< label Streaming >}} -Flink SQL provides the `DEDUPLICATE_KEEP_FIRST` process table function (PTF) for removing duplicate rows while keeping only the **first** record per key. Its output is always **insert-only**, which makes it a concise, append-in / append-out alternative to the `ROW_NUMBER()` [deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}) pattern. +Flink SQL provides the `DEDUPLICATE_KEEP_FIRST` process table function (PTF) for removing duplicate rows while keeping only the **first** record per key. Its output is always **insert-only**, which makes it a concise alternative to the `ROW_NUMBER()` [deduplication]({{< ref "docs/sql/reference/queries/deduplication" >}}) pattern with more configuration options. | Function | Description | |:---------|:------------| -| [DEDUPLICATE_KEEP_FIRST](#deduplicate_keep_first) | Keeps the first record per key as an insert-only stream (first arrival, or earliest event time with `on_time`) | +| [DEDUPLICATE_KEEP_FIRST](#deduplicate_keep_first) | Keeps the first record per key as an append-only table (first arrival, or earliest event time with `on_time`) | ## DEDUPLICATE_KEEP_FIRST @@ -39,31 +39,35 @@ The `DEDUPLICATE_KEEP_FIRST` PTF keeps the first record per key and drops every * **Processing time (default):** without `on_time`, the first record *observed* for a key is emitted; later records for that key are dropped while its state exists. This requires no watermark or event-time attribute. * **Event time:** with `on_time`, the record with the *smallest event time* per key is kept and emitted once the watermark makes the choice final. A record arriving later but carrying an earlier event time replaces the candidate before finalization; records that can no longer become the earliest are dropped as late. -The input may be insert-only or updating; the output is insert-only in every case. +The input may be an append-only or updating table; the output is append-only in every case. ### Syntax ```sql SELECT * FROM DEDUPLICATE_KEEP_FIRST( - input => TABLE source_table [PARTITION BY key_col [, key_col2 ...]], - [on_time => DESCRIPTOR(rowtime_column),] + input => TABLE source_table [PARTITION BY (key_col [, key_col2 ...])], [state_ttl => ,] - [reset_ttl_on_duplicate => ] + [reset_ttl_on_duplicate => ,] + [on_time => DESCRIPTOR(rowtime_column),] + [uid => ] ) ``` +`on_time` and `uid` are system arguments and always come last in the signature, which matters when arguments are passed by position. + ### Parameters | Parameter | Required | Description | |:----------|:---------|:------------| -| `input` | Yes | The input table (insert-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | -| `on_time` | No | A `DESCRIPTOR` naming a single rowtime attribute. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires insert-only input; combining `on_time` with an updating input is rejected at planning time. | -| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. If omitted, falls back to `table.exec.state.ttl`, which retains state indefinitely at its default of 0. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. `INTERVAL '0'` disables retention. | +| `input` | Yes | The input table (append-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | +| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. If omitted, falls back to `table.exec.state.ttl`; at its default of 0, state is retained indefinitely, but a retention can still be introduced later. `INTERVAL '0'` also retains state indefinitely, but excludes the TTL from the state layout, so a retention cannot be introduced later without losing the state. See [State TTL]({{< ref "docs/dev/table/functions/ptfs" >}}#state-ttl). | | `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful when a TTL applies, whether set through `state_ttl` or inherited from `table.exec.state.ttl`. | +| `on_time` | No | A `DESCRIPTOR` naming a single watermarked column. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires append-only input; combining `on_time` with an updating input is rejected at planning time. | +| `uid` | No | A unique identifier for the function's state in checkpoints and savepoints. Defaults to the function name, so it is required when `DEDUPLICATE_KEEP_FIRST` is called more than once in the same job. See [Query Evolution with UIDs]({{< ref "docs/dev/table/functions/ptfs" >}}#query-evolution-with-uids). | ### Output Schema -When `PARTITION BY` is used, the partition key columns are prepended to the output, followed by the remaining (non-key) input columns. In event-time mode a rowtime column is appended. +When `PARTITION BY` is used, the partition key columns are prepended to the output, followed by the remaining (non-key) input columns. In event-time mode a rowtime column is appended for (potentially following) time-based operations. ``` [partition_key_columns] + [remaining_input_columns] @@ -88,23 +92,24 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( The first row for `Vas` is emitted; the later `click` on the same key is dropped. -#### Whole input, no PARTITION BY +#### Exact duplicates (all columns as key) ```sql -- Input (append-only): --- +I[user_name:'Vas', action:'login'] --- +I[user_name:'Alice', action:'click'] --- +I[user_name:'Bob', action:'view'] +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'click'] SELECT * FROM DEDUPLICATE_KEEP_FIRST( - input => TABLE user_events + input => TABLE user_events PARTITION BY (user_name, action) ) -- Output (insert-only): -- +I[user_name:'Vas', action:'login'] +-- +I[user_name:'Vas', action:'click'] ``` -Without `PARTITION BY` the whole input is a single group at parallelism 1, so only the first row of the entire stream is emitted; every later row is dropped regardless of its content. +Partitioning by every column drops only exact duplicate rows while still distributing the work across parallel instances. #### Bounded state with state_ttl @@ -165,7 +170,7 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( -- +I[user_name:'Vas', action:'login'] ``` -The input may be an updating changelog. `DEDUPLICATE_KEEP_FIRST` keeps the first `+I` or `+U` observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. A `-U` or `-D` for a key that has not been seen yet is ignored and does not mark the key as seen, so the next `+I` or `+U` for that key is still emitted. Event-time mode (`on_time`) is not supported with updating input. +The input may be an [updating changelog]({{< ref "docs/dev/table/functions/ptfs" >}}#change-data-capture-basics) containing inserts (`+I`), update-befores (`-U`), update-afters (`+U`), and deletes (`-D`). `DEDUPLICATE_KEEP_FIRST` keeps the first `+I` or `+U` observed per key and swallows every later change to that key (duplicate, `-U`, `+U`, `-D`), so the output stays insert-only. A `-U` or `-D` for a key that has not been seen yet is ignored and does not mark the key as seen, so the next `+I` or `+U` for that key is still emitted. Event-time mode (`on_time`) is not supported with updating input. #### Table API diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index cd28a5c5e45144..00d247c6e54934 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -109,7 +109,6 @@ import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.ARRAY_ELEMENT_ARG; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.ARRAY_FULLY_COMPARABLE; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.ARRAY_OF_ENTRIES_ARG; -import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.DEDUPLICATE_KEEP_FIRST_INPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.FROM_CHANGELOG_INPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.INDEX; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.JSON_ARGUMENT; @@ -123,7 +122,6 @@ import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.percentageArray; import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.plainJsonPath; import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.ARRAY_APPEND_PREPEND; -import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.DEDUPLICATE_KEEP_FIRST_OUTPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.FROM_CHANGELOG_OUTPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.LATERAL_SNAPSHOT_OUTPUT_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.ML_PREDICT_OUTPUT_TYPE_STRATEGY; @@ -960,8 +958,8 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) "state_ttl", DataTypes.INTERVAL(DataTypes.SECOND()), true), StaticArgument.scalar( "reset_ttl_on_duplicate", DataTypes.BOOLEAN(), true)) - .inputTypeStrategy(DEDUPLICATE_KEEP_FIRST_INPUT_TYPE_STRATEGY) - .outputTypeStrategy(DEDUPLICATE_KEEP_FIRST_OUTPUT_TYPE_STRATEGY) + .inputTypeStrategy(DeduplicateKeepFirstTypeStrategy.INPUT_TYPE_STRATEGY) + .outputTypeStrategy(DeduplicateKeepFirstTypeStrategy.OUTPUT_TYPE_STRATEGY) .stateTypeStrategies(DeduplicateKeepFirstTypeStrategy.stateTypeStrategies()) .runtimeClass( "org.apache.flink.table.runtime.functions.ptf.DeduplicateKeepFirst") diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java index 4a05b90a6807f8..65802ed66fbe35 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java @@ -31,12 +31,10 @@ import org.apache.flink.table.types.inference.TypeStrategy; import java.time.Duration; -import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; /** Type strategies for the {@code DEDUPLICATE_KEEP_FIRST} process table function. */ @@ -115,26 +113,24 @@ public Optional inferType(final CallContext callContext) { () -> new ValidationException( "First argument must be a table for DEDUPLICATE_KEEP_FIRST.")); + // Only event-time mode buffers a candidate. Without on_time a fixed type keeps + // the unused state independent of the input schema across query evolution. + if (tableSemantics.timeColumn() == -1) { + return Optional.of(ValueView.newValueViewDataType(DataTypes.BOOLEAN())); + } final List inputFields = DataType.getFields(tableSemantics.dataType()); - final List candidateFields = + final List payloadFields = Arrays.stream( ChangelogTypeStrategyUtils.computeOutputIndices( tableSemantics)) .mapToObj(inputFields::get) - .collect(Collectors.toCollection(ArrayList::new)); - // The trailing timestamp field is read positionally at runtime, so its name - // only needs to avoid clashing with a payload column of the same name. - final Set takenNames = - candidateFields.stream() - .map(Field::getName) - .collect(Collectors.toSet()); - String timestampField = "event_time"; - for (int i = 0; takenNames.contains(timestampField); i++) { - timestampField = "event_time_" + i; - } - candidateFields.add(DataTypes.FIELD(timestampField, DataTypes.BIGINT())); + .collect(Collectors.toList()); return Optional.of( - ValueView.newValueViewDataType(DataTypes.ROW(candidateFields))); + ValueView.newValueViewDataType( + DataTypes.ROW( + DataTypes.FIELD( + "payload", DataTypes.ROW(payloadFields)), + DataTypes.FIELD("event_time", DataTypes.BIGINT())))); } @Override @@ -168,7 +164,9 @@ private static Optional> validateStateTtl( return callContext.fail( throwOnFailure, "The 'state_ttl' argument must not be negative, but was '%s'. " - + "Use INTERVAL '0' to disable state retention.", + + "Use a positive INTERVAL to bound state retention, INTERVAL '0' to " + + "retain state indefinitely without a TTL, or omit the argument to " + + "fall back to 'table.exec.state.ttl'.", stateTtl.get()); } return Optional.empty(); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java index 40c0e2ae70da50..e012f310953b3b 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificInputTypeStrategies.java @@ -142,10 +142,6 @@ public static InputTypeStrategy plainJsonPath(final InputTypeStrategy signatures public static final InputTypeStrategy FROM_CHANGELOG_INPUT_TYPE_STRATEGY = FromChangelogTypeStrategy.INPUT_TYPE_STRATEGY; - /** Input strategy for {@link BuiltInFunctionDefinitions#DEDUPLICATE_KEEP_FIRST}. */ - public static final InputTypeStrategy DEDUPLICATE_KEEP_FIRST_INPUT_TYPE_STRATEGY = - DeduplicateKeepFirstTypeStrategy.INPUT_TYPE_STRATEGY; - /** Input strategy for {@link BuiltInFunctionDefinitions#SNAPSHOT}. */ public static final InputTypeStrategy LATERAL_SNAPSHOT_INPUT_TYPE_STRATEGY = LateralSnapshotTypeStrategy.INPUT_TYPE_STRATEGY; diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificTypeStrategies.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificTypeStrategies.java index 83b522e3c512ae..00a82abb95d95c 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificTypeStrategies.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/SpecificTypeStrategies.java @@ -243,10 +243,6 @@ public final class SpecificTypeStrategies { public static final TypeStrategy FROM_CHANGELOG_OUTPUT_TYPE_STRATEGY = FromChangelogTypeStrategy.OUTPUT_TYPE_STRATEGY; - /** Type strategy specific for {@link BuiltInFunctionDefinitions#DEDUPLICATE_KEEP_FIRST}. */ - public static final TypeStrategy DEDUPLICATE_KEEP_FIRST_OUTPUT_TYPE_STRATEGY = - DeduplicateKeepFirstTypeStrategy.OUTPUT_TYPE_STRATEGY; - /** Type strategy specific for {@link BuiltInFunctionDefinitions#SNAPSHOT}. */ public static final TypeStrategy LATERAL_SNAPSHOT_OUTPUT_TYPE_STRATEGY = LateralSnapshotTypeStrategy.OUTPUT_TYPE_STRATEGY; diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java index 81a675c35763d0..09b5a58cf9d9f9 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java @@ -18,24 +18,36 @@ package org.apache.flink.table.types.inference.strategies; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.dataview.ValueView; +import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.inference.utils.CallContextMock; +import org.apache.flink.table.types.inference.utils.TableSemanticsMock; import org.junit.jupiter.api.Test; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; +import static org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy.ARG_INPUT; import static org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy.ARG_RESET_TTL_ON_DUPLICATE; import static org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy.ARG_STATE_TTL; import static org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy.CANDIDATE_STATE_TYPE_STRATEGY; import static org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy.SEEN_STATE_TYPE_STRATEGY; import static org.assertj.core.api.Assertions.assertThat; -/** Tests for the state retention contract of {@link DeduplicateKeepFirstTypeStrategy}. */ +/** Tests for the state contract of {@link DeduplicateKeepFirstTypeStrategy}. */ class DeduplicateKeepFirstTypeStrategyTest { + private static final DataType INPUT_TYPE = + DataTypes.ROW( + DataTypes.FIELD("user_name", DataTypes.STRING()), + DataTypes.FIELD("action", DataTypes.STRING()), + DataTypes.FIELD("ts", DataTypes.TIMESTAMP_LTZ(3))); + @Test void testCandidateStateDisablesTtl() { // The event-time candidate buffer is cleared by its timer, so it must declare a disabled @@ -46,6 +58,28 @@ void testCandidateStateDisablesTtl() { .contains(Duration.ZERO); } + @Test + void testCandidateStateWithoutOnTimeIsFixed() { + assertThat(CANDIDATE_STATE_TYPE_STRATEGY.inferType(callContext(-1))) + .contains(ValueView.newValueViewDataType(DataTypes.BOOLEAN())); + } + + @Test + void testCandidateStateWithOnTimeNestsPayload() { + assertThat(CANDIDATE_STATE_TYPE_STRATEGY.inferType(callContext(2))) + .contains( + ValueView.newValueViewDataType( + DataTypes.ROW( + DataTypes.FIELD( + "payload", + DataTypes.ROW( + DataTypes.FIELD( + "action", DataTypes.STRING()), + DataTypes.FIELD( + "ts", DataTypes.TIMESTAMP_LTZ(3)))), + DataTypes.FIELD("event_time", DataTypes.BIGINT())))); + } + @Test void testSeenStateTtlFollowsStateTtlArgument() { assertThat(SEEN_STATE_TYPE_STRATEGY.getTimeToLive(callContext(Duration.ofSeconds(5)))) @@ -63,4 +97,14 @@ private static CallContextMock callContext(Duration stateTtl) { callContext.argumentValues = values; return callContext; } + + private static CallContextMock callContext(final int timeColumn) { + final CallContextMock callContext = new CallContextMock(); + callContext.tableSemantics = + Map.of( + ARG_INPUT, + new TableSemanticsMock( + INPUT_TYPE, new int[] {0}, new int[0], timeColumn, null)); + return callContext; + } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java index fc3e699597d4e3..e5d0438cfe1771 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java @@ -48,9 +48,9 @@ public List programs() { DeduplicateKeepFirstTestPrograms.NO_PARTITION_BY, DeduplicateKeepFirstTestPrograms.RESET_WITH_ZERO_TTL, DeduplicateKeepFirstTestPrograms.NO_TTL_RESET, - DeduplicateKeepFirstTestPrograms.EVENT_TIME_COLUMN_CLASH, DeduplicateKeepFirstTestPrograms.WHOLE_ROW_KEY, DeduplicateKeepFirstTestPrograms.EVENT_TIME, + DeduplicateKeepFirstTestPrograms.EVENT_TIME_CANDIDATE_FIELD_NAMES, DeduplicateKeepFirstTestPrograms.EVENT_TIME_MULTI_KEY, DeduplicateKeepFirstTestPrograms.EVENT_TIME_LATE_DROPPED, DeduplicateKeepFirstTestPrograms.UPDATING_INPUT, diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java index 6997ae957ff56a..2b0bf1ab571c1f 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java @@ -216,7 +216,7 @@ public class DeduplicateKeepFirstTestPrograms { public static final TableTestProgram RESET_WITH_ZERO_TTL = TableTestProgram.of( "deduplicate-keep-first-reset-with-zero-ttl", - "reset_ttl_on_duplicate=TRUE with state_ttl=INTERVAL '0' (retention " + "reset_ttl_on_duplicate=TRUE with state_ttl=INTERVAL '0' (TTL " + "disabled): only the first row per key is emitted") .setupTableSource( SourceTestStep.newBuilder("user_events") @@ -264,26 +264,6 @@ public class DeduplicateKeepFirstTestPrograms { + "reset_ttl_on_duplicate => FALSE)") .build(); - public static final TableTestProgram EVENT_TIME_COLUMN_CLASH = - TableTestProgram.of( - "deduplicate-keep-first-event-time-column-clash", - "an input column named event_time is deduplicated like any other; " - + "only the first row per key is emitted") - .setupTableSource( - SourceTestStep.newBuilder("user_events") - .addSchema("user_name STRING", "event_time BIGINT") - .producedValues(Row.of("Vas", 100L), Row.of("Vas", 200L)) - .build()) - .setupTableSink( - SinkTestStep.newBuilder("sink") - .addSchema("user_name STRING", "event_time BIGINT") - .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", 100L)) - .build()) - .runSql( - "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" - + "input => TABLE user_events PARTITION BY user_name)") - .build(); - public static final TableTestProgram WHOLE_ROW_KEY = TableTestProgram.of( "deduplicate-keep-first-whole-row-key", @@ -292,12 +272,17 @@ public class DeduplicateKeepFirstTestPrograms { .setupTableSource( SourceTestStep.newBuilder("user_events") .addSchema(USER_EVENTS_SCHEMA) - .producedValues(Row.of("Vas", "login"), Row.of("Vas", "login")) + .producedValues( + Row.of("Vas", "login"), + Row.of("Vas", "login"), + Row.of("Vas", "click")) .build()) .setupTableSink( SinkTestStep.newBuilder("sink") .addSchema(USER_EVENTS_SCHEMA) - .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) + .consumedValues( + Row.ofKind(RowKind.INSERT, "Vas", "login"), + Row.ofKind(RowKind.INSERT, "Vas", "click")) .build()) .runSql( "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" @@ -328,6 +313,38 @@ public class DeduplicateKeepFirstTestPrograms { + "on_time => DESCRIPTOR(ts))") .build(); + public static final TableTestProgram EVENT_TIME_CANDIDATE_FIELD_NAMES = + TableTestProgram.of( + "deduplicate-keep-first-event-time-candidate-field-names", + "with on_time, input columns named like the buffered candidate's " + + "fields are emitted unchanged for the earliest row") + .setupTableSource( + SourceTestStep.newBuilder("user_events") + .addSchema( + "user_name STRING", + "payload STRING", + "event_time BIGINT", + "ts TIMESTAMP_LTZ(3)", + "WATERMARK FOR ts AS ts - INTERVAL '10' SECOND") + .producedValues( + Row.of("Vas", "login", 3L, Instant.ofEpochMilli(3000)), + Row.of("Vas", "click", 1L, Instant.ofEpochMilli(1000))) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema( + "user_name STRING", + "payload STRING", + "event_time BIGINT") + .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "click", 1L)) + .build()) + .runSql( + "INSERT INTO sink SELECT user_name, payload, event_time " + + "FROM DEDUPLICATE_KEEP_FIRST(" + + "input => TABLE user_events PARTITION BY user_name, " + + "on_time => DESCRIPTOR(ts))") + .build(); + public static final TableTestProgram EVENT_TIME_MULTI_KEY = TableTestProgram.of( "deduplicate-keep-first-event-time-multi-key", diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/plan/deduplicate-keep-first-event-time-pending-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/plan/deduplicate-keep-first-event-time-pending-restore.json index 0b6cdd7600ad32..0f0841b716420f 100644 --- a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/plan/deduplicate-keep-first-event-time-pending-restore.json +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/plan/deduplicate-keep-first-event-time-pending-restore.json @@ -309,7 +309,7 @@ } }, "inputChangelogMode" : [ "INSERT" ], - "upsertMaterializeStrategy" : "MAP", + "upsertMaterializeStrategy" : "ADAPTIVE", "inputProperties" : [ { "requiredDistribution" : { "type" : "UNKNOWN" diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/savepoint/_metadata index f895fbd95b2e952941327e3ce5b50e0d7e0234b3..1ea051ac1b999a98257eda15b18583e779340b25 100644 GIT binary patch delta 602 zcmb7>&ubGw6vtSF{Cd$Mx zur5PQ-Y5xpFS#HaBXPaDcKt?mZGSi%rjt&?JNbolQUIaS1i3Ofkv?|n^>>toAIb`E zRU{0V{9S&aZD39={K4=|Y{8fL=oCgdFNmW~8wqh2t}Wato~#E=guSpGZ^lt8gqKnu zD(Wg>qqWeHr#3TU;@PqM%SN=_jS)J8BlZ3I|Npe>o4V<19??9_AuM1n@pacHOgGGc O*`ZOd`~20#^2{+t$e;uO delta 399 zcmccC$9Srhal=O@CKbVrjU9}Wt6A*hjf{*eEG-NSO^wZs%?&M03@ps`ixbOI3-UAb zN^}#A4U&>Abj>Z(3{or&l9E!5&Ge1aEG-Pv49s*B4a^O7O-wBebQ4WXlXX)QlPxR~ z&C?7MQ!*1()4?F9KbnFqv0RVzR8>!8p=G!y+}=!py`n2^a>crn)9(Nv6O+GfC1l lGcq$zOEpR}OG!0kU=W!J3XREmdNP~mSekHc?y)?@3jqGuZOH%t diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/plan/deduplicate-keep-first-global-ttl-expired-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/plan/deduplicate-keep-first-global-ttl-expired-restore.json index 79cc4c4d53e63c..93ca4313bcfd7c 100644 --- a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/plan/deduplicate-keep-first-global-ttl-expired-restore.json +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/plan/deduplicate-keep-first-global-ttl-expired-restore.json @@ -1,7 +1,7 @@ { "flinkVersion" : "2.4", "nodes" : [ { - "id" : 5, + "id" : 14, "type" : "stream-exec-table-source-scan_2", "scanTableSource" : { "table" : { @@ -22,7 +22,7 @@ "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events]], fields=[user_name, action])" }, { - "id" : 6, + "id" : 15, "type" : "stream-exec-exchange_1", "inputProperties" : [ { "requiredDistribution" : { @@ -35,7 +35,7 @@ "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", "description" : "Exchange(distribution=[hash[user_name]])" }, { - "id" : 7, + "id" : 16, "type" : "stream-exec-process-table-function_1", "inputProperties" : [ { "requiredDistribution" : { @@ -82,7 +82,7 @@ "outputChangelogMode" : [ "INSERT" ], "inputUpsertKeys" : [ [ ] ] }, { - "id" : 8, + "id" : 17, "type" : "stream-exec-sink_2", "configuration" : { "table.exec.sink.keyed-shuffle" : "AUTO", @@ -108,7 +108,6 @@ } }, "inputChangelogMode" : [ "INSERT" ], - "upsertMaterializeStrategy" : "MAP", "inputProperties" : [ { "requiredDistribution" : { "type" : "UNKNOWN" @@ -120,22 +119,22 @@ "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_name, action])" } ], "edges" : [ { - "source" : 5, - "target" : 6, + "source" : 14, + "target" : 15, "shuffle" : { "type" : "FORWARD" }, "shuffleMode" : "PIPELINED" }, { - "source" : 6, - "target" : 7, + "source" : 15, + "target" : 16, "shuffle" : { "type" : "FORWARD" }, "shuffleMode" : "PIPELINED" }, { - "source" : 7, - "target" : 8, + "source" : 16, + "target" : 17, "shuffle" : { "type" : "FORWARD" }, diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/savepoint/_metadata index 67bae9c1ae41ba77632ccd54f2f24eed9c68d657..416039a2736d7d711388afb1645e3731d0ea7fd0 100644 GIT binary patch delta 561 zcmbV}v1=4T6vk&R;vq;-Jjk6OIY|OGLuPi~?#x_P(xnz+AslhpnVo?{*^}M7Q&2E} zL5PeNVr5|=*c^*kiXz(B2qGeal?Jp)qlFgd8vG0L;C=9TkH`1p+u6p?irr4TeW4f{ zTb377ebsJ^b2AzY^FE8FG{uef$Lb;e)VQ-d?{6MiPgoF!VFa}Rql^cj*lL+h?6u(_ zACXKd4jG9w3#l_HkCwr=0*{g0=F)tni_>rXAU8B6jk`dQlpBS z(^J*rLWOv%S+@BY{P<{-b)>z1|lnb<;2CX#={uP66@pCq?F zeA$a%t$!{a``L1g@B4V=G=A!DuN?+Lf(f)Sj2ID60#-YsK>}f5L2{Y~flU!=y{nzo z5IwWs> zTXw1bg6rPKs4J~8no;4X(;~`1X%jihsS!4+V~~Dh?ZzQO0h(pnIYG3coB)S_C~zT> ZsSqkuptxeSgYO4C{!IL}I)1(}_ZP7PsZjs` delta 1007 zcmbVL%TE(Q9GVUZ)y6PU~|z$MS^xe9BdsN=lI38)Q+kbrp!ioSY@#&*_b#+JQagsEXfLy2rGvKfknJXS)yYIeFWFn%O!#3^Oq0j?B?oTqND{?&2KkP=UTsE(@z3C)ucC=5q+J{U%sU`lB zpG!MDyWJg4SsiS@M6#rHlxT_8zLb_YIP+od^{s|w4}iU3YnF^=Xif)}8=(Hom`wHS z5AM`%n_ro6bh(7}IRLz;f7*JXUGv|9k05l-Fu%<_tYKjBnr3UI_(c_5zo^YOVBPm; zcNQ1`jD9kZJ_CaQB;QQjus+o`3OiNIM@jQ!= z$gzYIc$VW-iV#u7xWd`=tM(ZT%pVENdp8@a17P8Mq}KYv*Ob7G$^LWIL#d3e(Du5F zLG_rM8%7k5fD#-t+?7^o@6C&)QTFV!7&gld$#-nU4$Qt diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-keyed-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-keyed-restore/savepoint/_metadata index c59dd5545a963112b75cff4ca18dd45eca44217f..8675957fc440c1a6b8683ac9ccb425e0b72e3a6b 100644 GIT binary patch delta 371 zcmbWz&niVh7y$4&_hLsO*Po(9ihP~<^L=w>I+U%2jfIU;Gv~}H3*6AzdjVv!u%o;H z^8_|l%2wImjW@7T-he0g)$jEp`#Qb8%VNBm+uW1Z8bmI+(1MeaR8W5A+R#5aJU#~! zCddl7Ls4j>9Vwr4tScF81B$c&Y{-K#N`WgHwB8tDtxoG%+w07Hq^If6PQ34pb~E0d-6=G{oQf9NeA<}HpWV^;o zix)3F7#ux#kru?07g+?syC*N=RjCILh2q7dAaQqXsd&?YVc?thz4yMyx8HnsA9kL? zN1`)$afvX6N}5I_rZP!UsLBd0?pcj`)o~FvFfQvz(y&e>OTx@n3*3}#Vj2=6+#rY& ztRb#39Z?GtQs8N2ZMJK!n=J;JSn~ z+h$B-rj1n-fZ-MZAnD`Dv6z2jXtLk`e*O{^D7Fhn#ZG@NXL6V4OZB@m!5bhCkN6P( zEKZJC{_?3=IGCFU3HM&ZS_2y((QY(ckmO~zT6bE4r_PCI$+7$~xDo#LxuEyfRO#~E zRB5x@?Y6>NHozaZJj-!c+-l8gH9;Ye9%s|lO3`r_1KtBw7z3akb{vr8 zAY#jix%7275t`qO%-?jIRj2Zo|8^tfITLk^drgqK!#$o~AK^!fkpAd|`l f4Hg2~suC=PLI~1mSw^&sIaXytFvb3w@+1BWmp9fu diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/plan/deduplicate-keep-first-ttl-expired-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/plan/deduplicate-keep-first-ttl-expired-restore.json index eba9144495ddfb..9aa08778f3b64e 100644 --- a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/plan/deduplicate-keep-first-ttl-expired-restore.json +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/plan/deduplicate-keep-first-ttl-expired-restore.json @@ -1,7 +1,7 @@ { "flinkVersion" : "2.4", "nodes" : [ { - "id" : 1, + "id" : 10, "type" : "stream-exec-table-source-scan_2", "scanTableSource" : { "table" : { @@ -22,7 +22,7 @@ "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events]], fields=[user_name, action])" }, { - "id" : 2, + "id" : 11, "type" : "stream-exec-exchange_1", "inputProperties" : [ { "requiredDistribution" : { @@ -35,7 +35,7 @@ "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", "description" : "Exchange(distribution=[hash[user_name]])" }, { - "id" : 3, + "id" : 12, "type" : "stream-exec-process-table-function_1", "inputProperties" : [ { "requiredDistribution" : { @@ -81,7 +81,7 @@ "outputChangelogMode" : [ "INSERT" ], "inputUpsertKeys" : [ [ ] ] }, { - "id" : 4, + "id" : 13, "type" : "stream-exec-sink_2", "configuration" : { "table.exec.sink.keyed-shuffle" : "AUTO", @@ -107,6 +107,7 @@ } }, "inputChangelogMode" : [ "INSERT" ], + "upsertMaterializeStrategy" : "ADAPTIVE", "inputProperties" : [ { "requiredDistribution" : { "type" : "UNKNOWN" @@ -118,22 +119,22 @@ "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_name, action])" } ], "edges" : [ { - "source" : 1, - "target" : 2, + "source" : 10, + "target" : 11, "shuffle" : { "type" : "FORWARD" }, "shuffleMode" : "PIPELINED" }, { - "source" : 2, - "target" : 3, + "source" : 11, + "target" : 12, "shuffle" : { "type" : "FORWARD" }, "shuffleMode" : "PIPELINED" }, { - "source" : 3, - "target" : 4, + "source" : 12, + "target" : 13, "shuffle" : { "type" : "FORWARD" }, diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/savepoint/_metadata index 750f297d9fbe871d084a15de8a1aafcdef591d75..1368d5d796a79568b18a4735c6ed81d9c3539547 100644 GIT binary patch delta 605 zcmbVJziU%b6uvi=iXcU>CTT$%sn|iTxxarT@yDctOQG393Gdx^FA2(ve>EAEX^Zi>u`bma1Qit;+XO{<1tb1HqGg(rk5w#F`UgxWR%6EQz7@EZT6l z2mN$}gpt-KNJ>JPbJ%h$>x>GFV=YjW5QR97DT+iakYm`I1bbz?0AOJT05A)R6X%yp z#f{TjCyR@yfcff*&#%|F^Vf~#`)A+n^x$+Fcn48g8;pn6HEQF`4ZCi`rK7Av@H#j~ za5wum`>30jAKiIyuJ>zq>D%7j-LMG*CJ@wY#59YU(^hFsDVN%LuZT~KH$jsMwBrZ# zPyoo=%?t3bCL~HZqm)DuODN(LQ)ILkh%>AcNrIDT2 z7IKHMb#Roh?tYVg+S_h?`0;#t=EbY_`!BuCD=qkJ^WNf-H;4aNoOp>DkC7*qAg&^d mwABh>qCB_Oc|kk?^Phvh@G%ab&gXwlE#F|9DZ-zR^7BL|7EW3`j^1Q-T6PGc2LULFp(4@7F*}d%r09xyUomQMxdTL zw1<>ehv*O%5s^^3gh3D#R37RO9fkDJO}d2C$?VN$Ds-xWH}B1Rzu))1?>FCX(acW4 z>6+Hk!Gv+KBVdttgc}APF6t~hs-ArRc-yzWcxU-R`g3I;wgO%7K7JZF?0yHq44!m% zofjmYUCX%=PR1cU*Rky4m1&!*xX+yOWNh89}(+YI?CvvlJ*)=m=TCk;M_@A}#ti%f% zt5F)_94#S4$*Lq$Dox8gjaZT9h!+0~LdcVYWzDYKvBLRsJkq!VoA9gR>N-uamG(2q z%g4#Q%^HC5lx2dWxcf*84mK~~kqSL`J=hCts{4T3zLL@}feGLqPo-?&QSzE_gEw5_ z>=wfbXDY$nba{Cd3GnI}-LRu}VoWEPK?hUpNyNj3Jw$-E?u0IK=k9a@+!H<-?jOqU zR$9(k1-?q3hOG|o$vp2t+e{eozo!C0F;P&7 zj8e$lQ!Bh2}@IM=fW;Dbj#8P5NVknMf6iOBtkz#m}RX9eLG|lnSIvJ#a?$#f| PN&g4$PV3}&!By}JvyvmF diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java index d0326dcfbfaed8..d81fdcaa8558f4 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java @@ -39,24 +39,22 @@ import javax.annotation.Nullable; -import java.util.stream.IntStream; - /** Implementation of the {@code DEDUPLICATE_KEEP_FIRST} process table function. */ @Internal public class DeduplicateKeepFirst extends BuiltInProcessTableFunction { private static final long serialVersionUID = 1L; - private final int[] outputIndices; + private static final int CANDIDATE_PAYLOAD = 0; - private final int eventTimeIndex; + private static final int CANDIDATE_EVENT_TIME = 1; + + private final int[] outputIndices; private final RowType inputRowType; private transient ProjectedRowData projectedOutput; - private transient ProjectedRowData emitProjection; - private transient RowData.FieldGetter[] payloadGetters; private transient TypeSerializer candidateSerializer; @@ -69,7 +67,6 @@ public DeduplicateKeepFirst(final SpecializedContext context) { .getTableSemantics(DeduplicateKeepFirstTypeStrategy.ARG_INPUT) .orElseThrow(() -> new IllegalStateException("Table argument expected.")); this.outputIndices = ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics); - this.eventTimeIndex = outputIndices.length; this.inputRowType = (RowType) tableSemantics.dataType().getLogicalType(); } @@ -77,20 +74,14 @@ public DeduplicateKeepFirst(final SpecializedContext context) { public void open(final FunctionContext context) throws Exception { super.open(context); projectedOutput = ProjectedRowData.from(outputIndices); - emitProjection = ProjectedRowData.from(IntStream.range(0, eventTimeIndex).toArray()); payloadGetters = new RowData.FieldGetter[outputIndices.length]; + final LogicalType[] payloadTypes = new LogicalType[outputIndices.length]; for (int i = 0; i < outputIndices.length; i++) { - payloadGetters[i] = - RowData.createFieldGetter( - inputRowType.getTypeAt(outputIndices[i]), outputIndices[i]); - } - - final LogicalType[] candidateTypes = new LogicalType[eventTimeIndex + 1]; - for (int i = 0; i < eventTimeIndex; i++) { - candidateTypes[i] = inputRowType.getTypeAt(outputIndices[i]); + payloadTypes[i] = inputRowType.getTypeAt(outputIndices[i]); + payloadGetters[i] = RowData.createFieldGetter(payloadTypes[i], outputIndices[i]); } - candidateTypes[eventTimeIndex] = new BigIntType(); - candidateSerializer = InternalSerializers.create(RowType.of(candidateTypes)); + candidateSerializer = + InternalSerializers.create(RowType.of(RowType.of(payloadTypes), new BigIntType())); } public void eval( @@ -120,7 +111,7 @@ public void eval( if (seen.getValue() == null) { // not yet emitted → buffer the earliest candidate final RowData currLowest = candidate.getValue(); final Long currentLowestEventTime = - currLowest == null ? null : currLowest.getLong(eventTimeIndex); + currLowest == null ? null : currLowest.getLong(CANDIDATE_EVENT_TIME); if (currentLowestEventTime == null || rowtime < currentLowestEventTime) { if (currentLowestEventTime != null) { context.clearTimer(currentLowestEventTime); @@ -159,18 +150,16 @@ public void onTimer( if (currLowest == null) { return; } - emitProjection.replaceRow(currLowest); - collect(emitProjection); + collect(currLowest.getRow(CANDIDATE_PAYLOAD, outputIndices.length)); candidate.clear(); seen.setValue(true); } private GenericRowData materializeCandidate(final RowData input, final long rowtime) { - final GenericRowData row = new GenericRowData(eventTimeIndex + 1); - for (int i = 0; i < eventTimeIndex; i++) { - row.setField(i, payloadGetters[i].getFieldOrNull(input)); + final GenericRowData payload = new GenericRowData(outputIndices.length); + for (int i = 0; i < outputIndices.length; i++) { + payload.setField(i, payloadGetters[i].getFieldOrNull(input)); } - row.setField(eventTimeIndex, rowtime); - return row; + return GenericRowData.of(payload, rowtime); } } From 24d8f345395fdef4e5d45b03d986a2d48ebedc01 Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Mon, 19 Oct 2026 15:59:07 +0100 Subject: [PATCH 4/5] [FLINK-40896][table] Drop late rows at the watermark, unify dedup candidate state - Drop DEDUPLICATE_KEEP_FIRST rows with ts <= watermark, use one candidate state type in both modes, and derive the event-time mode once in the constructor - Trim the tests (drop Table API, multi-key and TTL-expiry cases, add restore rows) and polish the docs --- .../queries/deduplicate-keep-first.md | 4 +- .../queries/deduplicate-keep-first.md | 4 +- .../DeduplicateKeepFirstTypeStrategy.java | 7 +- .../DeduplicateKeepFirstTypeStrategyTest.java | 31 ++-- .../DeduplicateKeepFirstRestoreTest.java | 4 +- ...duplicateKeepFirstRestoreTestPrograms.java | 65 +------- .../DeduplicateKeepFirstSemanticTests.java | 3 - .../DeduplicateKeepFirstTestPrograms.java | 136 ++--------------- .../savepoint/_metadata | Bin 18262 -> 18455 bytes ...keep-first-global-ttl-expired-restore.json | 143 ------------------ .../savepoint/_metadata | Bin 14647 -> 0 bytes .../savepoint/_metadata | Bin 14699 -> 17446 bytes ...licate-keep-first-ttl-expired-restore.json | 143 ------------------ .../savepoint/_metadata | Bin 14647 -> 0 bytes .../functions/ptf/DeduplicateKeepFirst.java | 34 ++--- 15 files changed, 57 insertions(+), 517 deletions(-) delete mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/plan/deduplicate-keep-first-global-ttl-expired-restore.json delete mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/savepoint/_metadata delete mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/plan/deduplicate-keep-first-ttl-expired-restore.json delete mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/savepoint/_metadata diff --git a/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md b/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md index 10ab2f787509e6..4e307552e4ff21 100644 --- a/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md +++ b/docs/content.zh/docs/sql/reference/queries/deduplicate-keep-first.md @@ -60,7 +60,7 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( | Parameter | Required | Description | |:----------|:---------|:------------| | `input` | Yes | The input table (append-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | -| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. If omitted, falls back to `table.exec.state.ttl`; at its default of 0, state is retained indefinitely, but a retention can still be introduced later. `INTERVAL '0'` also retains state indefinitely, but excludes the TTL from the state layout, so a retention cannot be introduced later without losing the state. See [State TTL]({{< ref "docs/dev/table/functions/ptfs" >}}#state-ttl). | +| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. If omitted, falls back to `table.exec.state.ttl`; at its default of 0, state is retained indefinitely, but a retention can still be introduced later. `state_ttl => INTERVAL '0'` also retains state indefinitely, but excludes the TTL from the state layout, so a retention cannot be introduced later without losing the state. See [State TTL]({{< ref "docs/dev/table/functions/ptfs" >}}#state-ttl). | | `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful when a TTL applies, whether set through `state_ttl` or inherited from `table.exec.state.ttl`. | | `on_time` | No | A `DESCRIPTOR` naming a single watermarked column. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires append-only input; combining `on_time` with an updating input is rejected at planning time. | | `uid` | No | A unique identifier for the function's state in checkpoints and savepoints. Defaults to the function name, so it is required when `DEDUPLICATE_KEEP_FIRST` is called more than once in the same job. See [Query Evolution with UIDs]({{< ref "docs/dev/table/functions/ptfs" >}}#query-evolution-with-uids). | @@ -149,7 +149,7 @@ SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST( -- +I[user_name:'Vas', action:'click'] ``` -With `on_time`, the record with the smallest event time per key is kept (here `click` at `ts 1.000`, not `login` which arrived first) and emitted once the watermark passes that timestamp. +With `on_time`, the record with the smallest event time per key is kept (here `click` at `ts 1.000`, not `login` which arrived first) and emitted once the watermark passes that timestamp. Late events (i.e. events with timestamp smaller or equal to the received watermark) are not taken into consideration. #### Updating input diff --git a/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md b/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md index 10ab2f787509e6..4e307552e4ff21 100644 --- a/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md +++ b/docs/content/docs/sql/reference/queries/deduplicate-keep-first.md @@ -60,7 +60,7 @@ SELECT * FROM DEDUPLICATE_KEEP_FIRST( | Parameter | Required | Description | |:----------|:---------|:------------| | `input` | Yes | The input table (append-only or updating). Use `PARTITION BY` to deduplicate per key; all rows for a key are routed to the same parallel instance. Without `PARTITION BY`, the whole input is a single group processed at parallelism 1, so only the first row of the entire stream is emitted and every later row is dropped regardless of its content. | -| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. If omitted, falls back to `table.exec.state.ttl`; at its default of 0, state is retained indefinitely, but a retention can still be introduced later. `INTERVAL '0'` also retains state indefinitely, but excludes the TTL from the state layout, so a retention cannot be introduced later without losing the state. See [State TTL]({{< ref "docs/dev/table/functions/ptfs" >}}#state-ttl). | +| `state_ttl` | No | An `INTERVAL` giving the processing-time retention for the per-key deduplication state. After a key's state expires, a later record for that key starts a new deduplication period and may be emitted again. If omitted, falls back to `table.exec.state.ttl`; at its default of 0, state is retained indefinitely, but a retention can still be introduced later. `state_ttl => INTERVAL '0'` also retains state indefinitely, but excludes the TTL from the state layout, so a retention cannot be introduced later without losing the state. See [State TTL]({{< ref "docs/dev/table/functions/ptfs" >}}#state-ttl). | | `reset_ttl_on_duplicate` | No | Whether a later duplicate refreshes the key's `state_ttl`. Defaults to `TRUE`, so retention tracks the most recent occurrence of a key. Only meaningful when a TTL applies, whether set through `state_ttl` or inherited from `table.exec.state.ttl`. | | `on_time` | No | A `DESCRIPTOR` naming a single watermarked column. When provided, the row with the smallest event time per key is kept and emitted once the watermark passes that timestamp (late rows are dropped). When omitted, the function keeps the first row observed for a key (arrival order). Requires append-only input; combining `on_time` with an updating input is rejected at planning time. | | `uid` | No | A unique identifier for the function's state in checkpoints and savepoints. Defaults to the function name, so it is required when `DEDUPLICATE_KEEP_FIRST` is called more than once in the same job. See [Query Evolution with UIDs]({{< ref "docs/dev/table/functions/ptfs" >}}#query-evolution-with-uids). | @@ -149,7 +149,7 @@ SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST( -- +I[user_name:'Vas', action:'click'] ``` -With `on_time`, the record with the smallest event time per key is kept (here `click` at `ts 1.000`, not `login` which arrived first) and emitted once the watermark passes that timestamp. +With `on_time`, the record with the smallest event time per key is kept (here `click` at `ts 1.000`, not `login` which arrived first) and emitted once the watermark passes that timestamp. Late events (i.e. events with timestamp smaller or equal to the received watermark) are not taken into consideration. #### Updating input diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java index 65802ed66fbe35..74c489dbff8cba 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java @@ -113,11 +113,6 @@ public Optional inferType(final CallContext callContext) { () -> new ValidationException( "First argument must be a table for DEDUPLICATE_KEEP_FIRST.")); - // Only event-time mode buffers a candidate. Without on_time a fixed type keeps - // the unused state independent of the input schema across query evolution. - if (tableSemantics.timeColumn() == -1) { - return Optional.of(ValueView.newValueViewDataType(DataTypes.BOOLEAN())); - } final List inputFields = DataType.getFields(tableSemantics.dataType()); final List payloadFields = Arrays.stream( @@ -166,7 +161,7 @@ private static Optional> validateStateTtl( "The 'state_ttl' argument must not be negative, but was '%s'. " + "Use a positive INTERVAL to bound state retention, INTERVAL '0' to " + "retain state indefinitely without a TTL, or omit the argument to " - + "fall back to 'table.exec.state.ttl'.", + + "fall back to session defaults.", stateTtl.get()); } return Optional.empty(); diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java index 09b5a58cf9d9f9..59ec5195982793 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategyTest.java @@ -59,25 +59,18 @@ void testCandidateStateDisablesTtl() { } @Test - void testCandidateStateWithoutOnTimeIsFixed() { - assertThat(CANDIDATE_STATE_TYPE_STRATEGY.inferType(callContext(-1))) - .contains(ValueView.newValueViewDataType(DataTypes.BOOLEAN())); - } - - @Test - void testCandidateStateWithOnTimeNestsPayload() { - assertThat(CANDIDATE_STATE_TYPE_STRATEGY.inferType(callContext(2))) - .contains( - ValueView.newValueViewDataType( - DataTypes.ROW( - DataTypes.FIELD( - "payload", - DataTypes.ROW( - DataTypes.FIELD( - "action", DataTypes.STRING()), - DataTypes.FIELD( - "ts", DataTypes.TIMESTAMP_LTZ(3)))), - DataTypes.FIELD("event_time", DataTypes.BIGINT())))); + void testCandidateStateNestsPayloadInBothModes() { + final DataType expected = + ValueView.newValueViewDataType( + DataTypes.ROW( + DataTypes.FIELD( + "payload", + DataTypes.ROW( + DataTypes.FIELD("action", DataTypes.STRING()), + DataTypes.FIELD("ts", DataTypes.TIMESTAMP_LTZ(3)))), + DataTypes.FIELD("event_time", DataTypes.BIGINT()))); + assertThat(CANDIDATE_STATE_TYPE_STRATEGY.inferType(callContext(-1))).contains(expected); + assertThat(CANDIDATE_STATE_TYPE_STRATEGY.inferType(callContext(2))).contains(expected); } @Test diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java index d4430c64c6e674..fa9ad95f08559e 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTest.java @@ -41,9 +41,7 @@ public DeduplicateKeepFirstRestoreTest() { public List programs() { return List.of( DeduplicateKeepFirstRestoreTestPrograms.KEYED_RESTORE, - DeduplicateKeepFirstRestoreTestPrograms.EVENT_TIME_PENDING_RESTORE, - DeduplicateKeepFirstRestoreTestPrograms.TTL_EXPIRED_RESTORE, - DeduplicateKeepFirstRestoreTestPrograms.GLOBAL_TTL_EXPIRED_RESTORE); + DeduplicateKeepFirstRestoreTestPrograms.EVENT_TIME_PENDING_RESTORE); } @Override diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java index ad21aad63e1881..f33d91892d0926 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstRestoreTestPrograms.java @@ -18,14 +18,12 @@ package org.apache.flink.table.planner.plan.nodes.exec.stream; -import org.apache.flink.table.api.config.ExecutionConfigOptions; import org.apache.flink.table.test.program.SinkTestStep; import org.apache.flink.table.test.program.SourceTestStep; import org.apache.flink.table.test.program.TableTestProgram; import org.apache.flink.types.Row; import org.apache.flink.types.RowKind; -import java.time.Duration; import java.time.Instant; /** Restore {@link TableTestProgram}s for the built-in DEDUPLICATE_KEEP_FIRST PTF. */ @@ -77,7 +75,9 @@ public class DeduplicateKeepFirstRestoreTestPrograms { SourceTestStep.newBuilder("user_events") .addSchema(TIMED_EVENTS_SCHEMA) .producedBeforeRestore( - Row.of("Bob", "first", Instant.ofEpochMilli(5))) + Row.of("Bob", "first", Instant.ofEpochMilli(5)), + Row.of("Alice", "first", Instant.ofEpochMilli(5)), + Row.of("Carol", "first", Instant.ofEpochMilli(5))) .producedAfterRestore( Row.of("Bob", "later", Instant.ofEpochMilli(100))) .build()) @@ -86,66 +86,13 @@ public class DeduplicateKeepFirstRestoreTestPrograms { .addSchema(USER_EVENTS_SCHEMA) .consumedBeforeRestore(new Row[0]) .consumedAfterRestore( - Row.ofKind(RowKind.INSERT, "Bob", "first")) + Row.ofKind(RowKind.INSERT, "Bob", "first"), + Row.ofKind(RowKind.INSERT, "Alice", "first"), + Row.ofKind(RowKind.INSERT, "Carol", "first")) .build()) .runSql( "INSERT INTO sink SELECT user_name, action FROM DEDUPLICATE_KEEP_FIRST(" + "input => TABLE user_events PARTITION BY user_name, " + "on_time => DESCRIPTOR(ts))") .build(); - - public static final TableTestProgram TTL_EXPIRED_RESTORE = - TableTestProgram.of( - "deduplicate-keep-first-ttl-expired-restore", - "the 'seen' state in the savepoint has outlived state_ttl by the time " - + "the test restores it, so an already-seen key is emitted " - + "again in a new deduplication period") - .setupTableSource( - SourceTestStep.newBuilder("user_events") - .addSchema(USER_EVENTS_SCHEMA) - .producedBeforeRestore( - Row.of("Vas", "login"), Row.of("Vas", "click")) - .producedAfterRestore(Row.of("Vas", "logout")) - .build()) - .setupTableSink( - SinkTestStep.newBuilder("sink") - .addSchema(USER_EVENTS_SCHEMA) - .consumedBeforeRestore( - Row.ofKind(RowKind.INSERT, "Vas", "login")) - .consumedAfterRestore( - Row.ofKind(RowKind.INSERT, "Vas", "logout")) - .build()) - .runSql( - "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" - + "input => TABLE user_events PARTITION BY user_name, " - + "state_ttl => INTERVAL '10' SECOND)") - .build(); - - public static final TableTestProgram GLOBAL_TTL_EXPIRED_RESTORE = - TableTestProgram.of( - "deduplicate-keep-first-global-ttl-expired-restore", - "without state_ttl, the 'seen' state falls back to " - + "table.exec.state.ttl and has expired by the time the test " - + "restores it, so an already-seen key is emitted again") - .setupConfig( - ExecutionConfigOptions.IDLE_STATE_RETENTION, Duration.ofSeconds(10)) - .setupTableSource( - SourceTestStep.newBuilder("user_events") - .addSchema(USER_EVENTS_SCHEMA) - .producedBeforeRestore( - Row.of("Vas", "login"), Row.of("Vas", "click")) - .producedAfterRestore(Row.of("Vas", "logout")) - .build()) - .setupTableSink( - SinkTestStep.newBuilder("sink") - .addSchema(USER_EVENTS_SCHEMA) - .consumedBeforeRestore( - Row.ofKind(RowKind.INSERT, "Vas", "login")) - .consumedAfterRestore( - Row.ofKind(RowKind.INSERT, "Vas", "logout")) - .build()) - .runSql( - "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" - + "input => TABLE user_events PARTITION BY user_name)") - .build(); } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java index e5d0438cfe1771..8d21c8355b1d66 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstSemanticTests.java @@ -40,8 +40,6 @@ protected void applyDefaultEnvironmentOptions(TableConfig config) { public List programs() { return List.of( DeduplicateKeepFirstTestPrograms.KEYED, - DeduplicateKeepFirstTestPrograms.TABLE_API, - DeduplicateKeepFirstTestPrograms.TABLE_API_WITH_ARGS, DeduplicateKeepFirstTestPrograms.MULTI_KEY, DeduplicateKeepFirstTestPrograms.MULTI_COLUMN_KEY, DeduplicateKeepFirstTestPrograms.NULL_KEY, @@ -51,7 +49,6 @@ public List programs() { DeduplicateKeepFirstTestPrograms.WHOLE_ROW_KEY, DeduplicateKeepFirstTestPrograms.EVENT_TIME, DeduplicateKeepFirstTestPrograms.EVENT_TIME_CANDIDATE_FIELD_NAMES, - DeduplicateKeepFirstTestPrograms.EVENT_TIME_MULTI_KEY, DeduplicateKeepFirstTestPrograms.EVENT_TIME_LATE_DROPPED, DeduplicateKeepFirstTestPrograms.UPDATING_INPUT, DeduplicateKeepFirstTestPrograms.UNSEEN_RETRACTION_IGNORED, diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java index 2b0bf1ab571c1f..2ecebb7e7984dc 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeduplicateKeepFirstTestPrograms.java @@ -25,12 +25,8 @@ import org.apache.flink.types.Row; import org.apache.flink.types.RowKind; -import java.time.Duration; import java.time.Instant; -import static org.apache.flink.table.api.Expressions.$; -import static org.apache.flink.table.api.Expressions.lit; - /** {@link TableTestProgram} definitions for testing the built-in DEDUPLICATE_KEEP_FIRST PTF. */ public class DeduplicateKeepFirstTestPrograms { @@ -62,57 +58,6 @@ public class DeduplicateKeepFirstTestPrograms { + "input => TABLE user_events PARTITION BY user_name)") .build(); - public static final TableTestProgram TABLE_API = - TableTestProgram.of( - "deduplicate-keep-first-table-api", - "the generic Table API process() call invokes the PTF, matching " - + "the SQL form") - .setupTableSource( - SourceTestStep.newBuilder("user_events") - .addSchema(USER_EVENTS_SCHEMA) - .producedValues(Row.of("Vas", "login"), Row.of("Vas", "click")) - .build()) - .setupTableSink( - SinkTestStep.newBuilder("sink") - .addSchema(USER_EVENTS_SCHEMA) - .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) - .build()) - .runTableApi( - env -> - env.from("user_events") - .partitionBy($("user_name")) - .process("DEDUPLICATE_KEEP_FIRST"), - "sink") - .build(); - - public static final TableTestProgram TABLE_API_WITH_ARGS = - TableTestProgram.of( - "deduplicate-keep-first-table-api-with-args", - "the Table API process() call passes the optional state_ttl and " - + "reset_ttl_on_duplicate as named arguments") - .setupTableSource( - SourceTestStep.newBuilder("user_events") - .addSchema(USER_EVENTS_SCHEMA) - .producedValues(Row.of("Vas", "login"), Row.of("Vas", "click")) - .build()) - .setupTableSink( - SinkTestStep.newBuilder("sink") - .addSchema(USER_EVENTS_SCHEMA) - .consumedValues(Row.ofKind(RowKind.INSERT, "Vas", "login")) - .build()) - .runTableApi( - env -> - env.from("user_events") - .partitionBy($("user_name")) - .process( - "DEDUPLICATE_KEEP_FIRST", - lit(Duration.ofSeconds(5)) - .asArgument("state_ttl"), - lit(false) - .asArgument("reset_ttl_on_duplicate")), - "sink") - .build(); - public static final TableTestProgram MULTI_KEY = TableTestProgram.of( "deduplicate-keep-first-multi-key", @@ -155,15 +100,16 @@ public class DeduplicateKeepFirstTestPrograms { .build()) .setupTableSink( SinkTestStep.newBuilder("sink") - .addSchema("region STRING", "user_name STRING", "action STRING") + .addSchema("user_name STRING", "region STRING", "action STRING") .consumedValues( - Row.ofKind(RowKind.INSERT, "EU", "Vas", "login"), - Row.ofKind(RowKind.INSERT, "US", "Vas", "signup"), - Row.ofKind(RowKind.INSERT, "EU", "Alice", "view")) + Row.ofKind(RowKind.INSERT, "Vas", "EU", "login"), + Row.ofKind(RowKind.INSERT, "Vas", "US", "signup"), + Row.ofKind(RowKind.INSERT, "Alice", "EU", "view")) .build()) .runSql( "INSERT INTO sink SELECT * FROM DEDUPLICATE_KEEP_FIRST(" - + "input => TABLE user_events PARTITION BY (region, user_name))") + + "input => TABLE user_events PARTITION BY " + + "(user_name, region))") .build(); public static final TableTestProgram NULL_KEY = @@ -345,69 +291,13 @@ public class DeduplicateKeepFirstTestPrograms { + "on_time => DESCRIPTOR(ts))") .build(); - public static final TableTestProgram EVENT_TIME_MULTI_KEY = - TableTestProgram.of( - "deduplicate-keep-first-event-time-multi-key", - "each key emits its earliest row with that row's event time as " - + "rowtime; replacing one key's candidate leaves other keys' " - + "timers intact, and on equal event times the first arrival " - + "wins") - .setupTableSource( - SourceTestStep.newBuilder("user_events") - .addSchema(TIMED_EVENTS_SCHEMA) - .producedValues( - Row.of("Alice", "Hi", Instant.ofEpochMilli(1)), - Row.of("Alice", "Hello", Instant.ofEpochMilli(3)), - Row.of("Alice", "Hello world", Instant.ofEpochMilli(2)), - Row.of("Bob", "I am fine.", Instant.ofEpochMilli(3)), - Row.of("Bob", "Comment#1", Instant.ofEpochMilli(6)), - Row.of("Carol", "Comment#2", Instant.ofEpochMilli(3)), - Row.of("Carol", "Comment#3", Instant.ofEpochMilli(2)), - Row.of("Dave", "Comment#4", Instant.ofEpochMilli(4)), - Row.of("Dave", "Comment#5", Instant.ofEpochMilli(4))) - .build()) - .setupTableSink( - SinkTestStep.newBuilder("sink") - .addSchema( - "user_name STRING", - "action STRING", - "rowtime TIMESTAMP_LTZ(3)") - .consumedValues( - Row.ofKind( - RowKind.INSERT, - "Alice", - "Hi", - Instant.ofEpochMilli(1)), - Row.ofKind( - RowKind.INSERT, - "Bob", - "I am fine.", - Instant.ofEpochMilli(3)), - Row.ofKind( - RowKind.INSERT, - "Carol", - "Comment#3", - Instant.ofEpochMilli(2)), - Row.ofKind( - RowKind.INSERT, - "Dave", - "Comment#4", - Instant.ofEpochMilli(4))) - .build()) - .runSql( - "INSERT INTO sink SELECT user_name, action, rowtime FROM " - + "DEDUPLICATE_KEEP_FIRST(" - + "input => TABLE user_events PARTITION BY user_name, " - + "on_time => DESCRIPTOR(ts))") - .build(); - public static final TableTestProgram EVENT_TIME_LATE_DROPPED = TableTestProgram.of( "deduplicate-keep-first-event-time-late-dropped", - "a row below the watermark is dropped, even though its event time is " - + "earlier than the already-finalized candidate; a late first " - + "row for an unseen key is dropped without marking the key " - + "as seen") + "a row at or below the watermark is dropped, even though its event " + + "time is earlier than the already-finalized candidate; a late " + + "first row for an unseen key is dropped without marking the " + + "key as seen") .setupTableSource( SourceTestStep.newBuilder("user_events") .addSchema(TIMED_EVENTS_SCHEMA) @@ -416,6 +306,11 @@ public class DeduplicateKeepFirstTestPrograms { Row.of("Bob", "advance", Instant.ofEpochMilli(20000)), Row.of("Bob", "late", Instant.ofEpochMilli(500)), Row.of("Carol", "late", Instant.ofEpochMilli(500)), + Row.of( + "Dave", + "at-watermark", + Instant.ofEpochMilli(10000)), + Row.of("Dave", "on-time", Instant.ofEpochMilli(15000)), Row.of("Carol", "on-time", Instant.ofEpochMilli(30000))) .build()) .setupTableSink( @@ -423,6 +318,7 @@ public class DeduplicateKeepFirstTestPrograms { .addSchema("user_name STRING", "action STRING") .consumedValues( Row.ofKind(RowKind.INSERT, "Bob", "early"), + Row.ofKind(RowKind.INSERT, "Dave", "on-time"), Row.ofKind(RowKind.INSERT, "Carol", "on-time")) .build()) .runSql( diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-event-time-pending-restore/savepoint/_metadata index 1ea051ac1b999a98257eda15b18583e779340b25..37b4d90141272f314896284566e92773516702dd 100644 GIT binary patch delta 562 zcmccC$2ffg;|5N~%^#V%7$;7MW77CDvC)r7GkIg91>@vu7W+6$6LS+o3sVzQGjmIG zOEUul3q$?l#In?a{LH)(-4qjZi&PWcRD%?=FbHh~gBnwLuO9lqbQw$6YY?}mBm^Rz6=c?8JXJcR}_Dc*Xh-8Rl zU^p9OXH=)azyeewH?e_%l~I5d$P{4Z(g0G@&;|5N~%?gYc87EE<-^|I_#W=Z|#XjED)WFEh+{Dtz(#Y7z($dV-++4pn zu`IP9KQpgH*C5RxDa~Bhz|hphz{JeJ(%3RpKiSC4(7+(cP}e9q%|zG4*u+BD!ob2% zH_?aLn}=D|9^(b_w}}s;akfj zQzO$P!$e)fL_-T*lVlT1-6SK+BwZ6DW7A|4^Hk%M6rkHhW&&*o`u2", - "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events]], fields=[user_name, action])" - }, { - "id" : 15, - "type" : "stream-exec-exchange_1", - "inputProperties" : [ { - "requiredDistribution" : { - "type" : "HASH", - "keys" : [ 0 ] - }, - "damBehavior" : "PIPELINED", - "priority" : 0 - } ], - "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", - "description" : "Exchange(distribution=[hash[user_name]])" - }, { - "id" : 16, - "type" : "stream-exec-process-table-function_1", - "inputProperties" : [ { - "requiredDistribution" : { - "type" : "UNKNOWN" - }, - "damBehavior" : "PIPELINED", - "priority" : 0 - } ], - "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", - "description" : "ProcessTableFunction(invocation=[DEDUPLICATE_KEEP_FIRST(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE_KEEP_FIRST], select=[user_name,action], rowType=[RecordType(VARCHAR(2147483647) user_name, VARCHAR(2147483647) action)])", - "uid" : "DEDUPLICATE_KEEP_FIRST", - "functionCall" : { - "kind" : "CALL", - "internalName" : "$DEDUPLICATE_KEEP_FIRST$1", - "operands" : [ { - "kind" : "TABLE_ARG_CALL", - "inputIndex" : 0, - "partitionKeys" : [ 0 ], - "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "INTERVAL SECOND(6)" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "BOOLEAN" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "DESCRIPTOR" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "VARCHAR(2147483647)" - } ], - "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" - }, - "inputChangelogModes" : [ [ "INSERT" ] ], - "outputChangelogMode" : [ "INSERT" ], - "inputUpsertKeys" : [ [ ] ] - }, { - "id" : 17, - "type" : "stream-exec-sink_2", - "configuration" : { - "table.exec.sink.keyed-shuffle" : "AUTO", - "table.exec.sink.not-null-enforcer" : "ERROR", - "table.exec.sink.rowtime-inserter" : "ENABLED", - "table.exec.sink.type-length-enforcer" : "IGNORE", - "table.exec.sink.upsert-materialize" : "AUTO" - }, - "dynamicTableSink" : { - "table" : { - "identifier" : "`default_catalog`.`default_database`.`sink`", - "resolvedTable" : { - "schema" : { - "columns" : [ { - "name" : "user_name", - "dataType" : "VARCHAR(2147483647)" - }, { - "name" : "action", - "dataType" : "VARCHAR(2147483647)" - } ] - } - } - } - }, - "inputChangelogMode" : [ "INSERT" ], - "inputProperties" : [ { - "requiredDistribution" : { - "type" : "UNKNOWN" - }, - "damBehavior" : "PIPELINED", - "priority" : 0 - } ], - "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", - "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_name, action])" - } ], - "edges" : [ { - "source" : 14, - "target" : 15, - "shuffle" : { - "type" : "FORWARD" - }, - "shuffleMode" : "PIPELINED" - }, { - "source" : 15, - "target" : 16, - "shuffle" : { - "type" : "FORWARD" - }, - "shuffleMode" : "PIPELINED" - }, { - "source" : 16, - "target" : 17, - "shuffle" : { - "type" : "FORWARD" - }, - "shuffleMode" : "PIPELINED" - } ] -} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-global-ttl-expired-restore/savepoint/_metadata deleted file mode 100644 index 416039a2736d7d711388afb1645e3731d0ea7fd0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14647 zcmeHOZ;Ts96(9R9f37sSOM8S|fa)r)k)n2Yz4rQSCrIGC^C_n|cb4tDrgsWvygPfh z$$Ho9t{q>5!c`T5PpyOmQbmD6`voaTAXJI3NGSaPeBfWj2P8fqQ3(o)1Oh1(g*W3} zdwpvs7pRX@yDRO#nVmOp-n^N4@AqbxKD)^wgbu?=k4XE5M$oB(Wy-|1%ebnO#fBj} znrUo^vsci`h1|kwA-^=AE$1p1bGbrgaj94;zoUNm-N{$pdg=D%Kb(54cJFI0wH->) zBTJvr5q3pr$GwNQ9toWKZO?1?5=!8GDuD?4imK`4oY2IUpqjcuEL(_G1Z%6Q#;(_{ zih4qHDpo34k*kIhmtt^i)`hZgwP9!uC&rV>JmF~U_7a)uMJlpty{R*lISgr-J7-fTjQqOxgi z^01;`8K^? zI?;9Xh-NQ0be*;?kB-v1i>B!~Xf_Y*32hMwOauZGfxy@kT=oPXSz7LVs~)*&A!$!d z+f~+W9(oKRXw~tnxQTfk8=HL2Xw>|fbq5_;%NFM^WQ$+8ckdpWeD>c|tn6GLP;;>3 zN{*!&o4fk1U!t%3Mj|Mx#$Y9z0#zfHfpyNnHKK4-DJ+|-;kwIc1WjDbJ(pXkl*)9u zEI(Jsp^;130%S+g#9B7LnyZv@#ieY1X+2kDsh+ZNgu|^{&SG z8AQk4W;*5*N)!!QD5&rdoaaqrv&;HS?`uy>CYQ69a-~9cJ|L;JKz&~K+75OIzh-L6 zGW3fL1DN)8tH%L=0;)Z4}oC`zSa6RW4Uq0{J$K@aHdJ$9+>SOnLA9?UKp zZ`O&09n-S;GF6~1t352WyENLPEg*8<5!mU1U2jdKhrZvNRt>&I6gW?0$U-& z1^fUn;KLptPxp+E-HS-W@EZ-Ve1ywLXy+b8J9n6N)|s9m;#)$((ZM{;a1lA^Uf?iC z$EZz+!CVXGTm)RnL(JWP4+c4t7arguIgZYk9lb>Tr9~`*53)T&-#ajd->7t*%3cp- zI8I+ssDpr)T&wMYcUhhZCiQ8B2fsWWkWjm(_Q1P#~0{j3~Y>VKl$T7 zUylHo-Z_q!Anb8&+J@_fn7V?Fh^Y#F`NyrV-aWy6{;l7?{mGwv>UY+6p#X*6cW6N` z-W#D9G?)wYd(I+`1r`R}?S}5KYTfT9Uq+9*p9Her*!L(=T9V^cSr$1t6_0Tesj6I6 zN=uw55h2nD&1GO@k=W;MUQ`MbPw0Ay>?nSBh=~U}33LE-uZjmX}tRE0
    11p8r+_?20@(uF^a&3oM+8wuM8( zWI|rEAvR>818zE<4qA3#^{^lxbPhd6qG%KhHP+U-Z5m{7(73Om5STf?fyUkZf@y1X z|Ad1c-|*RcE7hw+cEI2py`~w8gQojfdbC~vLWvjQ3;6PQ-6iE!I_0Z055tTo7qGoW zt7oR&Gr^YhaVvp-uUQBEc$o2Se-7P5k%Jdq+!rHnq`2}dn8#?_&rlP4D3z>pmzdQbv_{KXgPOwohszZt=yJsFupi6ja4(iZ{lx^(D3l64YBAJjYdYVv&k6RlYAJRpkeGqI-;4{ z!;BV1BcS*Sv1LoEx0U)apH8{CS>B63`@xYr|7!mF>$mU3-u$C-8Ol+Mbq8S2G?C@17)z-!wVIk$ zq)b(wRbz>moFcQZX+I60J?efAH+ImX^kArQ1f6KtUnCfQ3}6B3cJ6-vH6XlCQD9GWhwAW zbu^)9wu~)h9vdR6+Cg!G62LeSws+kS>j%VQ1X6Kp~YYQ(yxxb zetGgo3|4ytc&uTk%}O$e2Fs>J1kxfhfRUylbOw(-A>3dM9e`K zf>rfvwV&@9@J9laKo$^uPXvTO`Q|%(^L_aC)M+n2eT>a#msghI|HC|NpKyD4x5?kuOv5k$NW238gz9)G$T$LW1oiu7XvK% z>zLOPBBpB>oeCx4lk{mQDR;tU*4OQ!O^;5VyErgTmL_Y&J2fR{k9 zN(u{BtqdZ2MH35iS*v_|FxUrav$=$B|74zr!pv*E+NE{J3p z0Keec1wLa}+!4Co`Gd~%Y<2BwUIvO^9|@&{5oR8Jg-&{Zk)}%E`dg tv;HU4kYD`wg`34#5Jgi%nuY}=n-WHbiA^MCbwk0XBw#_R&nrLMegj*p68r!F delta 366 zcmbWzyGjE=6adiKO|VlCF+M;+2o{pd&g0(M8IZ-+!p6o@=I#S*G^peotW=oF&O+?t zlvaL#*b4S(tZl3WKOtXm;5^@C-`znWk%Bd3+o!_A>Dg zp3VV!)0RXwfNu-}y2b%>iBJ<7V$ak{7Gh&n2(gfscZYd8+M2E;2`P)?y)GUWv-nyq x^^ZSZw&Qy>?C-qIZsSk&H2(n`6C|UDMs*&L6he@V1>IU?wun@RG&=b0{Q", - "description" : "TableSourceScan(table=[[default_catalog, default_database, user_events]], fields=[user_name, action])" - }, { - "id" : 11, - "type" : "stream-exec-exchange_1", - "inputProperties" : [ { - "requiredDistribution" : { - "type" : "HASH", - "keys" : [ 0 ] - }, - "damBehavior" : "PIPELINED", - "priority" : 0 - } ], - "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", - "description" : "Exchange(distribution=[hash[user_name]])" - }, { - "id" : 12, - "type" : "stream-exec-process-table-function_1", - "inputProperties" : [ { - "requiredDistribution" : { - "type" : "UNKNOWN" - }, - "damBehavior" : "PIPELINED", - "priority" : 0 - } ], - "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", - "description" : "ProcessTableFunction(invocation=[DEDUPLICATE_KEEP_FIRST(TABLE(#0) PARTITION BY($0), 10000:INTERVAL SECOND, DEFAULT(), DEFAULT(), DEFAULT())], uid=[DEDUPLICATE_KEEP_FIRST], select=[user_name,action], rowType=[RecordType(VARCHAR(2147483647) user_name, VARCHAR(2147483647) action)])", - "uid" : "DEDUPLICATE_KEEP_FIRST", - "functionCall" : { - "kind" : "CALL", - "internalName" : "$DEDUPLICATE_KEEP_FIRST$1", - "operands" : [ { - "kind" : "TABLE_ARG_CALL", - "inputIndex" : 0, - "partitionKeys" : [ 0 ], - "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" - }, { - "kind" : "LITERAL", - "value" : "10000", - "type" : "INTERVAL SECOND(6) NOT NULL" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "BOOLEAN" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "DESCRIPTOR" - }, { - "kind" : "CALL", - "syntax" : "SPECIAL", - "internalName" : "$DEFAULT$1", - "type" : "VARCHAR(2147483647)" - } ], - "type" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)> NOT NULL" - }, - "inputChangelogModes" : [ [ "INSERT" ] ], - "outputChangelogMode" : [ "INSERT" ], - "inputUpsertKeys" : [ [ ] ] - }, { - "id" : 13, - "type" : "stream-exec-sink_2", - "configuration" : { - "table.exec.sink.keyed-shuffle" : "AUTO", - "table.exec.sink.not-null-enforcer" : "ERROR", - "table.exec.sink.rowtime-inserter" : "ENABLED", - "table.exec.sink.type-length-enforcer" : "IGNORE", - "table.exec.sink.upsert-materialize" : "AUTO" - }, - "dynamicTableSink" : { - "table" : { - "identifier" : "`default_catalog`.`default_database`.`sink`", - "resolvedTable" : { - "schema" : { - "columns" : [ { - "name" : "user_name", - "dataType" : "VARCHAR(2147483647)" - }, { - "name" : "action", - "dataType" : "VARCHAR(2147483647)" - } ] - } - } - } - }, - "inputChangelogMode" : [ "INSERT" ], - "upsertMaterializeStrategy" : "ADAPTIVE", - "inputProperties" : [ { - "requiredDistribution" : { - "type" : "UNKNOWN" - }, - "damBehavior" : "PIPELINED", - "priority" : 0 - } ], - "outputType" : "ROW<`user_name` VARCHAR(2147483647), `action` VARCHAR(2147483647)>", - "description" : "Sink(table=[default_catalog.default_database.sink], fields=[user_name, action])" - } ], - "edges" : [ { - "source" : 10, - "target" : 11, - "shuffle" : { - "type" : "FORWARD" - }, - "shuffleMode" : "PIPELINED" - }, { - "source" : 11, - "target" : 12, - "shuffle" : { - "type" : "FORWARD" - }, - "shuffleMode" : "PIPELINED" - }, { - "source" : 12, - "target" : 13, - "shuffle" : { - "type" : "FORWARD" - }, - "shuffleMode" : "PIPELINED" - } ] -} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/deduplicate-keep-first-ttl-expired-restore/savepoint/_metadata deleted file mode 100644 index 1368d5d796a79568b18a4735c6ed81d9c3539547..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14647 zcmeHOU2G#)6~1<|KPlZ!whhZ>fhCA^rB&6wbXfwr4WV zZWihgwPVTibAQe~_uO;uJ?A@jmcFnVLkJy*KmA3V8yZ39N~R$Z%c|nKMi$$;WUGe0 zA;zzxvkQfV)lzY3K3^@=t`-WV+Tv2VQhi(b=zEhd{^7aTpZd$W?>Be96(|WaEhktH#wWGhn(5gZM8`_d%H#2wyIfN*|@cUZCoLyiZ%5%F=qlP6&<&%hG7Hw zQTRt*{VO_VnMl~D4NBYMG-Fehu*R)nbH0JiDkQq<5Q(D55<0FCeba7q4oau!(DT!Y zuA_%kYq_myw0A{xlD1to4BJNOBCy9hBH)<_cqRg#vB$gY@gB0Y-fX88xnrW_fx5P* zuG>BI2tv@S(>L%I<}|Etas|EJ^k%kfbYd-Ep1+bWe`$Aj7fn9-Zz@*yU*D(Y;J{UE zQ`I;3jNPC_&+v^zP*f3MCz}FQBc_hEn2wu7j!~sBtymMcTt*{k;%edP!a}W5rQ2oo z=~4lWT+5drJBlXO^2OCctx_m2<%>(}g)&S1T!3}lQaQnDmKz&wyEIC6O^$lsbK!pobcgOK6i?cutd&1B7CJEUhx*g7so{_+I$Z!Kc z!W;Oo=f~r|`LTBsY3p920k)5D8wu^)qiE+2)6Tl+nNPwCT$dd zU<3xxmn{>W=^I{ihM^Ix|7P0U`TdLU{tbYmb#!bCYi-hL9T^-+2JnOq1$dfUSt%Cs%gE!6M^I2}_lIs=aeCA?kmA$Y{|^i@CShGxWmp}$gEj!r zy6-9!luJ090a88~p8DaQ&4&Xce@JHYF_^?>zXz8IcQ&InY|+PO>17OTjPHK>r*AbP z0H*hTAFCK`QzDnADezfi2D-H&3fIw?DAX*L_si_;XTRS0@=W9JcR%%~ced|BU5di* zM9{Q^6WK&PL9$XNn~lel=`2x{jHt*tB?sY_3>16VEe<#K>jsM>4uE~eX}?S`#D|!j z;+8t(;E|lkkX$B{j^TtHk0s+mJeJEUsaP^0WaYF75vX_qIC~5J5Il;YfqXl`iRqP6 zp`5R-l-;Pn!cwJLUYc92F0CxruB|K-&_o3S?A2N?qX1O*1DDN!8M6PTgH01}b937^ zDVb_hwbd<>r%B#j=pLEA4T%kd=W@CN@v+O(7F;*P#C3E+Ow{O${_A(%y!?wlzp#7Z zr5{#*_C|ws#SQ=a#v1HOT0UkH+XPb-?p9l~nSQ$ECZ9(SyF~(7uN`_gu`VlVA(5=d z6j2gm$qbfaSt*-|31UV`N)l00q5}58Z=u7W5&QRU7)o(ecHBjkXtLF{h5z-VS3mKC zXTDPd1GI=vD>Sd01*Y(XV+zxFDqmDB2qK#3h}+ob19Q)D8zA=U&Ww>L8U>4<^>uEW zMx$&r?&&B*@MbsAxSL-xESQvrZlgyxJdVLi{RWY2@C-(8s=92W=>aYQnB^7^?Oufv zQ1W!kCDmo!XFk9a!JZ7jAAr8!{)cfq&Um-Ji0+`s(F;Q#iZ?J) zUU?GysEf?wqWa6t8=3|~219Ahtm;ieRuz@S)iy11P~xW|cj#@xe|~^z@OfzUIBY(2;Aoi(5qh1b=LxC^+g9eMj99%FF+FC3}1J$8*%ZZtZ1*y zwG1d(pErtY4Ggxs-QfF0`7RDl>j1vOgb_i2QHTc-LZ$nW3a$R~SATc%m8T|u!qAe3 zfX61b9ad67kWVs9!jle>0An(AzDH=WSFMYu*k~ZuG6|@l-5-QJ13GMUCD_!UUVHhT z0W&i|31k7m_e4Mlly9!bH#dN9CPKIz { private final RowType inputRowType; + private final boolean isEventTime; + private transient ProjectedRowData projectedOutput; private transient RowData.FieldGetter[] payloadGetters; - private transient TypeSerializer candidateSerializer; + private transient TypeSerializer[] payloadSerializers; public DeduplicateKeepFirst(final SpecializedContext context) { super(BuiltInFunctionDefinitions.DEDUPLICATE_KEEP_FIRST, context); @@ -68,6 +69,7 @@ public DeduplicateKeepFirst(final SpecializedContext context) { .orElseThrow(() -> new IllegalStateException("Table argument expected.")); this.outputIndices = ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics); this.inputRowType = (RowType) tableSemantics.dataType().getLogicalType(); + this.isEventTime = tableSemantics.timeColumn() != -1; } @Override @@ -75,13 +77,12 @@ public void open(final FunctionContext context) throws Exception { super.open(context); projectedOutput = ProjectedRowData.from(outputIndices); payloadGetters = new RowData.FieldGetter[outputIndices.length]; - final LogicalType[] payloadTypes = new LogicalType[outputIndices.length]; + payloadSerializers = new TypeSerializer[outputIndices.length]; for (int i = 0; i < outputIndices.length; i++) { - payloadTypes[i] = inputRowType.getTypeAt(outputIndices[i]); - payloadGetters[i] = RowData.createFieldGetter(payloadTypes[i], outputIndices[i]); + final LogicalType payloadType = inputRowType.getTypeAt(outputIndices[i]); + payloadGetters[i] = RowData.createFieldGetter(payloadType, outputIndices[i]); + payloadSerializers[i] = InternalSerializers.create(payloadType); } - candidateSerializer = - InternalSerializers.create(RowType.of(RowType.of(payloadTypes), new BigIntType())); } public void eval( @@ -92,19 +93,18 @@ public void eval( @Nullable final Long stateTtl, @Nullable final Boolean resetTtlOnDuplicate) throws Exception { - // state_ttl is applied at plan time via getTimeToLive(); the PTF codegen still requires - // a parameter for every declared argument, so it is unused here. + // state_ttl and reset_ttl_on_duplicate are applied at plan time via getTimeToLive(); the + // PTF codegen still requires a parameter for every declared argument, so it is unused here. final RowKind kind = input.getRowKind(); if (kind == RowKind.UPDATE_BEFORE || kind == RowKind.DELETE) { return; } - final TimeContext context = ctx.timeContext(Long.class); - final Long rowtime = context.time(); - - if (rowtime != null) { // watermark mode + if (isEventTime) { + final TimeContext context = ctx.timeContext(Long.class); + final Long rowtime = context.time(); final Long watermark = context.tableWatermark(); - if (watermark != null && rowtime < watermark) { + if (watermark != null && rowtime <= watermark) { return; } @@ -117,8 +117,7 @@ public void eval( context.clearTimer(currentLowestEventTime); } // object reuse may overwrite the input's memory before the timer fires - candidate.setValue( - candidateSerializer.copy(materializeCandidate(input, rowtime))); + candidate.setValue(materializeCandidate(input, rowtime)); context.registerOnTime(rowtime); } } else { // already emitted → drop, optionally refresh the TTL @@ -158,7 +157,8 @@ public void onTimer( private GenericRowData materializeCandidate(final RowData input, final long rowtime) { final GenericRowData payload = new GenericRowData(outputIndices.length); for (int i = 0; i < outputIndices.length; i++) { - payload.setField(i, payloadGetters[i].getFieldOrNull(input)); + final Object field = payloadGetters[i].getFieldOrNull(input); + payload.setField(i, field == null ? null : payloadSerializers[i].copy(field)); } return GenericRowData.of(payload, rowtime); } From b328f5303a63e71e0528409ca2a5aeaee6c02a68 Mon Sep 17 00:00:00 2001 From: Vas Shabu Date: Tue, 20 Oct 2026 17:18:24 +0100 Subject: [PATCH 5/5] [FLINK-40896][table] Optimised state assigment, reconfigured eval to go into different functions given the input args --- .../DeduplicateKeepFirstTypeStrategy.java | 2 +- .../functions/ptf/DeduplicateKeepFirst.java | 101 ++++++++++++------ 2 files changed, 68 insertions(+), 35 deletions(-) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java index 74c489dbff8cba..28232e0846d4c2 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/DeduplicateKeepFirstTypeStrategy.java @@ -45,7 +45,7 @@ public final class DeduplicateKeepFirstTypeStrategy { public static final int ARG_STATE_TTL = 1; public static final int ARG_RESET_TTL_ON_DUPLICATE = 2; private static final String SEEN_STATE_NAME = "seen"; - private static final String CANDIDATE_STATE_NAME = "candidate"; + public static final String CANDIDATE_STATE_NAME = "candidate"; public static final InputTypeStrategy INPUT_TYPE_STRATEGY = new ValidationOnlyInputTypeStrategy() { diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java index 23ecde7d28da58..dd2ee20104fc89 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/DeduplicateKeepFirst.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.table.api.dataview.ValueView; +import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.utils.ProjectedRowData; @@ -30,6 +31,8 @@ import org.apache.flink.table.functions.TableSemantics; import org.apache.flink.table.runtime.typeutils.InternalSerializers; import org.apache.flink.table.types.inference.CallContext; +import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.TypeInference; import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils; import org.apache.flink.table.types.inference.strategies.DeduplicateKeepFirstTypeStrategy; import org.apache.flink.table.types.logical.LogicalType; @@ -38,6 +41,8 @@ import javax.annotation.Nullable; +import java.util.LinkedHashMap; + /** Implementation of the {@code DEDUPLICATE_KEEP_FIRST} process table function. */ @Internal public class DeduplicateKeepFirst extends BuiltInProcessTableFunction { @@ -85,10 +90,29 @@ public void open(final FunctionContext context) throws Exception { } } + @Override + public TypeInference getTypeInference(final DataTypeFactory typeFactory) { + final TypeInference typeInference = super.getTypeInference(typeFactory); + if (isEventTime) { + return typeInference; + } + final LinkedHashMap stateTypeStrategies = + new LinkedHashMap<>(typeInference.getStateTypeStrategies()); + stateTypeStrategies.remove(DeduplicateKeepFirstTypeStrategy.CANDIDATE_STATE_NAME); + return TypeInference.newBuilder() + .staticArguments( + typeInference.getStaticArguments().orElseThrow(IllegalStateException::new)) + .inputTypeStrategy(typeInference.getInputTypeStrategy()) + .stateTypeStrategies(stateTypeStrategies) + .outputTypeStrategy(typeInference.getOutputTypeStrategy()) + .disableSystemArguments(typeInference.disableSystemArguments()) + .build(); + } + + /** Watermarkless mode: emits the first arriving row per key. */ public void eval( final Context ctx, final ValueView seen, - final ValueView candidate, final RowData input, @Nullable final Long stateTtl, @Nullable final Boolean resetTtlOnDuplicate) @@ -100,46 +124,55 @@ public void eval( return; } - if (isEventTime) { - final TimeContext context = ctx.timeContext(Long.class); - final Long rowtime = context.time(); - final Long watermark = context.tableWatermark(); - if (watermark != null && rowtime <= watermark) { - return; - } + if (seen.getValue() == null) { + projectedOutput.replaceRow(input); + projectedOutput.setRowKind(RowKind.INSERT); + collect(projectedOutput); + seen.setValue(true); + } else if (resetTtlOnDuplicate == null || resetTtlOnDuplicate) { + seen.setValue(true); + } + } - if (seen.getValue() == null) { // not yet emitted → buffer the earliest candidate - final RowData currLowest = candidate.getValue(); - final Long currentLowestEventTime = - currLowest == null ? null : currLowest.getLong(CANDIDATE_EVENT_TIME); - if (currentLowestEventTime == null || rowtime < currentLowestEventTime) { - if (currentLowestEventTime != null) { - context.clearTimer(currentLowestEventTime); - } - // object reuse may overwrite the input's memory before the timer fires - candidate.setValue(materializeCandidate(input, rowtime)); - context.registerOnTime(rowtime); - } - } else { // already emitted → drop, optionally refresh the TTL - if (resetTtlOnDuplicate == null || resetTtlOnDuplicate) { - seen.setValue(true); - } - } + /** Event-time mode: emits the row with the earliest event time per key. */ + public void eval( + final Context ctx, + final ValueView seen, + final ValueView candidate, + final RowData input, + @Nullable final Long stateTtl, + @Nullable final Boolean resetTtlOnDuplicate) + throws Exception { + final TimeContext context = ctx.timeContext(Long.class); + final Long rowtime = context.time(); + final Long watermark = context.tableWatermark(); + if (watermark != null && rowtime <= watermark) { + return; + } - } else { // watermarkless mode - if (seen.getValue() == null) { - projectedOutput.replaceRow(input); - projectedOutput.setRowKind(RowKind.INSERT); - collect(projectedOutput); - seen.setValue(true); - } else { - if (resetTtlOnDuplicate == null || resetTtlOnDuplicate) { - seen.setValue(true); + if (seen.getValue() == null) { // not yet emitted → buffer the earliest candidate + final RowData currLowest = candidate.getValue(); + final Long currentLowestEventTime = + currLowest == null ? null : currLowest.getLong(CANDIDATE_EVENT_TIME); + if (currentLowestEventTime == null || rowtime < currentLowestEventTime) { + if (currentLowestEventTime != null) { + context.clearTimer(currentLowestEventTime); } + // object reuse may overwrite the input's memory before the timer fires + candidate.setValue(materializeCandidate(input, rowtime)); + context.registerOnTime(rowtime); } + } else if (resetTtlOnDuplicate == null || resetTtlOnDuplicate) { + seen.setValue(true); } } + /** + * Never called: watermarkless mode registers no timers, but codegen requires an {@code onTimer} + * matching the state layout once any {@code onTimer} is declared. + */ + public void onTimer(final OnTimerContext ctx, final ValueView seen) {} + public void onTimer( final OnTimerContext ctx, final ValueView seen,