You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2026/05/29 09:20:16 UTC
(flink) branch master updated (692507671e8 -> c033b17fae2)
This is an automated email from the ASF dual-hosted git repository.
gustavodemorais pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 692507671e8 [FLINK-39425][metrics] `OpenTelemetryEventReporter` ignores `service.name` and `service.version config`
new bca2c0a6170 [FLINK-39636][table] Add produces_full_deletes parameter to TO_CHANGELOG
new 340fbf879fd [FLINK-39636][table] Refactor ToChangelogTypeStrategy
new d763453e045 [FLINK-39735][table] Expose input upsert key on TableSemantics
new 32f9e60eb9d [FLINK-39636][table] Emit partial DELETE rows in TO_CHANGELOG via upsert key
new e400c2c2213 [FLINK-39636][table] Remove unnecessary validation in TO_CHANGELOG
new c033b17fae2 [FLINK-39636][table] Widen output column nullability for partial deletes
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../docs/sql/reference/queries/changelog.md | 115 +++++++++++--
flink-python/pyflink/table/table.py | 19 ++-
.../apache/flink/table/api/PartitionedTable.java | 10 +-
.../java/org/apache/flink/table/api/Table.java | 10 +-
.../resolver/rules/ResolveCallByArgumentsRule.java | 5 +
.../functions/BuiltInFunctionDefinitions.java | 46 +++--
.../flink/table/functions/TableSemantics.java | 33 ++++
.../table/types/inference/BuiltInCondition.java | 1 +
.../table/types/inference/TraitCondition.java | 11 ++
.../strategies/ChangelogTypeStrategyUtils.java | 24 +++
.../strategies/ToChangelogTypeStrategy.java | 182 +++++++++++++-------
.../apache/flink/table/utils/UpsertKeyUtils.java | 67 ++++++++
.../ToChangelogInputTypeStrategyTest.java | 133 +++++++++++++++
.../ToChangelogOutputTypeStrategyTest.java | 110 ++++++++++++
.../types/inference/utils/TableSemanticsMock.java | 25 +++
.../functions/bridging/BridgingSqlFunction.java | 19 ++-
.../inference/CallBindingCallContext.java | 6 +
.../inference/OperatorBindingCallContext.java | 48 +++++-
.../stream/StreamExecProcessTableFunction.java | 38 +++--
.../stream/StreamPhysicalProcessTableFunction.java | 19 ++-
.../table/planner/plan/utils/UpsertKeyUtil.java | 23 +--
.../codegen/ProcessTableRunnerGenerator.scala | 10 +-
.../exec/stream/ToChangelogSemanticTests.java | 9 +-
.../nodes/exec/stream/ToChangelogTestPrograms.java | 190 +++++++++++++++++++--
.../planner/plan/stream/sql/ToChangelogTest.java | 38 +++++
.../planner/plan/stream/sql/ToChangelogTest.xml | 66 +++++--
.../plan/to-changelog-retract-restore.json | 10 +-
.../runtime/functions/ptf/ToChangelogFunction.java | 132 ++++++++++----
.../operators/process/RuntimeTableSemantics.java | 11 +-
...essSetTableOperatorInterruptibleTimersTest.java | 3 +-
.../functions/TestHarnessTableSemantics.java | 14 ++
31 files changed, 1235 insertions(+), 192 deletions(-)
create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/UpsertKeyUtils.java
create mode 100644 flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java
create mode 100644 flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogOutputTypeStrategyTest.java
(flink) 05/06: [FLINK-39636][table] Remove unnecessary validation in TO_CHANGELOG
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gustavodemorais pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit e400c2c221368e98e80d5589cacbcb0b1585ba66
Author: Ramin Gharib <rg...@confluent.io>
AuthorDate: Thu May 28 15:25:50 2026 +0200
[FLINK-39636][table] Remove unnecessary validation in TO_CHANGELOG
Drop two checks that protected against user mistakes the engine can already handle silently:
- op_mapping entries that reference change operations the input cannot produce no longer throw. The extra entries are unreachable at runtime, not harmful.
- produces_full_deletes=true against an append-only input no longer throws. The argument is a no-op there, equivalent to omitting it.
The remaining check covers the only case where the argument changes semantics in a way the function cannot honor:
produces_full_deletes=false with no upsert key and no PARTITION BY, leaving nothing to preserve when nulling non-key columns. Rename validateProducesFullDeletes to validateProducesPartialDeletes to reflect the surviving case.
Clean up the now-redundant test cases, javadocs, and docs sections.
---
.../docs/sql/reference/queries/changelog.md | 32 +++++-----
.../apache/flink/table/api/PartitionedTable.java | 4 +-
.../java/org/apache/flink/table/api/Table.java | 4 +-
.../flink/table/functions/TableSemantics.java | 29 +++++----
.../strategies/ToChangelogTypeStrategy.java | 39 -------------
.../ToChangelogInputTypeStrategyTest.java | 14 -----
.../stream/StreamExecProcessTableFunction.java | 8 +--
.../exec/stream/ToChangelogSemanticTests.java | 2 -
.../nodes/exec/stream/ToChangelogTestPrograms.java | 26 ---------
.../runtime/functions/ptf/ToChangelogFunction.java | 68 ++++------------------
10 files changed, 48 insertions(+), 178 deletions(-)
diff --git a/docs/content/docs/sql/reference/queries/changelog.md b/docs/content/docs/sql/reference/queries/changelog.md
index c91107b2df2..641bb7245f7 100644
--- a/docs/content/docs/sql/reference/queries/changelog.md
+++ b/docs/content/docs/sql/reference/queries/changelog.md
@@ -308,18 +308,18 @@ SELECT * FROM TO_CHANGELOG(
| `input` | Yes | The input table. With `PARTITION BY`, rows with the same key are co-located and run in the same operator instance. Without `PARTITION BY`, each row is processed independently. Accepts insert-only, retract, and upsert tables. For upsert tables, the provided `PARTITION BY` key should match or be a subset of the upsert key of the subquery. |
| `op` | No | A `DESCRIPTOR` with a single column name for the operation code column. Defaults to `op`. |
| `op_mapping` | No | A `MAP<STRING, STRING>` mapping change operation names to custom output codes. Keys can contain comma-separated names to map multiple operations to the same code (e.g., `'INSERT, UPDATE_AFTER'`). When provided, only mapped operations are forwarded - unmapped events are dropped. Each change operation may appear at most once across all entries. |
-| `produces_full_deletes` | No | A `BOOLEAN` literal that controls how DELETE rows are emitted. When `true` (default), DELETE rows carry all columns, the full image. When `false`, only the identifying key columns are preserved and the rest are nulled. See [Full vs partial deletes](#full-vs-partial-deletes) for more details. |
+| `produces_full_deletes` | No | A `BOOLEAN` literal that controls how DELETE rows are emitted. When `true` (default), DELETE rows carry all columns, the full image. When `false`, only the identifying key columns are preserved and the rest are nulled. See [Full vs partial deletes](#full-vs-partial-deletes) for more details. |
#### Default op_mapping
When `op_mapping` is omitted, all four change operations are mapped to their standard names:
| Change Operation | Output value |
-|:----------------|:------------------|
-| INSERT | `'INSERT'` |
-| UPDATE_BEFORE | `'UPDATE_BEFORE'` |
-| UPDATE_AFTER | `'UPDATE_AFTER'` |
-| DELETE | `'DELETE'` |
+|:-----------------|:------------------|
+| INSERT | `'INSERT'` |
+| UPDATE_BEFORE | `'UPDATE_BEFORE'` |
+| UPDATE_AFTER | `'UPDATE_AFTER'` |
+| DELETE | `'DELETE'` |
### Output Schema
@@ -406,26 +406,22 @@ An **upsert key** is a column or set of columns that uniquely identifies a row a
The planner derives the upsert key from the input table:
* A declared `PRIMARY KEY` on the source table when reading directly.
-* The grouping columns of an upstream `GROUP BY <key>`.
+* The grouping columns of an upstream `GROUP BY <key>` or `PARTITION BY <key>`.
* The keys propagated by operators that preserve them (e.g. lookup joins, calc-projections that keep the key columns).
-When no upsert key can be derived (e.g. a plain append-only source with no key constraint and no grouping upstream), the input has no row identity and downstream operators must treat it as append-only or fall back to retract semantics.
-
-`TO_CHANGELOG` consumes the upsert key to decide which columns to preserve when emitting partial DELETE rows. See [Full vs partial deletes](#full-vs-partial-deletes) below.
-
#### Full vs partial deletes
The `produces_full_deletes` argument controls how DELETE rows are emitted and what the planner requires from the input. The matrix below shows each combination with `PARTITION BY` (set semantics) and without (row semantics). When `false`, the function relies on the input table's [upsert key](#upsert-key) to decide which columns to preserve.
##### `produces_full_deletes => true` (default)
-The planner requires fully-populated DELETE rows on the input. For upsert sources that emit key-only deletes, a `ChangelogNormalize` operator is inserted upstream to materialize the full pre-image from state. For sources that already emit a full pre-image (e.g. retract), the flag is a no-op. The function then passes the input row through unchanged on DELETE.
+The planner requires fully-populated DELETE rows on the input. For upsert sources that emit key-only deletes, a `ChangelogNormalize` operator is inserted upstream to calculate the full pre-image from state. For sources that already emit a full pre-image (e.g. retract), the flag is a no-op. The function then passes the input row through unchanged on DELETE.
**Row semantics** (no `PARTITION BY`):
```sql
--- Upsert source: -D[id:5] (key-only).
--- ChangelogNormalize materializes the full pre-image from state.
+-- Upsert source: -D[id:5, name: null] (key-only).
+-- ChangelogNormalize calculates the full pre-image from state.
-- Output: +I[op:'DELETE', id:5, name:'Alice']
SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source)
@@ -438,8 +434,8 @@ SELECT * FROM TO_CHANGELOG(input => TABLE retract_source)
**Set semantics** (`PARTITION BY`):
```sql
--- Upsert source: -D[id:5] (key-only).
--- ChangelogNormalize materializes the full pre-image from state.
+-- Upsert source: -D[id:5, name: null] (key-only).
+-- ChangelogNormalize calculates the full pre-image from state.
-- Output: +I[id:5, op:'DELETE', name:'Alice']
SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source PARTITION BY id)
@@ -451,7 +447,7 @@ SELECT * FROM TO_CHANGELOG(input => TABLE retract_source PARTITION BY id)
##### `produces_full_deletes => false`
-The planner skips `ChangelogNormalize` and the function emits partial DELETE rows. This avoids the stateful normalization operator for upsert sources (e.g. Kafka compacted topics) where the full pre-image is not needed downstream. Requires an [upsert key](#upsert-key) to be present for the input table (row semantics) or `PARTITION BY` (set semantics); otherwise the call is rejected with a validation error.
+The planner skips `ChangelogNormalize` and the function emits partial DELETE rows. This avoids the stateful normalization operator for upsert sources (e.g. Kafka compacted topics) where the full pre-image is not needed downstream. This requires an [upsert key](#upsert-key) to be present for the input table (row semantics) or `PARTITION BY` (set semantics); otherwise the call is rejected with a validation error.
**Row semantics** (no `PARTITION BY`): the function preserves the planner-derived upsert key columns on DELETE rows and nulls the rest. The upsert key is typically a declared `PRIMARY KEY` when directly reading from a source or the key provided in a `GROUP BY <key>`.
@@ -465,7 +461,7 @@ SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source, produces_full_deletes =
SELECT * FROM TO_CHANGELOG(input => TABLE retract_source, produces_full_deletes => false)
```
-**Set semantics** (`PARTITION BY`): the function preserves the partition key and nulls every non-partition-key column on DELETE rows. The key used as the partition-key column should be the unique key that will be used as the record identifier. This matches the shape expected by upsert sinks and Kafka compacted topics.
+**Set semantics** (`PARTITION BY`): the function preserves the partition key and nulls every non-partition-key column on DELETE rows. The key used as the partition-key column should be the unique key that will be used as the record identifier. This matches the shape expected by upsert sinks such as Kafka compacted topics.
```sql
-- Upsert source: -D[id:5] (key-only).
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
index 401317d125d..ecf58fbe9b3 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
@@ -207,9 +207,7 @@ public interface PartitionedTable {
*
* // Opt out of full-delete semantics. When `true` (default), DELETE rows carry the full
* // pre-image. When `false`, only the identifying key columns are preserved and the rest
- * // are nulled. See [Full vs partial deletes](
- * // https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/sql/queries/changelog/#full-vs-partial-deletes)
- * // for more details.
+ * // are nulled. See in documentation for more details.
* Table result = table
* .partitionBy($("id"))
* .toChangelog(lit(false).asArgument("produces_full_deletes"));
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java
index ab4a617a2df..9d8183d7e4d 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java
@@ -1462,9 +1462,7 @@ public interface Table extends Explainable<Table>, Executable {
*
* // Opt out of full-delete semantics. When `true` (default), DELETE rows carry the full
* // pre-image. When `false`, only the identifying key columns are preserved and the rest
- * // are nulled. See [Full vs partial deletes](
- * // https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/sql/queries/changelog/#full-vs-partial-deletes)
- * // for more details.
+ * // are nulled. See in documentation for more details.
* Table result = table.toChangelog(
* lit(false).asArgument("produces_full_deletes")
* );
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/TableSemantics.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/TableSemantics.java
index 3d17401c5ef..1289d054ff6 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/TableSemantics.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/TableSemantics.java
@@ -133,20 +133,29 @@ public interface TableSemantics {
* Upsert key candidates derived from the passed table's metadata.
*
* <p>Returns a list of 0-based column index arrays that uniquely identify a row for upsert
- * semantics. This is distinct from {@link #partitionByColumns()}: partition keys describe
- * distribution and co-location, upsert keys describe row identity. Useful for functions that
- * need to emit key-only deletes, match UPDATE_BEFORE / UPDATE_AFTER pairs, or want to have a
- * unique identifier to interact with state.
+ * semantics. Useful for functions that need to emit key-only deletes, match UPDATE_BEFORE /
+ * UPDATE_AFTER pairs, or use a stable identifier to interact with state.
+ *
+ * <p>The upsert key describes row identity and is distinct from {@link #partitionByColumns()},
+ * which describes distribution and co-location. They are independent and frequently disagree:
+ *
+ * <pre>{@code
+ * -- Source declares PRIMARY KEY (id); the call partitions by region.
+ * -- partitionByColumns() = [region] (chosen by the caller)
+ * -- upsertKeyColumns() = [[id]] (derived from the source's PK)
+ * TO_CHANGELOG(input => TABLE source PARTITION BY region)
+ * }</pre>
*
* <p>Returns an empty list when no upsert key is derivable, or when the planner has not yet
* computed metadata (during type inference).
*
- * <p>When the planner derives multiple candidate upsert keys for the same input (e.g., a table
- * with several primary key constraints), all of them are returned. Picking which candidate to
- * use is the function's responsibility, and the choice must be stable across releases to keep
- * PTF state consistent after job restarts and upgrades. The order of the returned list is not
- * part of the contract; PTF authors should not depend on it. A typical choice is the smallest
- * candidate by cardinality, with ties broken by the column indices in ascending order.
+ * <p>The planner may derive more than one candidate for the same input. For example, an inner
+ * join of two tables each carrying their own primary key produces a result with both keys as
+ * separate candidates. Picking which candidate to use is the function's responsibility, and the
+ * choice must be stable across releases so PTF state stays consistent after job restarts and
+ * upgrades. The order of the returned list is not part of the contract; PTF authors should not
+ * depend on it. A typical stable choice is the smallest candidate by cardinality, with ties
+ * broken by the column indices in ascending order.
*
* @return Candidate upsert keys of the passed table, or an empty list if none.
*/
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
index 4c27aae4723..1434b221077 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
@@ -113,11 +113,6 @@ public final class ToChangelogTypeStrategy {
return error;
}
- error = validateProducesFullDeletes(callContext, throwOnFailure);
- if (error.isPresent()) {
- return error;
- }
-
return Optional.of(callContext.getArgumentDataTypes());
}
@@ -198,40 +193,6 @@ public final class ToChangelogTypeStrategy {
return Optional.empty();
}
- @SuppressWarnings("rawtypes")
- private static Optional<List<DataType>> validateProducesFullDeletes(
- final CallContext callContext, final boolean throwOnFailure) {
- final boolean isExplicit = !callContext.isArgumentNull(ARG_PRODUCES_FULL_DELETES);
- if (!isExplicit) {
- return Optional.empty();
- }
- if (!callContext.isArgumentLiteral(ARG_PRODUCES_FULL_DELETES)) {
- return callContext.fail(
- throwOnFailure,
- "The 'produces_full_deletes' argument must be a constant BOOLEAN literal.");
- }
- final boolean producesFullDeletes =
- callContext.getArgumentValue(ARG_PRODUCES_FULL_DELETES, Boolean.class).orElse(true);
- if (!producesFullDeletes) {
- return Optional.empty();
- }
- // The check against the input changelog mode lives in the function constructor since
- // TableSemantics#changelogMode() returns empty here at type-inference time. The mapping
- // check below only needs the literal op_mapping argument, so it lives here. Only runs
- // when the user explicitly set produces_full_deletes=true; the default true is not
- // validated since it is a safe no-op for any input.
- final Optional<Map> opMapping = callContext.getArgumentValue(ARG_OP_MAPPING, Map.class);
- if (opMapping.isPresent() && !mapsDelete((Map<String, String>) opMapping.get())) {
- return callContext.fail(
- throwOnFailure,
- "Invalid 'produces_full_deletes' for TO_CHANGELOG: the active 'op_mapping' "
- + "does not map DELETE rows, so no DELETE rows are emitted. Remove "
- + "the 'produces_full_deletes' argument or add a DELETE entry to "
- + "'op_mapping'.");
- }
- return Optional.empty();
- }
-
/**
* Returns {@code true} when at least one {@code op_mapping} key references {@code DELETE}. Keys
* may be comma-separated (e.g., {@code "INSERT, DELETE"}) per the user-facing contract.
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java
index 6b5773949db..36607885bd8 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java
@@ -98,20 +98,6 @@ class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
.calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, false)
.expectArgumentTypes(TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
- // Error: produces_full_deletes=true with op_mapping that strips DELETE
- TestSpec.forStrategy(
- "produces_full_deletes=true rejected when op_mapping omits DELETE",
- TO_CHANGELOG_INPUT_TYPE_STRATEGY)
- .calledWithArgumentTypes(
- TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
- .calledWithTableSemanticsAt(ARG_TABLE, new TableSemanticsMock(TABLE_TYPE))
- .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
- .calledWithLiteralAt(ARG_OP_MAPPING, Map.of("INSERT, UPDATE_AFTER", "X"))
- .calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, true)
- .expectErrorMessage(
- "Invalid 'produces_full_deletes' for TO_CHANGELOG: the active "
- + "'op_mapping' does not map DELETE rows"),
-
// Error: multi-column descriptor for `op`
TestSpec.forStrategy(
"Descriptor with multiple columns",
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecProcessTableFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecProcessTableFunction.java
index e0cf5f09187..8c00acb8e57 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecProcessTableFunction.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecProcessTableFunction.java
@@ -314,9 +314,9 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
private RuntimeTableSemantics createRuntimeTableSemantics(
StaticArgument tableArg, RexTableArgCall tableArgCall, List<Integer> inputTimeColumns) {
+ final int inputIndex = tableArgCall.getInputIndex();
final RuntimeChangelogMode consumedChangelogMode =
- RuntimeChangelogMode.serialize(
- inputChangelogModes.get(tableArgCall.getInputIndex()));
+ RuntimeChangelogMode.serialize(inputChangelogModes.get(inputIndex));
final DataType dataType;
if (tableArg.getDataType().isPresent()) {
dataType = tableArg.getDataType().get();
@@ -324,9 +324,7 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
dataType = DataTypes.of(FlinkTypeFactory.toLogicalRowType(tableArgCall.type));
}
- final int timeColumn = inputTimeColumns.get(tableArgCall.getInputIndex());
-
- final int inputIndex = tableArgCall.getInputIndex();
+ final int timeColumn = inputTimeColumns.get(inputIndex);
final List<int[]> upsertKeys =
inputIndex < inputUpsertKeys.size() ? inputUpsertKeys.get(inputIndex) : List.of();
return new RuntimeTableSemantics(
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
index 41c53979067..c35394c7694 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
@@ -54,9 +54,7 @@ public class ToChangelogSemanticTests extends SemanticTestBase {
ToChangelogTestPrograms.DELETION_FLAG,
ToChangelogTestPrograms.INVALID_DESCRIPTOR,
ToChangelogTestPrograms.INVALID_OP_MAPPING,
- ToChangelogTestPrograms.OP_MAPPING_REFERENCES_UNSUPPORTED_KIND,
ToChangelogTestPrograms.DUPLICATE_ROW_KIND,
- ToChangelogTestPrograms.INVALID_PRODUCES_FULL_DELETES_FOR_APPEND_ONLY,
ToChangelogTestPrograms.RETRACT_PRODUCES_PARTIAL_DELETES,
ToChangelogTestPrograms.UPSERT_PRODUCES_FULL_DELETES,
ToChangelogTestPrograms.UPSERT_PRODUCES_PARTIAL_DELETES,
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
index 909e2f4d1ef..71731109eda 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
@@ -573,19 +573,6 @@ public class ToChangelogTestPrograms {
"Unknown change operation: 'INVALID_KIND'")
.build();
- public static final TableTestProgram OP_MAPPING_REFERENCES_UNSUPPORTED_KIND =
- TableTestProgram.of(
- "to-changelog-op-mapping-references-unsupported-kind",
- "fails when op_mapping references a change operation the input cannot produce")
- .setupTableSource(SIMPLE_SOURCE)
- .runFailingSql(
- "SELECT * FROM TO_CHANGELOG("
- + "input => TABLE t, "
- + "op_mapping => MAP['INSERT', 'I', 'DELETE', 'D'])",
- ValidationException.class,
- "the input table only produces [INSERT] and does not produce [DELETE]")
- .build();
-
public static final TableTestProgram DUPLICATE_ROW_KIND =
TableTestProgram.of(
"to-changelog-duplicate-rowkind",
@@ -599,19 +586,6 @@ public class ToChangelogTestPrograms {
"Duplicate change operation: 'DELETE'")
.build();
- public static final TableTestProgram INVALID_PRODUCES_FULL_DELETES_FOR_APPEND_ONLY =
- TableTestProgram.of(
- "to-changelog-invalid-produces-full-deletes-for-append-only",
- "fails when produces_full_deletes=true on an input that never emits DELETE rows")
- .setupTableSource(SIMPLE_SOURCE)
- .runFailingSql(
- "SELECT * FROM TO_CHANGELOG("
- + "input => TABLE t, "
- + "produces_full_deletes => true)",
- ValidationException.class,
- "the input table only produces [INSERT] and never emits DELETE rows")
- .build();
-
// --------------------------------------------------------------------------------------------
// Full vs partial deletes matrix (input kind x PARTITION BY x produces_full_deletes)
// --------------------------------------------------------------------------------------------
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
index 66162e205b9..b2e88dee1dd 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
@@ -20,7 +20,6 @@ package org.apache.flink.table.runtime.functions.ptf;
import org.apache.flink.annotation.Internal;
import org.apache.flink.table.api.ValidationException;
-import org.apache.flink.table.connector.ChangelogMode;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RowData;
@@ -46,7 +45,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.stream.Collectors;
import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_OP_MAPPING;
import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_PRODUCES_FULL_DELETES;
@@ -96,17 +94,14 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
final Map<String, String> opMapping =
callContext.getArgumentValue(ARG_OP_MAPPING, Map.class).orElse(null);
this.rawOpMap = buildOpMap(opMapping);
- if (opMapping != null) {
- validateOpMap(this.rawOpMap, tableSemantics);
- }
- final boolean producesFullDeletesArg =
+
+ this.producesFullDelete =
callContext.getArgumentValue(ARG_PRODUCES_FULL_DELETES, Boolean.class).orElse(true);
final boolean isExplicit = !callContext.isArgumentNull(ARG_PRODUCES_FULL_DELETES);
- validateProducesFullDeletes(producesFullDeletesArg, isExplicit, tableSemantics);
+ validateProducesPartialDeletes(producesFullDelete, isExplicit, tableSemantics);
this.outputIndices = ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics);
this.inputRowType = (RowType) tableSemantics.dataType().getLogicalType();
- this.producesFullDelete = producesFullDeletesArg;
this.upsertKeyColumn =
computeUpsertKeyColumn(
this.outputIndices,
@@ -164,70 +159,27 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
return result;
}
- /**
- * Rejects user-provided mappings that reference change operations the input changelog cannot
- * produce. Without this check the extra entries are dead code: the corresponding rows never
- * arrive. E.g., if the input is INSERT-only, then UPDATE_BEFORE and DELETE mappings are
- * ignored, which is likely a user mistake.
- *
- * <p>Lives here rather than in the input type strategy because {@link
- * TableSemantics#changelogMode()} returns empty during type inference and is only populated at
- * specialization time, which is when this constructor runs.
- */
- private static void validateOpMap(
- final Map<RowKind, String> mapping, final TableSemantics tableSemantics) {
- final ChangelogMode inputMode = tableSemantics.changelogMode().orElse(null);
- if (inputMode == null) {
- return;
- }
- final List<RowKind> unsupported =
- mapping.keySet().stream()
- .filter(kind -> !inputMode.contains(kind))
- .collect(Collectors.toList());
- if (!unsupported.isEmpty()) {
- throw new ValidationException(
- String.format(
- "Invalid 'op_mapping' for TO_CHANGELOG: the input table only produces "
- + "%s and does not produce %s. Remove those entries from the "
- + "mapping.",
- inputMode.getContainedKinds(), unsupported));
- }
- }
-
/**
* Validates an explicit {@code produces_full_deletes} argument against the input.
*
- * <p>For {@code produces_full_deletes=true}, the input changelog must emit DELETE rows;
- * otherwise the parameter is dead. For {@code produces_full_deletes=false}, the input must
- * declare an upsert key or the call must use {@code PARTITION BY}; otherwise the function has
- * no identifying columns to preserve when nulling the rest.
+ * <p>For {@code produces_full_deletes=false}, the input must declare an upsert key or the call
+ * must use {@code PARTITION BY}; otherwise the function has no identifying columns to preserve
+ * when nulling the rest.
*
- * <p>No validation runs when the argument is absent, since the default (full deletes) is safe
- * for any input.
+ * <p>No validation runs when the argument is absent or is set to true (default).
*
* <p>Lives here rather than in the input type strategy because {@link
* TableSemantics#changelogMode()} and {@link TableSemantics#upsertKeyColumns()} are only
* populated at specialization time.
*/
- private static void validateProducesFullDeletes(
+ private static void validateProducesPartialDeletes(
final boolean producesFullDeletesArg,
final boolean isExplicit,
final TableSemantics tableSemantics) {
- if (!isExplicit) {
- return;
- }
- if (producesFullDeletesArg) {
- final ChangelogMode inputMode = tableSemantics.changelogMode().orElse(null);
- if (inputMode != null && !inputMode.contains(RowKind.DELETE)) {
- throw new ValidationException(
- String.format(
- "Invalid 'produces_full_deletes' for TO_CHANGELOG: the input "
- + "table only produces %s and never emits DELETE rows. "
- + "Remove the 'produces_full_deletes' argument.",
- inputMode.getContainedKinds()));
- }
+ if (!isExplicit || producesFullDeletesArg) {
return;
}
+
final boolean hasPartitionBy = tableSemantics.partitionByColumns().length > 0;
final boolean hasUpsertKey = !tableSemantics.upsertKeyColumns().isEmpty();
if (!hasPartitionBy && !hasUpsertKey) {
(flink) 04/06: [FLINK-39636][table] Emit partial DELETE rows in TO_CHANGELOG via upsert key
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gustavodemorais pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 32f9e60eb9d7c33bac4fb3f91dc0200f03507c0c
Author: Ramin Gharib <ra...@gmail.com>
AuthorDate: Tue May 26 12:59:44 2026 +0200
[FLINK-39636][table] Emit partial DELETE rows in TO_CHANGELOG via upsert key
Wires TO_CHANGELOG to consume the upsert key surfaced by TableSemantics so row-semantic partial DELETE rows preserve identifying columns without requiring PARTITION BY. Also flips the produces_full_deletes default to true so the safe full-pre-image behavior is the default, and false is the explicit opt-in to partial deletes.
---
.../docs/sql/reference/queries/changelog.md | 91 +++++++++----
flink-python/pyflink/table/table.py | 19 +--
.../apache/flink/table/api/PartitionedTable.java | 10 +-
.../java/org/apache/flink/table/api/Table.java | 10 +-
.../functions/BuiltInFunctionDefinitions.java | 20 ++-
.../strategies/ToChangelogTypeStrategy.java | 54 ++++----
.../ToChangelogInputTypeStrategyTest.java | 75 +++++------
.../inference/OperatorBindingCallContext.java | 7 +-
.../exec/stream/ToChangelogSemanticTests.java | 14 +-
.../nodes/exec/stream/ToChangelogTestPrograms.java | 81 ++++++------
.../runtime/functions/ptf/ToChangelogFunction.java | 144 +++++++++++++++------
11 files changed, 321 insertions(+), 204 deletions(-)
diff --git a/docs/content/docs/sql/reference/queries/changelog.md b/docs/content/docs/sql/reference/queries/changelog.md
index 7a1598ee850..c91107b2df2 100644
--- a/docs/content/docs/sql/reference/queries/changelog.md
+++ b/docs/content/docs/sql/reference/queries/changelog.md
@@ -308,7 +308,7 @@ SELECT * FROM TO_CHANGELOG(
| `input` | Yes | The input table. With `PARTITION BY`, rows with the same key are co-located and run in the same operator instance. Without `PARTITION BY`, each row is processed independently. Accepts insert-only, retract, and upsert tables. For upsert tables, the provided `PARTITION BY` key should match or be a subset of the upsert key of the subquery. |
| `op` | No | A `DESCRIPTOR` with a single column name for the operation code column. Defaults to `op`. |
| `op_mapping` | No | A `MAP<STRING, STRING>` mapping change operation names to custom output codes. Keys can contain comma-separated names to map multiple operations to the same code (e.g., `'INSERT, UPDATE_AFTER'`). When provided, only mapped operations are forwarded - unmapped events are dropped. Each change operation may appear at most once across all entries. |
-| `produces_full_deletes` | No | A `BOOLEAN` literal that controls how DELETE rows are emitted. When `true`, the function requires fully-populated DELETE rows from the input. The planner inserts a `ChangelogNormalize` operator for upsert sources that emit key-only deletes, so downstream sees the full pre-image on DELETE. When `false` (default), no full-delete requirement is enforced. Partial DELETE rows from the input pass through unchanged. With `PARTITION BY` (set semantics), the [...]
+| `produces_full_deletes` | No | A `BOOLEAN` literal that controls how DELETE rows are emitted. When `true` (default), DELETE rows carry all columns, the full image. When `false`, only the identifying key columns are preserved and the rest are nulled. See [Full vs partial deletes](#full-vs-partial-deletes) for more details. |
#### Default op_mapping
@@ -399,41 +399,89 @@ SELECT * FROM TO_CHANGELOG(
-- UPDATE_BEFORE is dropped (not in the mapping)
```
-#### Delete handling
+#### Upsert key
-The `produces_full_deletes` argument controls how DELETE rows are emitted and what the planner requires from the input. The behavior depends on whether `PARTITION BY` is used (set semantics) or not (row semantics).
+An **upsert key** is a column or set of columns that uniquely identifies a row across its lifecycle in a changelog. It is what downstream operators and sinks use to decide which earlier row a new INSERT, UPDATE_AFTER, or DELETE refers to.
-**With `produces_full_deletes => true`.** The planner requires the input to produce DELETE rows with all columns populated. For upsert sources, a `ChangelogNormalize` operator is inserted to materialize the full pre-image from state. The function then emits fully-populated DELETE rows downstream.
+The planner derives the upsert key from the input table:
+
+* A declared `PRIMARY KEY` on the source table when reading directly.
+* The grouping columns of an upstream `GROUP BY <key>`.
+* The keys propagated by operators that preserve them (e.g. lookup joins, calc-projections that keep the key columns).
+
+When no upsert key can be derived (e.g. a plain append-only source with no key constraint and no grouping upstream), the input has no row identity and downstream operators must treat it as append-only or fall back to retract semantics.
+
+`TO_CHANGELOG` consumes the upsert key to decide which columns to preserve when emitting partial DELETE rows. See [Full vs partial deletes](#full-vs-partial-deletes) below.
+
+#### Full vs partial deletes
+
+The `produces_full_deletes` argument controls how DELETE rows are emitted and what the planner requires from the input. The matrix below shows each combination with `PARTITION BY` (set semantics) and without (row semantics). When `false`, the function relies on the input table's [upsert key](#upsert-key) to decide which columns to preserve.
+
+##### `produces_full_deletes => true` (default)
+
+The planner requires fully-populated DELETE rows on the input. For upsert sources that emit key-only deletes, a `ChangelogNormalize` operator is inserted upstream to materialize the full pre-image from state. For sources that already emit a full pre-image (e.g. retract), the flag is a no-op. The function then passes the input row through unchanged on DELETE.
+
+**Row semantics** (no `PARTITION BY`):
```sql
-- Upsert source: -D[id:5] (key-only).
-- ChangelogNormalize materializes the full pre-image from state.
-- Output: +I[op:'DELETE', id:5, name:'Alice']
-SELECT * FROM TO_CHANGELOG(
- input => TABLE upsert_source,
- produces_full_deletes => true
-)
+SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source)
+
+-- Retract source: -D[id:5, name:'Alice'] (full pre-image).
+-- No ChangelogNormalize inserted; the input row is passed through unchanged.
+-- Output: +I[op:'DELETE', id:5, name:'Alice']
+SELECT * FROM TO_CHANGELOG(input => TABLE retract_source)
```
-**With `produces_full_deletes => false` (default).** The planner does not require fully-populated DELETE rows on the input. For upsert sources that emit key-only deletes (e.g. Kafka compacted topics), this avoids the stateful `ChangelogNormalize` operator that would otherwise materialize the full pre-image of each deleted row.
+**Set semantics** (`PARTITION BY`):
-In **row semantics** (no `PARTITION BY`) the function passes the input row through unchanged. If the source emits partial DELETE rows they remain partial downstream; if it emits full DELETE rows they remain full.
+```sql
+-- Upsert source: -D[id:5] (key-only).
+-- ChangelogNormalize materializes the full pre-image from state.
+-- Output: +I[id:5, op:'DELETE', name:'Alice']
+SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source PARTITION BY id)
+
+-- Retract source: -D[id:5, name:'Alice'] (full pre-image).
+-- No ChangelogNormalize inserted; the input row is passed through unchanged.
+-- Output: +I[id:5, op:'DELETE', name:'Alice']
+SELECT * FROM TO_CHANGELOG(input => TABLE retract_source PARTITION BY id)
+```
+
+##### `produces_full_deletes => false`
+
+The planner skips `ChangelogNormalize` and the function emits partial DELETE rows. This avoids the stateful normalization operator for upsert sources (e.g. Kafka compacted topics) where the full pre-image is not needed downstream. Requires an [upsert key](#upsert-key) to be present for the input table (row semantics) or `PARTITION BY` (set semantics); otherwise the call is rejected with a validation error.
+
+**Row semantics** (no `PARTITION BY`): the function preserves the planner-derived upsert key columns on DELETE rows and nulls the rest. The upsert key is typically a declared `PRIMARY KEY` when directly reading from a source or the key provided in a `GROUP BY <key>`.
```sql
--- Source emits -D[id:5] (key-only).
+-- Upsert source with PRIMARY KEY (id): -D[id:5] (key-only).
-- Output: +I[op:'DELETE', id:5, name:null]
-SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source)
+SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source, produces_full_deletes => false)
+
+-- Retract source with PRIMARY KEY (id): -D[id:5, name:'Alice'] (full pre-image).
+-- Output: +I[op:'DELETE', id:5, name:null]
+SELECT * FROM TO_CHANGELOG(input => TABLE retract_source, produces_full_deletes => false)
```
-In **set semantics** (`PARTITION BY`) the function additionally nulls every non-partition-key column on DELETE rows. This forces the output to carry only the partition key on DELETE even when the input row was fully populated, which matches the shape expected by upsert sinks and Kafka compacted topics.
+**Set semantics** (`PARTITION BY`): the function preserves the partition key and nulls every non-partition-key column on DELETE rows. The key used as the partition-key column should be the unique key that will be used as the record identifier. This matches the shape expected by upsert sinks and Kafka compacted topics.
```sql
--- Source emits -D[id:5, name:'Alice'] (full pre-image, e.g. from a retract source).
+-- Upsert source: -D[id:5] (key-only).
-- Output: +I[id:5, op:'DELETE', name:null]
-SELECT * FROM TO_CHANGELOG(input => TABLE retract_source PARTITION BY id)
-```
+SELECT * FROM TO_CHANGELOG(
+ input => TABLE upsert_source PARTITION BY id,
+ produces_full_deletes => false
+)
-There is no way to derive a partial DELETE in row semantics when the input emits a full pre-image, since the function has no key column to preserve. Use `PARTITION BY` for that case.
+-- Retract source: -D[id:5, name:'Alice'] (full pre-image).
+-- Output: +I[id:5, op:'DELETE', name:null]
+SELECT * FROM TO_CHANGELOG(
+ input => TABLE retract_source PARTITION BY id,
+ produces_full_deletes => false
+)
+```
#### Partitioning by a key
@@ -472,12 +520,11 @@ Table result = myTable.toChangelog(
map("INSERT, UPDATE_AFTER", "false", "DELETE", "true").asArgument("op_mapping")
);
-// Require fully-populated DELETE rows from the input (inserts a ChangelogNormalize for
-// upsert sources). When false (default), no full-delete requirement is enforced; in row
-// semantics the input passes through unchanged, in set semantics non-partition-key columns
-// are nulled on DELETE.
+// Opt out of full-delete semantics. When `true` (default), DELETE rows carry the full
+// pre-image. When `false`, only the identifying key columns are preserved and the rest
+// are nulled. See [Full vs partial deletes](#full-vs-partial-deletes) for more details.
Table result = myTable.toChangelog(
- lit(true).asArgument("produces_full_deletes")
+ lit(false).asArgument("produces_full_deletes")
);
// Set semantics: co-locate rows with the same key in the same parallel operator instance.
diff --git a/flink-python/pyflink/table/table.py b/flink-python/pyflink/table/table.py
index bb06642db9e..e7898e8752d 100644
--- a/flink-python/pyflink/table/table.py
+++ b/flink-python/pyflink/table/table.py
@@ -1197,12 +1197,13 @@ class Table(object):
(INSERT, UPDATE_BEFORE, UPDATE_AFTER, DELETE).
The optional ``produces_full_deletes`` boolean controls how DELETE rows are
- emitted. When ``True``, the planner inserts a ``ChangelogNormalize`` operator
- for upsert sources that emit key-only deletes so the function emits fully
- populated DELETE rows downstream. When ``False`` (default), no full-delete
- requirement is enforced. In row semantics the input is passed through unchanged,
- and in set semantics (``PARTITION BY``) non-partition-key columns are nulled on
- DELETE rows.
+ emitted. When ``True`` (default), the planner inserts a ``ChangelogNormalize``
+ operator for upsert sources that emit key-only deletes so the function emits
+ fully populated DELETE rows downstream. When ``False``, the function emits
+ partial DELETE rows: row semantics preserves the planner-derived upsert key
+ columns and nulls the rest, set semantics (``PARTITION BY``) preserves the
+ partition key and nulls the rest. Requires an upsert key or ``PARTITION BY``;
+ otherwise the call is rejected.
Example:
::
@@ -1221,10 +1222,10 @@ class Table(object):
... map_("INSERT, UPDATE_AFTER", "false",
... "DELETE", "true").as_argument("op_mapping")
... )
- >>> # Require fully populated DELETE rows from the input. Inserts a
- >>> # ChangelogNormalize for upsert sources.
+ >>> # Opt out of full-delete semantics to emit partial DELETE rows.
+ >>> # Requires an upsert key or PARTITION BY; otherwise rejected.
>>> result = table.to_changelog(
- ... lit(True).as_argument("produces_full_deletes")
+ ... lit(False).as_argument("produces_full_deletes")
... )
:param arguments: Optional named arguments for ``op``, ``op_mapping``, and
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
index 074bdfdbaef..401317d125d 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
@@ -205,12 +205,14 @@ public interface PartitionedTable {
* map("INSERT, UPDATE_AFTER", "false", "DELETE", "true").asArgument("op_mapping")
* );
*
- * // Require fully-populated DELETE rows from the input (inserts a ChangelogNormalize for
- * // upsert sources). When false (default), DELETE rows on upsert inputs may omit non-key
- * // columns, which avoids the stateful normalization operator upstream.
+ * // Opt out of full-delete semantics. When `true` (default), DELETE rows carry the full
+ * // pre-image. When `false`, only the identifying key columns are preserved and the rest
+ * // are nulled. See [Full vs partial deletes](
+ * // https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/sql/queries/changelog/#full-vs-partial-deletes)
+ * // for more details.
* Table result = table
* .partitionBy($("id"))
- * .toChangelog(lit(true).asArgument("produces_full_deletes"));
+ * .toChangelog(lit(false).asArgument("produces_full_deletes"));
* }</pre>
*
* @param arguments optional named arguments for {@code op}, {@code op_mapping}, and {@code
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java
index de6ce9d3e5e..ab4a617a2df 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java
@@ -1460,11 +1460,13 @@ public interface Table extends Explainable<Table>, Executable {
* map("INSERT, UPDATE_AFTER", "false", "DELETE", "true").asArgument("op_mapping")
* );
*
- * // Require fully-populated DELETE rows from the input (inserts a ChangelogNormalize for
- * // upsert sources). When false (default), no full-delete requirement is enforced and partial
- * // DELETE rows from the input pass through unchanged.
+ * // Opt out of full-delete semantics. When `true` (default), DELETE rows carry the full
+ * // pre-image. When `false`, only the identifying key columns are preserved and the rest
+ * // are nulled. See [Full vs partial deletes](
+ * // https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/sql/queries/changelog/#full-vs-partial-deletes)
+ * // for more details.
* Table result = table.toChangelog(
- * lit(true).asArgument("produces_full_deletes")
+ * lit(false).asArgument("produces_full_deletes")
* );
* }</pre>
*
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
index 19b8219e98b..0b9ea76b175 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
@@ -839,14 +839,20 @@ public final class BuiltInFunctionDefinitions {
"UPDATE_BEFORE"))))
.withConditionalTrait(
StaticArgumentTrait.REQUIRE_FULL_DELETE,
- // Require full deletes only when the user explicitly
- // asks for them via produces_full_deletes=TRUE *and*
- // the active op_mapping includes DELETE. Otherwise the
- // planner can skip ChangelogNormalize for upsert
- // sources that emit key-only deletes.
+ // Require full deletes by default. The user can opt
+ // out via produces_full_deletes=FALSE.
+ // REQUIRE_FULL_DELETE
+ // still gates on the active op_mapping mapping DELETE;
+ // otherwise no DELETE rows reach the function and there
+ // is no point inserting ChangelogNormalize upstream.
TraitCondition.and(
- TraitCondition.argIsEqualTo(
- "produces_full_deletes", Boolean.TRUE),
+ TraitCondition.or(
+ TraitCondition.not(
+ TraitCondition.argIsPresent(
+ "produces_full_deletes")),
+ TraitCondition.argIsEqualTo(
+ "produces_full_deletes",
+ Boolean.TRUE)),
TraitCondition.or(
TraitCondition.not(
TraitCondition.argIsPresent(
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
index d406875aa2f..4c27aae4723 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.types.inference.CallContext;
import org.apache.flink.table.types.inference.InputTypeStrategy;
import org.apache.flink.table.types.inference.TypeStrategy;
import org.apache.flink.types.ColumnList;
+import org.apache.flink.types.RowKind;
import java.util.ArrayList;
import java.util.Arrays;
@@ -52,6 +53,8 @@ public final class ToChangelogTypeStrategy {
private static final Set<String> VALID_ROW_KIND_NAMES =
Set.of("INSERT", "UPDATE_BEFORE", "UPDATE_AFTER", "DELETE");
+ private static final String DELETE = RowKind.DELETE.name();
+
// --------------------------------------------------------------------------------------------
// Input validation
// --------------------------------------------------------------------------------------------
@@ -152,7 +155,8 @@ public final class ToChangelogTypeStrategy {
final Optional<Map> opMapping = callContext.getArgumentValue(ARG_OP_MAPPING, Map.class);
if (opMapping.isPresent()) {
- return validateOpMappingKeys(callContext, opMapping.get(), throwOnFailure);
+ return validateOpMappingKeys(
+ callContext, (Map<String, String>) opMapping.get(), throwOnFailure);
}
return Optional.empty();
}
@@ -163,16 +167,13 @@ public final class ToChangelogTypeStrategy {
* trimmed. Names are case-sensitive and must match exactly (e.g., {@code INSERT}, not {@code
* insert}). Each name must be valid and appear at most once across all entries.
*/
- @SuppressWarnings("rawtypes")
private static Optional<List<DataType>> validateOpMappingKeys(
- final CallContext callContext, final Map opMapping, final boolean throwOnFailure) {
+ final CallContext callContext,
+ final Map<String, String> opMapping,
+ final boolean throwOnFailure) {
final Set<String> allRowKindsSeen = new HashSet<>();
- for (final Object key : opMapping.keySet()) {
- if (!(key instanceof String)) {
- return callContext.fail(
- throwOnFailure, "Invalid target mapping for argument 'op_mapping'.");
- }
- final String[] rowKindNames = ((String) key).split(",");
+ for (final String key : opMapping.keySet()) {
+ final String[] rowKindNames = key.split(",");
for (final String rawName : rowKindNames) {
final String rowKindName = rawName.trim();
if (!VALID_ROW_KIND_NAMES.contains(rowKindName)) {
@@ -180,7 +181,7 @@ public final class ToChangelogTypeStrategy {
throwOnFailure,
String.format(
"Invalid target mapping for argument 'op_mapping'. "
- + "Unknown change operation: '%s'. Valid values are: %s.",
+ + "Unknown change operation: '%s'. Operations are case-sensitive. Valid values are: %s.",
rowKindName, VALID_ROW_KIND_NAMES));
}
final boolean isDuplicate = !allRowKindsSeen.add(rowKindName);
@@ -200,24 +201,27 @@ public final class ToChangelogTypeStrategy {
@SuppressWarnings("rawtypes")
private static Optional<List<DataType>> validateProducesFullDeletes(
final CallContext callContext, final boolean throwOnFailure) {
- final boolean hasArgProvided = !callContext.isArgumentNull(ARG_PRODUCES_FULL_DELETES);
- if (hasArgProvided && !callContext.isArgumentLiteral(ARG_PRODUCES_FULL_DELETES)) {
+ final boolean isExplicit = !callContext.isArgumentNull(ARG_PRODUCES_FULL_DELETES);
+ if (!isExplicit) {
+ return Optional.empty();
+ }
+ if (!callContext.isArgumentLiteral(ARG_PRODUCES_FULL_DELETES)) {
return callContext.fail(
throwOnFailure,
"The 'produces_full_deletes' argument must be a constant BOOLEAN literal.");
}
final boolean producesFullDeletes =
- callContext
- .getArgumentValue(ARG_PRODUCES_FULL_DELETES, Boolean.class)
- .orElse(false);
+ callContext.getArgumentValue(ARG_PRODUCES_FULL_DELETES, Boolean.class).orElse(true);
if (!producesFullDeletes) {
return Optional.empty();
}
// The check against the input changelog mode lives in the function constructor since
// TableSemantics#changelogMode() returns empty here at type-inference time. The mapping
- // check below only needs the literal op_mapping argument, so it lives here.
+ // check below only needs the literal op_mapping argument, so it lives here. Only runs
+ // when the user explicitly set produces_full_deletes=true; the default true is not
+ // validated since it is a safe no-op for any input.
final Optional<Map> opMapping = callContext.getArgumentValue(ARG_OP_MAPPING, Map.class);
- if (opMapping.isPresent() && !mapsDelete(opMapping.get())) {
+ if (opMapping.isPresent() && !mapsDelete((Map<String, String>) opMapping.get())) {
return callContext.fail(
throwOnFailure,
"Invalid 'produces_full_deletes' for TO_CHANGELOG: the active 'op_mapping' "
@@ -229,17 +233,13 @@ public final class ToChangelogTypeStrategy {
}
/**
- * Returns {@code true} when at least one {@code op_mapping} key references {@code DELETE}.
- * Keys may be comma-separated (e.g., {@code "INSERT, DELETE"}) per the user-facing contract.
+ * Returns {@code true} when at least one {@code op_mapping} key references {@code DELETE}. Keys
+ * may be comma-separated (e.g., {@code "INSERT, DELETE"}) per the user-facing contract.
*/
- @SuppressWarnings("rawtypes")
- private static boolean mapsDelete(final Map opMapping) {
- for (final Object key : opMapping.keySet()) {
- if (!(key instanceof String)) {
- continue;
- }
- for (final String rawName : ((String) key).split(",")) {
- if ("DELETE".equals(rawName.trim())) {
+ private static boolean mapsDelete(final Map<String, String> opMapping) {
+ for (final String key : opMapping.keySet()) {
+ for (final String rawName : key.split(",")) {
+ if (DELETE.equals(rawName.trim())) {
return true;
}
}
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java
index e9b20a7a5ba..6b5773949db 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java
@@ -28,6 +28,10 @@ import java.util.Map;
import java.util.stream.Stream;
import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.TO_CHANGELOG_INPUT_TYPE_STRATEGY;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_OP;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_OP_MAPPING;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_PRODUCES_FULL_DELETES;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_TABLE;
/** Tests for {@link ToChangelogTypeStrategy#INPUT_TYPE_STRATEGY}. */
class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
@@ -52,12 +56,11 @@ class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
TO_CHANGELOG_INPUT_TYPE_STRATEGY)
.calledWithArgumentTypes(
TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
- .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
- .calledWithLiteralAt(1, ColumnList.of("op"))
- .calledWithLiteralAt(2, null)
- .calledWithLiteralAt(3, true)
- .expectArgumentTypes(
- TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
+ .calledWithTableSemanticsAt(ARG_TABLE, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(ARG_OP_MAPPING, null)
+ .calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, true)
+ .expectArgumentTypes(TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
// Valid: produces_full_deletes=true with op_mapping that includes DELETE
TestSpec.forStrategy(
@@ -65,12 +68,11 @@ class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
TO_CHANGELOG_INPUT_TYPE_STRATEGY)
.calledWithArgumentTypes(
TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
- .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
- .calledWithLiteralAt(1, ColumnList.of("op"))
- .calledWithLiteralAt(2, Map.of("INSERT", "I", "DELETE", "D"))
- .calledWithLiteralAt(3, true)
- .expectArgumentTypes(
- TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
+ .calledWithTableSemanticsAt(ARG_TABLE, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(ARG_OP_MAPPING, Map.of("INSERT", "I", "DELETE", "D"))
+ .calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, true)
+ .expectArgumentTypes(TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
// Valid: produces_full_deletes=true with comma-separated DELETE key
TestSpec.forStrategy(
@@ -78,12 +80,11 @@ class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
TO_CHANGELOG_INPUT_TYPE_STRATEGY)
.calledWithArgumentTypes(
TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
- .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
- .calledWithLiteralAt(1, ColumnList.of("op"))
- .calledWithLiteralAt(2, Map.of("INSERT, DELETE", "X"))
- .calledWithLiteralAt(3, true)
- .expectArgumentTypes(
- TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
+ .calledWithTableSemanticsAt(ARG_TABLE, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(ARG_OP_MAPPING, Map.of("INSERT, DELETE", "X"))
+ .calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, true)
+ .expectArgumentTypes(TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
// Valid: produces_full_deletes=false with op_mapping that omits DELETE
TestSpec.forStrategy(
@@ -91,12 +92,11 @@ class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
TO_CHANGELOG_INPUT_TYPE_STRATEGY)
.calledWithArgumentTypes(
TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
- .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
- .calledWithLiteralAt(1, ColumnList.of("op"))
- .calledWithLiteralAt(2, Map.of("INSERT, UPDATE_AFTER", "X"))
- .calledWithLiteralAt(3, false)
- .expectArgumentTypes(
- TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
+ .calledWithTableSemanticsAt(ARG_TABLE, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(ARG_OP_MAPPING, Map.of("INSERT, UPDATE_AFTER", "X"))
+ .calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, false)
+ .expectArgumentTypes(TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
// Error: produces_full_deletes=true with op_mapping that strips DELETE
TestSpec.forStrategy(
@@ -104,22 +104,22 @@ class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
TO_CHANGELOG_INPUT_TYPE_STRATEGY)
.calledWithArgumentTypes(
TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
- .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
- .calledWithLiteralAt(1, ColumnList.of("op"))
- .calledWithLiteralAt(2, Map.of("INSERT, UPDATE_AFTER", "X"))
- .calledWithLiteralAt(3, true)
+ .calledWithTableSemanticsAt(ARG_TABLE, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(ARG_OP_MAPPING, Map.of("INSERT, UPDATE_AFTER", "X"))
+ .calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, true)
.expectErrorMessage(
"Invalid 'produces_full_deletes' for TO_CHANGELOG: the active "
+ "'op_mapping' does not map DELETE rows"),
- // Error: multi-column descriptor
+ // Error: multi-column descriptor for `op`
TestSpec.forStrategy(
"Descriptor with multiple columns",
TO_CHANGELOG_INPUT_TYPE_STRATEGY)
.calledWithArgumentTypes(
TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
- .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
- .calledWithLiteralAt(1, ColumnList.of("a", "b"))
+ .calledWithTableSemanticsAt(ARG_TABLE, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("a", "b"))
.expectErrorMessage("must contain exactly one column name"),
// Error: invalid RowKind in op_mapping key
@@ -127,9 +127,9 @@ class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
"Invalid RowKind in mapping key", TO_CHANGELOG_INPUT_TYPE_STRATEGY)
.calledWithArgumentTypes(
TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
- .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
- .calledWithLiteralAt(1, ColumnList.of("op"))
- .calledWithLiteralAt(2, Map.of("INVALID_KIND", "X"))
+ .calledWithTableSemanticsAt(ARG_TABLE, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(ARG_OP_MAPPING, Map.of("INVALID_KIND", "X"))
.expectErrorMessage("Unknown change operation: 'INVALID_KIND'"),
// Error: duplicate RowKind across entries
@@ -138,9 +138,10 @@ class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
TO_CHANGELOG_INPUT_TYPE_STRATEGY)
.calledWithArgumentTypes(
TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
- .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
- .calledWithLiteralAt(1, ColumnList.of("op"))
- .calledWithLiteralAt(2, Map.of("INSERT, DELETE", "A", "DELETE", "B"))
+ .calledWithTableSemanticsAt(ARG_TABLE, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(
+ ARG_OP_MAPPING, Map.of("INSERT, DELETE", "A", "DELETE", "B"))
.expectErrorMessage("Duplicate change operation: 'DELETE'"));
}
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/OperatorBindingCallContext.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/OperatorBindingCallContext.java
index 7d1d3940899..8f05d7e2718 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/OperatorBindingCallContext.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/OperatorBindingCallContext.java
@@ -195,10 +195,11 @@ public final class OperatorBindingCallContext extends AbstractSqlCallContext {
Optional.ofNullable(inputChangelogModes)
.map(m -> m.get(tableArgCall.getInputIndex()))
.orElse(null);
+ final int inputIndex = tableArgCall.getInputIndex();
final List<int[]> upsertKeys =
- Optional.ofNullable(inputUpsertKeys)
- .map(m -> m.get(tableArgCall.getInputIndex()))
- .orElse(List.of());
+ inputUpsertKeys != null && inputIndex < inputUpsertKeys.size()
+ ? inputUpsertKeys.get(inputIndex)
+ : List.of();
return Optional.of(
OperatorBindingTableSemantics.create(
argumentDataTypes.get(pos),
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
index b3376b3a346..41c53979067 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
@@ -56,12 +56,12 @@ public class ToChangelogSemanticTests extends SemanticTestBase {
ToChangelogTestPrograms.INVALID_OP_MAPPING,
ToChangelogTestPrograms.OP_MAPPING_REFERENCES_UNSUPPORTED_KIND,
ToChangelogTestPrograms.DUPLICATE_ROW_KIND,
- ToChangelogTestPrograms.PRODUCES_FULL_DELETES_ON_APPEND_ONLY_INPUT,
- ToChangelogTestPrograms.ROW_SEM_PARTIAL_DELETES,
- ToChangelogTestPrograms.ROW_SEM_FORCE_FULL_DELETES,
- ToChangelogTestPrograms.SET_SEM_PARTIAL_DELETES,
- ToChangelogTestPrograms.SET_SEM_FULL_DELETES,
- ToChangelogTestPrograms.SET_SEM_FORCE_FULL_DELETES,
- ToChangelogTestPrograms.SET_SEM_FORCE_PARTIAL_DELETES);
+ ToChangelogTestPrograms.INVALID_PRODUCES_FULL_DELETES_FOR_APPEND_ONLY,
+ ToChangelogTestPrograms.RETRACT_PRODUCES_PARTIAL_DELETES,
+ ToChangelogTestPrograms.UPSERT_PRODUCES_FULL_DELETES,
+ ToChangelogTestPrograms.UPSERT_PRODUCES_PARTIAL_DELETES,
+ ToChangelogTestPrograms.RETRACT_PARTITION_BY_PRODUCES_PARTIAL_DELETES,
+ ToChangelogTestPrograms.RETRACT_PARTITION_BY_PRODUCES_FULL_DELETES,
+ ToChangelogTestPrograms.UPSERT_PARTITION_BY_PRODUCES_FULL_DELETES);
}
}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
index 5126014b9a3..909e2f4d1ef 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
@@ -187,8 +187,7 @@ public class ToChangelogTestPrograms {
public static final TableTestProgram UPSERT_PARTITION_BY =
TableTestProgram.of(
"to-changelog-upsert-partition-by",
- "PARTITION BY upsert key + mapping without UB skips ChangelogNormalize; "
- + "default produces_full_deletes=false nulls non-key columns on DELETE")
+ "PARTITION BY upsert key + mapping without UB skips ChangelogNormalize")
.setupTableSource(
SourceTestStep.newBuilder("t")
.addSchema(
@@ -207,7 +206,7 @@ public class ToChangelogTestPrograms {
"+I[Alice, C, 10]",
"+I[Bob, C, 20]",
"+I[Alice, C, 30]",
- "+I[Bob, D, null]")
+ "+I[Bob, D, 20]")
.build())
.runSql(
"INSERT INTO sink SELECT * FROM TO_CHANGELOG("
@@ -600,9 +599,9 @@ public class ToChangelogTestPrograms {
"Duplicate change operation: 'DELETE'")
.build();
- public static final TableTestProgram PRODUCES_FULL_DELETES_ON_APPEND_ONLY_INPUT =
+ public static final TableTestProgram INVALID_PRODUCES_FULL_DELETES_FOR_APPEND_ONLY =
TableTestProgram.of(
- "to-changelog-produces-full-deletes-on-append-only-input",
+ "to-changelog-invalid-produces-full-deletes-for-append-only",
"fails when produces_full_deletes=true on an input that never emits DELETE rows")
.setupTableSource(SIMPLE_SOURCE)
.runFailingSql(
@@ -613,15 +612,14 @@ public class ToChangelogTestPrograms {
"the input table only produces [INSERT] and never emits DELETE rows")
.build();
-
// --------------------------------------------------------------------------------------------
- // Row semantics x delete handling matrix
+ // Full vs partial deletes matrix (input kind x PARTITION BY x produces_full_deletes)
// --------------------------------------------------------------------------------------------
- public static final TableTestProgram ROW_SEM_PARTIAL_DELETES =
+ public static final TableTestProgram RETRACT_PRODUCES_PARTIAL_DELETES =
TableTestProgram.of(
- "to-changelog-row-sem-partial-deletes",
- "row semantics: produces_full_deletes=false skips ChangelogNormalize and a partial DELETE row from the input passes through unchanged")
+ "to-changelog-retract-produces-partial-deletes",
+ "retract input in row semantics with produces_full_deletes=false: skips ChangelogNormalize and the partial DELETE row from the input passes through unchanged")
.setupTableSource(
SourceTestStep.newBuilder("t")
.addSchema(
@@ -650,10 +648,10 @@ public class ToChangelogTestPrograms {
+ "produces_full_deletes => false)")
.build();
- public static final TableTestProgram ROW_SEM_FORCE_FULL_DELETES =
+ public static final TableTestProgram UPSERT_PRODUCES_FULL_DELETES =
TableTestProgram.of(
- "to-changelog-row-sem-force-full-deletes",
- "row semantics: produces_full_deletes=true forces ChangelogNormalize to materialize the full DELETE row from an upsert source emitting key-only deletes")
+ "to-changelog-upsert-produces-full-deletes",
+ "upsert input in row semantics with produces_full_deletes=true: ChangelogNormalize materializes the full DELETE row from a key-only delete")
.setupTableSource(
SourceTestStep.newBuilder("t")
.addSchema(
@@ -676,44 +674,37 @@ public class ToChangelogTestPrograms {
+ "produces_full_deletes => true)")
.build();
- // --------------------------------------------------------------------------------------------
- // Set semantics x delete handling matrix
- // --------------------------------------------------------------------------------------------
-
- public static final TableTestProgram SET_SEM_FORCE_PARTIAL_DELETES =
+ public static final TableTestProgram UPSERT_PRODUCES_PARTIAL_DELETES =
TableTestProgram.of(
- "to-changelog-set-sem-force-partial-deletes",
- "set semantics: produces_full_deletes=false nulls non-partition-key columns on DELETE even when the input row is fully populated")
+ "to-changelog-upsert-produces-partial-deletes",
+ "upsert input in row semantics with single-column upsert key + "
+ + "produces_full_deletes=false: DELETE preserves the upsert key "
+ + "column and nulls the rest without requiring PARTITION BY")
.setupTableSource(
SourceTestStep.newBuilder("t")
.addSchema(
"name STRING PRIMARY KEY NOT ENFORCED", "score BIGINT")
- .addMode(ChangelogMode.all())
+ .addMode(ChangelogMode.upsert())
.producedValues(
Row.ofKind(RowKind.INSERT, "Alice", 10L),
- Row.ofKind(RowKind.INSERT, "Bob", 20L),
- Row.ofKind(RowKind.UPDATE_AFTER, "Alice", 30L),
- Row.ofKind(RowKind.DELETE, "Bob", 20L))
+ Row.ofKind(RowKind.DELETE, "Alice", 10L))
.build())
.setupTableSink(
SinkTestStep.newBuilder("sink")
- .addSchema("name STRING", "op STRING", "score BIGINT")
+ .addSchema("op STRING", "name STRING", "score BIGINT")
.consumedValues(
- "+I[Alice, INSERT, 10]",
- "+I[Bob, INSERT, 20]",
- "+I[Alice, UPDATE_AFTER, 30]",
- "+I[Bob, DELETE, null]")
+ "+I[INSERT, Alice, 10]", "+I[DELETE, Alice, null]")
.build())
.runSql(
"INSERT INTO sink SELECT * FROM TO_CHANGELOG("
- + "input => TABLE t PARTITION BY name,"
+ + "input => TABLE t, "
+ "produces_full_deletes => false)")
.build();
- public static final TableTestProgram SET_SEM_PARTIAL_DELETES =
+ public static final TableTestProgram RETRACT_PARTITION_BY_PRODUCES_PARTIAL_DELETES =
TableTestProgram.of(
- "to-changelog-set-sem-partial-deletes",
- "set semantics: produces_full_deletes=false (default) lets a partial DELETE row from the input pass through with non-partition-key columns null")
+ "to-changelog-retract-partition-by-produces-partial-deletes",
+ "retract input in set semantics with produces_full_deletes=false: nulls non-partition-key columns on DELETE even when the input row is fully populated")
.setupTableSource(
SourceTestStep.newBuilder("t")
.addSchema(
@@ -721,23 +712,29 @@ public class ToChangelogTestPrograms {
.addMode(ChangelogMode.all())
.producedValues(
Row.ofKind(RowKind.INSERT, "Alice", 10L),
- Row.ofKind(RowKind.DELETE, "Alice", null))
+ Row.ofKind(RowKind.INSERT, "Bob", 20L),
+ Row.ofKind(RowKind.UPDATE_AFTER, "Alice", 30L),
+ Row.ofKind(RowKind.DELETE, "Bob", 20L))
.build())
.setupTableSink(
SinkTestStep.newBuilder("sink")
.addSchema("name STRING", "op STRING", "score BIGINT")
.consumedValues(
- "+I[Alice, INSERT, 10]", "+I[Alice, DELETE, null]")
+ "+I[Alice, INSERT, 10]",
+ "+I[Bob, INSERT, 20]",
+ "+I[Alice, UPDATE_AFTER, 30]",
+ "+I[Bob, DELETE, null]")
.build())
.runSql(
"INSERT INTO sink SELECT * FROM TO_CHANGELOG("
- + "input => TABLE t PARTITION BY name)")
+ + "input => TABLE t PARTITION BY name,"
+ + "produces_full_deletes => false)")
.build();
- public static final TableTestProgram SET_SEM_FULL_DELETES =
+ public static final TableTestProgram RETRACT_PARTITION_BY_PRODUCES_FULL_DELETES =
TableTestProgram.of(
- "to-changelog-set-sem-full-deletes",
- "set semantics: produces_full_deletes=true on an input that already emits full deletes is a no-op for the planner and the full DELETE row reaches the output")
+ "to-changelog-retract-partition-by-produces-full-deletes",
+ "retract input in set semantics with produces_full_deletes=true (default): the input row passes through unchanged, full DELETE pre-image reaches the output")
.setupTableSource(
SourceTestStep.newBuilder("t")
.addSchema(
@@ -759,10 +756,10 @@ public class ToChangelogTestPrograms {
+ "produces_full_deletes => true)")
.build();
- public static final TableTestProgram SET_SEM_FORCE_FULL_DELETES =
+ public static final TableTestProgram UPSERT_PARTITION_BY_PRODUCES_FULL_DELETES =
TableTestProgram.of(
- "to-changelog-set-sem-force-full-deletes",
- "set semantics: produces_full_deletes=true forces ChangelogNormalize to materialize the full DELETE row from an upsert source emitting key-only deletes")
+ "to-changelog-upsert-partition-by-produces-full-deletes",
+ "upsert input in set semantics with produces_full_deletes=true: ChangelogNormalize materializes the full DELETE row from a key-only delete")
.setupTableSource(
SourceTestStep.newBuilder("t")
.addSchema(
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
index d8874a2be16..66162e205b9 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
@@ -33,21 +33,31 @@ import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext;
import org.apache.flink.table.functions.TableSemantics;
import org.apache.flink.table.types.inference.CallContext;
import org.apache.flink.table.types.inference.strategies.ChangelogTypeStrategyUtils;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.utils.UpsertKeyUtils;
import org.apache.flink.types.ColumnList;
import org.apache.flink.types.RowKind;
import javax.annotation.Nullable;
import java.util.EnumMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_OP_MAPPING;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_PRODUCES_FULL_DELETES;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_TABLE;
+
/**
* Runtime implementation of {@link BuiltInFunctionDefinitions#TO_CHANGELOG}.
*
- * <p>Converts each input row into an INSERT-only output row with an operation code column. The
- * output schema is {@code [op_column, ...all_input_columns...]}.
+ * <p>Converts each input row into an INSERT-only output row with an operation code column. Output
+ * schema is {@code [op_column, ...projected_input_columns...]}. Partition columns are prepended by
+ * the framework outside this function and are not part of the projection.
*
* <p>Uses {@link JoinedRowData} to combine the op column with the full input row.
*/
@@ -65,51 +75,55 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
private final Map<RowKind, String> rawOpMap;
private final int[] outputIndices;
+ private final RowType inputRowType;
private final boolean producesFullDelete;
+ private final boolean[] upsertKeyColumn;
private transient Map<RowKind, StringData> opMap;
private transient GenericRowData opRow;
private transient JoinedRowData output;
private transient ProjectedRowData projectedOutput;
- private transient GenericRowData nullPayloadRow;
+ private transient GenericRowData partialDeletePayload;
+ private transient RowData.FieldGetter[] preservedFieldGetters;
@SuppressWarnings("unchecked")
public ToChangelogFunction(final SpecializedContext context) {
super(BuiltInFunctionDefinitions.TO_CHANGELOG, context);
final CallContext callContext = context.getCallContext();
// Table argument is guaranteed by the type strategy's validation phase.
- final TableSemantics tableSemantics = callContext.getTableSemantics(0).get();
+ final TableSemantics tableSemantics = callContext.getTableSemantics(ARG_TABLE).get();
final Map<String, String> opMapping =
- callContext.getArgumentValue(2, Map.class).orElse(null);
+ callContext.getArgumentValue(ARG_OP_MAPPING, Map.class).orElse(null);
this.rawOpMap = buildOpMap(opMapping);
if (opMapping != null) {
validateOpMap(this.rawOpMap, tableSemantics);
}
final boolean producesFullDeletesArg =
- callContext.getArgumentValue(3, Boolean.class).orElse(false);
- validateProducesFullDeletes(producesFullDeletesArg, tableSemantics);
+ callContext.getArgumentValue(ARG_PRODUCES_FULL_DELETES, Boolean.class).orElse(true);
+ final boolean isExplicit = !callContext.isArgumentNull(ARG_PRODUCES_FULL_DELETES);
+ validateProducesFullDeletes(producesFullDeletesArg, isExplicit, tableSemantics);
this.outputIndices = ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics);
- this.producesFullDelete = resolveProducesFullDelete(producesFullDeletesArg, tableSemantics);
+ this.inputRowType = (RowType) tableSemantics.dataType().getLogicalType();
+ this.producesFullDelete = producesFullDeletesArg;
+ this.upsertKeyColumn =
+ computeUpsertKeyColumn(
+ this.outputIndices,
+ UpsertKeyUtils.smallestKey(tableSemantics.upsertKeyColumns()));
}
- /**
- * Decides whether this function emits full DELETE rows (input passed through unchanged) or
- * partial DELETE rows (only identifying columns preserved, rest nulled).
- *
- * <p>The framework prepends partition-key columns to the output without consulting this
- * function, so in set semantics partition keys are preserved on DELETE rows for free. In row
- * semantics there is no key column to preserve, so the function passes the input through
- * unchanged regardless of {@code produces_full_deletes}.
- */
- private static boolean resolveProducesFullDelete(
- final boolean producesFullDeletesArg, final TableSemantics tableSemantics) {
- if (producesFullDeletesArg) {
- return true;
+ private static boolean[] computeUpsertKeyColumn(
+ final int[] outputIndices, final int[] upsertKey) {
+ final Set<Integer> keepInputIndices = new HashSet<>();
+ for (final int key : upsertKey) {
+ keepInputIndices.add(key);
}
- final boolean hasPartitionBy = tableSemantics.partitionByColumns().length > 0;
- return !hasPartitionBy;
+ final boolean[] mask = new boolean[outputIndices.length];
+ for (int i = 0; i < outputIndices.length; i++) {
+ mask[i] = keepInputIndices.contains(outputIndices[i]);
+ }
+ return mask;
}
@Override
@@ -120,7 +134,16 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
opRow = new GenericRowData(1);
output = new JoinedRowData();
projectedOutput = ProjectedRowData.from(outputIndices);
- nullPayloadRow = new GenericRowData(outputIndices.length);
+ partialDeletePayload = new GenericRowData(outputIndices.length);
+ preservedFieldGetters = new RowData.FieldGetter[outputIndices.length];
+ final List<LogicalType> inputFieldTypes = inputRowType.getChildren();
+ for (int i = 0; i < outputIndices.length; i++) {
+ if (upsertKeyColumn[i]) {
+ preservedFieldGetters[i] =
+ RowData.createFieldGetter(
+ inputFieldTypes.get(outputIndices[i]), outputIndices[i]);
+ }
+ }
}
/**
@@ -172,29 +195,48 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
}
/**
- * Rejects {@code produces_full_deletes=true} when the input changelog never emits DELETE rows.
+ * Validates an explicit {@code produces_full_deletes} argument against the input.
+ *
+ * <p>For {@code produces_full_deletes=true}, the input changelog must emit DELETE rows;
+ * otherwise the parameter is dead. For {@code produces_full_deletes=false}, the input must
+ * declare an upsert key or the call must use {@code PARTITION BY}; otherwise the function has
+ * no identifying columns to preserve when nulling the rest.
+ *
+ * <p>No validation runs when the argument is absent, since the default (full deletes) is safe
+ * for any input.
*
* <p>Lives here rather than in the input type strategy because {@link
- * TableSemantics#changelogMode()} returns empty during type inference and is only populated at
- * specialization time. The complementary check against the literal {@code op_mapping}
- * argument runs earlier in {@code ToChangelogTypeStrategy}.
+ * TableSemantics#changelogMode()} and {@link TableSemantics#upsertKeyColumns()} are only
+ * populated at specialization time.
*/
private static void validateProducesFullDeletes(
- final boolean producesFullDeletesArg, final TableSemantics tableSemantics) {
- if (!producesFullDeletesArg) {
+ final boolean producesFullDeletesArg,
+ final boolean isExplicit,
+ final TableSemantics tableSemantics) {
+ if (!isExplicit) {
return;
}
- final ChangelogMode inputMode = tableSemantics.changelogMode().orElse(null);
- if (inputMode == null) {
+ if (producesFullDeletesArg) {
+ final ChangelogMode inputMode = tableSemantics.changelogMode().orElse(null);
+ if (inputMode != null && !inputMode.contains(RowKind.DELETE)) {
+ throw new ValidationException(
+ String.format(
+ "Invalid 'produces_full_deletes' for TO_CHANGELOG: the input "
+ + "table only produces %s and never emits DELETE rows. "
+ + "Remove the 'produces_full_deletes' argument.",
+ inputMode.getContainedKinds()));
+ }
return;
}
- if (!inputMode.contains(RowKind.DELETE)) {
+ final boolean hasPartitionBy = tableSemantics.partitionByColumns().length > 0;
+ final boolean hasUpsertKey = !tableSemantics.upsertKeyColumns().isEmpty();
+ if (!hasPartitionBy && !hasUpsertKey) {
throw new ValidationException(
- String.format(
- "Invalid 'produces_full_deletes' for TO_CHANGELOG: the input table "
- + "only produces %s and never emits DELETE rows. Remove the "
- + "'produces_full_deletes' argument.",
- inputMode.getContainedKinds()));
+ "Invalid 'produces_full_deletes=false' for TO_CHANGELOG: the input has no "
+ + "upsert key and the call has no PARTITION BY, so the function has "
+ + "no identifying columns to preserve on DELETE rows. Remove the "
+ + "argument (the default emits full DELETE rows) or add a "
+ + "PARTITION BY.");
}
}
@@ -210,10 +252,28 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
}
opRow.setField(0, opCode);
- final RowData payload =
- (input.getRowKind() == RowKind.DELETE && !producesFullDelete)
- ? nullPayloadRow
- : projectedOutput.replaceRow(input);
+ final RowData payload;
+ if (input.getRowKind() == RowKind.DELETE && !producesFullDelete) {
+ payload = buildPartialDeletePayload(input);
+ } else {
+ payload = projectedOutput.replaceRow(input);
+ }
collect(output.replace(opRow, payload));
}
+
+ /**
+ * Builds the payload for a partial DELETE row: upsert-key columns are copied from the input,
+ * all other columns are emitted as {@code null}. Partition-key columns are not included here
+ * since the framework prepends them outside the function's projected output.
+ */
+ private RowData buildPartialDeletePayload(final RowData input) {
+ for (int i = 0; i < outputIndices.length; i++) {
+ if (upsertKeyColumn[i]) {
+ partialDeletePayload.setField(i, preservedFieldGetters[i].getFieldOrNull(input));
+ } else {
+ partialDeletePayload.setField(i, null);
+ }
+ }
+ return partialDeletePayload;
+ }
}
(flink) 01/06: [FLINK-39636][table] Add produces_full_deletes parameter to TO_CHANGELOG
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gustavodemorais pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit bca2c0a6170adbbf39cfaf4327ec368f003e4f40
Author: Ramin Gharib <ra...@gmail.com>
AuthorDate: Thu May 21 17:57:55 2026 +0200
[FLINK-39636][table] Add produces_full_deletes parameter to TO_CHANGELOG
Introduces the optional produces_full_deletes boolean argument on the built-in TO_CHANGELOG PTF. Controls whether DELETE rows downstream carry the full pre-image. Only the partition-key path is covered here; row-semantic partial deletes via the upsert key follow in a later commit.
---
.../docs/sql/reference/queries/changelog.md | 58 +++++-
flink-python/pyflink/table/table.py | 18 +-
.../apache/flink/table/api/PartitionedTable.java | 10 +-
.../java/org/apache/flink/table/api/Table.java | 10 +-
.../functions/BuiltInFunctionDefinitions.java | 40 ++--
.../table/types/inference/BuiltInCondition.java | 1 +
.../table/types/inference/TraitCondition.java | 11 ++
.../strategies/ToChangelogTypeStrategy.java | 16 +-
.../exec/stream/ToChangelogSemanticTests.java | 10 +-
.../nodes/exec/stream/ToChangelogTestPrograms.java | 216 ++++++++++++++++++++-
.../planner/plan/stream/sql/ToChangelogTest.java | 38 ++++
.../planner/plan/stream/sql/ToChangelogTest.xml | 66 +++++--
.../plan/to-changelog-retract-restore.json | 7 +-
.../runtime/functions/ptf/ToChangelogFunction.java | 72 ++++++-
14 files changed, 527 insertions(+), 46 deletions(-)
diff --git a/docs/content/docs/sql/reference/queries/changelog.md b/docs/content/docs/sql/reference/queries/changelog.md
index 78f61e1ce10..7a1598ee850 100644
--- a/docs/content/docs/sql/reference/queries/changelog.md
+++ b/docs/content/docs/sql/reference/queries/changelog.md
@@ -296,17 +296,19 @@ This is useful when you need to materialize changelog events into a downstream s
SELECT * FROM TO_CHANGELOG(
input => TABLE source_table [PARTITION BY key_col],
[op => DESCRIPTOR(op_column_name),]
- [op_mapping => MAP['INSERT', 'I', 'DELETE', 'D', ...]]
+ [op_mapping => MAP['INSERT', 'I', 'DELETE', 'D', ...],]
+ [produces_full_deletes => BOOLEAN]
)
```
### Parameters
-| Parameter | Required | Description |
-|:-------------|:---------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `input` | Yes | The input table. With `PARTITION BY`, rows with the same key are co-located and run in the same operator instance. Without `PARTITION BY`, each row is processed independently. Accepts insert-only, retract, and upsert tables. For upsert tables, the provided `PARTITION BY` key should match or be a subset of the upsert key of the subquery. |
-| `op` | No | A `DESCRIPTOR` with a single column name for the operation code column. Defaults to `op`. |
-| `op_mapping` | No | A `MAP<STRING, STRING>` mapping change operation names to custom output codes. Keys can contain comma-separated names to map multiple operations to the same code (e.g., `'INSERT, UPDATE_AFTER'`). When provided, only mapped operations are forwarded - unmapped events are dropped. Each change operation may appear at most once across all entries. |
+| Parameter | Required | Description |
+|:------------------------|:---------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `input` | Yes | The input table. With `PARTITION BY`, rows with the same key are co-located and run in the same operator instance. Without `PARTITION BY`, each row is processed independently. Accepts insert-only, retract, and upsert tables. For upsert tables, the provided `PARTITION BY` key should match or be a subset of the upsert key of the subquery. |
+| `op` | No | A `DESCRIPTOR` with a single column name for the operation code column. Defaults to `op`. |
+| `op_mapping` | No | A `MAP<STRING, STRING>` mapping change operation names to custom output codes. Keys can contain comma-separated names to map multiple operations to the same code (e.g., `'INSERT, UPDATE_AFTER'`). When provided, only mapped operations are forwarded - unmapped events are dropped. Each change operation may appear at most once across all entries. |
+| `produces_full_deletes` | No | A `BOOLEAN` literal that controls how DELETE rows are emitted. When `true`, the function requires fully-populated DELETE rows from the input. The planner inserts a `ChangelogNormalize` operator for upsert sources that emit key-only deletes, so downstream sees the full pre-image on DELETE. When `false` (default), no full-delete requirement is enforced. Partial DELETE rows from the input pass through unchanged. With `PARTITION BY` (set semantics), the [...]
#### Default op_mapping
@@ -397,6 +399,42 @@ SELECT * FROM TO_CHANGELOG(
-- UPDATE_BEFORE is dropped (not in the mapping)
```
+#### Delete handling
+
+The `produces_full_deletes` argument controls how DELETE rows are emitted and what the planner requires from the input. The behavior depends on whether `PARTITION BY` is used (set semantics) or not (row semantics).
+
+**With `produces_full_deletes => true`.** The planner requires the input to produce DELETE rows with all columns populated. For upsert sources, a `ChangelogNormalize` operator is inserted to materialize the full pre-image from state. The function then emits fully-populated DELETE rows downstream.
+
+```sql
+-- Upsert source: -D[id:5] (key-only).
+-- ChangelogNormalize materializes the full pre-image from state.
+-- Output: +I[op:'DELETE', id:5, name:'Alice']
+SELECT * FROM TO_CHANGELOG(
+ input => TABLE upsert_source,
+ produces_full_deletes => true
+)
+```
+
+**With `produces_full_deletes => false` (default).** The planner does not require fully-populated DELETE rows on the input. For upsert sources that emit key-only deletes (e.g. Kafka compacted topics), this avoids the stateful `ChangelogNormalize` operator that would otherwise materialize the full pre-image of each deleted row.
+
+In **row semantics** (no `PARTITION BY`) the function passes the input row through unchanged. If the source emits partial DELETE rows they remain partial downstream; if it emits full DELETE rows they remain full.
+
+```sql
+-- Source emits -D[id:5] (key-only).
+-- Output: +I[op:'DELETE', id:5, name:null]
+SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source)
+```
+
+In **set semantics** (`PARTITION BY`) the function additionally nulls every non-partition-key column on DELETE rows. This forces the output to carry only the partition key on DELETE even when the input row was fully populated, which matches the shape expected by upsert sinks and Kafka compacted topics.
+
+```sql
+-- Source emits -D[id:5, name:'Alice'] (full pre-image, e.g. from a retract source).
+-- Output: +I[id:5, op:'DELETE', name:null]
+SELECT * FROM TO_CHANGELOG(input => TABLE retract_source PARTITION BY id)
+```
+
+There is no way to derive a partial DELETE in row semantics when the input emits a full pre-image, since the function has no key column to preserve. Use `PARTITION BY` for that case.
+
#### Partitioning by a key
```sql
@@ -434,6 +472,14 @@ Table result = myTable.toChangelog(
map("INSERT, UPDATE_AFTER", "false", "DELETE", "true").asArgument("op_mapping")
);
+// Require fully-populated DELETE rows from the input (inserts a ChangelogNormalize for
+// upsert sources). When false (default), no full-delete requirement is enforced; in row
+// semantics the input passes through unchanged, in set semantics non-partition-key columns
+// are nulled on DELETE.
+Table result = myTable.toChangelog(
+ lit(true).asArgument("produces_full_deletes")
+);
+
// Set semantics: co-locate rows with the same key in the same parallel operator instance.
// Equivalent to PARTITION BY in SQL. The partition keys are prepended to the output columns.
Table result = myTable.partitionBy($("id")).toChangelog();
diff --git a/flink-python/pyflink/table/table.py b/flink-python/pyflink/table/table.py
index 583167fa478..bb06642db9e 100644
--- a/flink-python/pyflink/table/table.py
+++ b/flink-python/pyflink/table/table.py
@@ -1196,10 +1196,18 @@ class Table(object):
INSERT-only row with a string ``op`` column indicating the original operation
(INSERT, UPDATE_BEFORE, UPDATE_AFTER, DELETE).
+ The optional ``produces_full_deletes`` boolean controls how DELETE rows are
+ emitted. When ``True``, the planner inserts a ``ChangelogNormalize`` operator
+ for upsert sources that emit key-only deletes so the function emits fully
+ populated DELETE rows downstream. When ``False`` (default), no full-delete
+ requirement is enforced. In row semantics the input is passed through unchanged,
+ and in set semantics (``PARTITION BY``) non-partition-key columns are nulled on
+ DELETE rows.
+
Example:
::
- >>> from pyflink.table.expressions import descriptor, map_
+ >>> from pyflink.table.expressions import descriptor, map_, lit, col
>>> # Default: adds 'op' column with standard change operation names
>>> result = table.to_changelog()
>>> # Custom op column name and mapping
@@ -1213,8 +1221,14 @@ class Table(object):
... map_("INSERT, UPDATE_AFTER", "false",
... "DELETE", "true").as_argument("op_mapping")
... )
+ >>> # Require fully populated DELETE rows from the input. Inserts a
+ >>> # ChangelogNormalize for upsert sources.
+ >>> result = table.to_changelog(
+ ... lit(True).as_argument("produces_full_deletes")
+ ... )
- :param arguments: Optional named arguments for ``op`` and ``op_mapping``.
+ :param arguments: Optional named arguments for ``op``, ``op_mapping``, and
+ ``produces_full_deletes``.
:return: An append-only :class:`~pyflink.table.Table` with an ``op`` column prepended
to the input columns.
"""
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
index 51086f1edfe..074bdfdbaef 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/PartitionedTable.java
@@ -204,9 +204,17 @@ public interface PartitionedTable {
* descriptor("deleted").asArgument("op"),
* map("INSERT, UPDATE_AFTER", "false", "DELETE", "true").asArgument("op_mapping")
* );
+ *
+ * // Require fully-populated DELETE rows from the input (inserts a ChangelogNormalize for
+ * // upsert sources). When false (default), DELETE rows on upsert inputs may omit non-key
+ * // columns, which avoids the stateful normalization operator upstream.
+ * Table result = table
+ * .partitionBy($("id"))
+ * .toChangelog(lit(true).asArgument("produces_full_deletes"));
* }</pre>
*
- * @param arguments optional named arguments for {@code op} and {@code op_mapping}
+ * @param arguments optional named arguments for {@code op}, {@code op_mapping}, and {@code
+ * produces_full_deletes}
* @return an append-only {@link Table} with output schema {@code [partition_keys, op,
* non_partition_input_columns]}
* @see Table#toChangelog(Expression...)
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java
index 0ffb90c5fcc..de6ce9d3e5e 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/Table.java
@@ -1459,9 +1459,17 @@ public interface Table extends Explainable<Table>, Executable {
* descriptor("deleted").asArgument("op"),
* map("INSERT, UPDATE_AFTER", "false", "DELETE", "true").asArgument("op_mapping")
* );
+ *
+ * // Require fully-populated DELETE rows from the input (inserts a ChangelogNormalize for
+ * // upsert sources). When false (default), no full-delete requirement is enforced and partial
+ * // DELETE rows from the input pass through unchanged.
+ * Table result = table.toChangelog(
+ * lit(true).asArgument("produces_full_deletes")
+ * );
* }</pre>
*
- * @param arguments optional named arguments for {@code op} and {@code op_mapping}
+ * @param arguments optional named arguments for {@code op}, {@code op_mapping}, and {@code
+ * produces_full_deletes}
* @return an append-only {@link Table} with an {@code op} column prepended to the input columns
*/
Table toChangelog(Expression... arguments);
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
index 4d0c9b28dac..19b8219e98b 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
@@ -839,25 +839,35 @@ public final class BuiltInFunctionDefinitions {
"UPDATE_BEFORE"))))
.withConditionalTrait(
StaticArgumentTrait.REQUIRE_FULL_DELETE,
- TraitCondition.or(
- // op_mapping omitted: default mapping includes
- // DELETE.
- TraitCondition.not(
- TraitCondition.argIsPresent(
- "op_mapping")),
- TraitCondition.argMatches(
- "op_mapping",
- Map.class,
- mapping ->
- opMappingContainsKey(
- (Map<String, String>)
- mapping,
- "DELETE")))),
+ // Require full deletes only when the user explicitly
+ // asks for them via produces_full_deletes=TRUE *and*
+ // the active op_mapping includes DELETE. Otherwise the
+ // planner can skip ChangelogNormalize for upsert
+ // sources that emit key-only deletes.
+ TraitCondition.and(
+ TraitCondition.argIsEqualTo(
+ "produces_full_deletes", Boolean.TRUE),
+ TraitCondition.or(
+ TraitCondition.not(
+ TraitCondition.argIsPresent(
+ "op_mapping")),
+ TraitCondition.argMatches(
+ "op_mapping",
+ Map.class,
+ mapping ->
+ opMappingContainsKey(
+ (Map<
+ String,
+ String>)
+ mapping,
+ "DELETE"))))),
StaticArgument.scalar("op", DataTypes.DESCRIPTOR(), true),
StaticArgument.scalar(
"op_mapping",
DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()),
- true))
+ true),
+ StaticArgument.scalar(
+ "produces_full_deletes", DataTypes.BOOLEAN(), true))
.inputTypeStrategy(TO_CHANGELOG_INPUT_TYPE_STRATEGY)
.outputTypeStrategy(TO_CHANGELOG_OUTPUT_TYPE_STRATEGY)
.runtimeClass(
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/BuiltInCondition.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/BuiltInCondition.java
index ba87cc0559d..4c0149f6533 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/BuiltInCondition.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/BuiltInCondition.java
@@ -40,6 +40,7 @@ final class BuiltInCondition implements TraitCondition {
ARG_IS_PRESENT,
NOT,
OR,
+ AND,
ARG_MATCHES
}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/TraitCondition.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/TraitCondition.java
index 14734db11f4..8b86b37cc9f 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/TraitCondition.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/TraitCondition.java
@@ -84,6 +84,17 @@ public interface TraitCondition {
ctx -> left.test(ctx) || right.test(ctx));
}
+ /**
+ * True when both the {@code left} and the {@code right} {@link TraitCondition} evaluate to
+ * true.
+ */
+ static TraitCondition and(final TraitCondition left, final TraitCondition right) {
+ return new BuiltInCondition(
+ BuiltInCondition.Kind.AND,
+ List.of(left, right),
+ ctx -> left.test(ctx) && right.test(ctx));
+ }
+
/** True when the named scalar argument was provided by the caller. */
static TraitCondition argIsPresent(final String argName) {
return new BuiltInCondition(
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
index d977deab4a9..8ff8101dc5e 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
@@ -59,7 +59,7 @@ public final class ToChangelogTypeStrategy {
new InputTypeStrategy() {
@Override
public ArgumentCount getArgumentCount() {
- return ConstantArgumentCount.between(1, 3);
+ return ConstantArgumentCount.between(1, 4);
}
@Override
@@ -77,7 +77,12 @@ public final class ToChangelogTypeStrategy {
Signature.of(
Argument.of("input", "TABLE"),
Argument.of("op", "DESCRIPTOR"),
- Argument.of("op_mapping", "MAP<STRING, STRING>")));
+ Argument.of("op_mapping", "MAP<STRING, STRING>")),
+ Signature.of(
+ Argument.of("input", "TABLE"),
+ Argument.of("op", "DESCRIPTOR"),
+ Argument.of("op_mapping", "MAP<STRING, STRING>"),
+ Argument.of("produces_full_deletes", "BOOLEAN")));
}
};
@@ -144,6 +149,13 @@ public final class ToChangelogTypeStrategy {
}
}
+ final boolean hasProducesFullDeletesArg = !callContext.isArgumentNull(3);
+ if (hasProducesFullDeletesArg && !callContext.isArgumentLiteral(3)) {
+ return callContext.fail(
+ throwOnFailure,
+ "The 'produces_full_deletes' argument must be a constant BOOLEAN literal.");
+ }
+
return Optional.of(callContext.getArgumentDataTypes());
}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
index bfffa8bd67d..762e08b3538 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
@@ -55,6 +55,14 @@ public class ToChangelogSemanticTests extends SemanticTestBase {
ToChangelogTestPrograms.INVALID_DESCRIPTOR,
ToChangelogTestPrograms.INVALID_OP_MAPPING,
ToChangelogTestPrograms.OP_MAPPING_REFERENCES_UNSUPPORTED_KIND,
- ToChangelogTestPrograms.DUPLICATE_ROW_KIND);
+ ToChangelogTestPrograms.DUPLICATE_ROW_KIND,
+ ToChangelogTestPrograms.PRODUCES_FULL_DELETES_ON_APPEND_ONLY_INPUT,
+ ToChangelogTestPrograms.PRODUCES_FULL_DELETES_WITHOUT_DELETE_IN_OP_MAPPING,
+ ToChangelogTestPrograms.ROW_SEM_PARTIAL_DELETES,
+ ToChangelogTestPrograms.ROW_SEM_FORCE_FULL_DELETES,
+ ToChangelogTestPrograms.SET_SEM_PARTIAL_DELETES,
+ ToChangelogTestPrograms.SET_SEM_FULL_DELETES,
+ ToChangelogTestPrograms.SET_SEM_FORCE_FULL_DELETES,
+ ToChangelogTestPrograms.SET_SEM_FORCE_PARTIAL_DELETES);
}
}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
index 250ad1f1d8f..f04d30d1d99 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
@@ -157,7 +157,7 @@ public class ToChangelogTestPrograms {
public static final TableTestProgram UPSERT =
TableTestProgram.of(
"to-changelog-upsert-input",
- "upsert input gets ChangelogNormalize for UPDATE_BEFORE and full deletes")
+ "upsert input in row semantics gets ChangelogNormalize for UPDATE_BEFORE and emits full deletes")
.setupTableSource(
SourceTestStep.newBuilder("t")
.addSchema(
@@ -187,7 +187,8 @@ public class ToChangelogTestPrograms {
public static final TableTestProgram UPSERT_PARTITION_BY =
TableTestProgram.of(
"to-changelog-upsert-partition-by",
- "PARTITION BY upsert key + mapping without UB skips ChangelogNormalize")
+ "PARTITION BY upsert key + mapping without UB skips ChangelogNormalize; "
+ + "default produces_full_deletes=false nulls non-key columns on DELETE")
.setupTableSource(
SourceTestStep.newBuilder("t")
.addSchema(
@@ -206,7 +207,7 @@ public class ToChangelogTestPrograms {
"+I[Alice, C, 10]",
"+I[Bob, C, 20]",
"+I[Alice, C, 30]",
- "+I[Bob, D, 20]")
+ "+I[Bob, D, null]")
.build())
.runSql(
"INSERT INTO sink SELECT * FROM TO_CHANGELOG("
@@ -598,4 +599,213 @@ public class ToChangelogTestPrograms {
ValidationException.class,
"Duplicate change operation: 'DELETE'")
.build();
+
+ public static final TableTestProgram PRODUCES_FULL_DELETES_ON_APPEND_ONLY_INPUT =
+ TableTestProgram.of(
+ "to-changelog-produces-full-deletes-on-append-only-input",
+ "fails when produces_full_deletes=true on an input that never emits DELETE rows")
+ .setupTableSource(SIMPLE_SOURCE)
+ .runFailingSql(
+ "SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE t, "
+ + "produces_full_deletes => true)",
+ ValidationException.class,
+ "the input table only produces [INSERT] and never emits DELETE rows")
+ .build();
+
+ public static final TableTestProgram PRODUCES_FULL_DELETES_WITHOUT_DELETE_IN_OP_MAPPING =
+ TableTestProgram.of(
+ "to-changelog-produces-full-deletes-without-delete-in-op-mapping",
+ "fails when produces_full_deletes=true but the active op_mapping strips DELETE")
+ .setupTableSource(
+ SourceTestStep.newBuilder("t")
+ .addSchema(
+ "name STRING PRIMARY KEY NOT ENFORCED", "score BIGINT")
+ .addMode(ChangelogMode.all())
+ .producedValues(Row.ofKind(RowKind.INSERT, "Alice", 10L))
+ .build())
+ .runFailingSql(
+ "SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE t, "
+ + "op_mapping => MAP['INSERT, UPDATE_AFTER', 'X'], "
+ + "produces_full_deletes => true)",
+ ValidationException.class,
+ "the active 'op_mapping' does not map DELETE rows")
+ .build();
+
+ // --------------------------------------------------------------------------------------------
+ // Row semantics x delete handling matrix
+ // --------------------------------------------------------------------------------------------
+
+ public static final TableTestProgram ROW_SEM_PARTIAL_DELETES =
+ TableTestProgram.of(
+ "to-changelog-row-sem-partial-deletes",
+ "row semantics: produces_full_deletes=false skips ChangelogNormalize and a partial DELETE row from the input passes through unchanged")
+ .setupTableSource(
+ SourceTestStep.newBuilder("t")
+ .addSchema(
+ "name STRING PRIMARY KEY NOT ENFORCED", "score BIGINT")
+ .addMode(ChangelogMode.all())
+ .producedValues(
+ Row.ofKind(RowKind.INSERT, "Alice", 10L),
+ Row.ofKind(RowKind.INSERT, "Bob", 20L),
+ Row.ofKind(RowKind.UPDATE_BEFORE, "Alice", 10L),
+ Row.ofKind(RowKind.UPDATE_AFTER, "Alice", 30L),
+ Row.ofKind(RowKind.DELETE, "Bob", null))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema("op STRING", "name STRING", "score BIGINT")
+ .consumedValues(
+ "+I[INSERT, Alice, 10]",
+ "+I[INSERT, Bob, 20]",
+ "+I[UPDATE_BEFORE, Alice, 10]",
+ "+I[UPDATE_AFTER, Alice, 30]",
+ "+I[DELETE, Bob, null]")
+ .build())
+ .runSql(
+ "INSERT INTO sink SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE t, "
+ + "produces_full_deletes => false)")
+ .build();
+
+ public static final TableTestProgram ROW_SEM_FORCE_FULL_DELETES =
+ TableTestProgram.of(
+ "to-changelog-row-sem-force-full-deletes",
+ "row semantics: produces_full_deletes=true forces ChangelogNormalize to materialize the full DELETE row from an upsert source emitting key-only deletes")
+ .setupTableSource(
+ SourceTestStep.newBuilder("t")
+ .addSchema(
+ "name STRING PRIMARY KEY NOT ENFORCED", "score BIGINT")
+ .addMode(ChangelogMode.upsert())
+ .producedValues(
+ Row.ofKind(RowKind.INSERT, "Alice", 10L),
+ // Key-only delete: ChangelogNormalize fills the row.
+ Row.ofKind(RowKind.DELETE, "Alice", null))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema("op STRING", "name STRING", "score BIGINT")
+ .consumedValues(
+ "+I[INSERT, Alice, 10]", "+I[DELETE, Alice, 10]")
+ .build())
+ .runSql(
+ "INSERT INTO sink SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE t, "
+ + "produces_full_deletes => true)")
+ .build();
+
+ // --------------------------------------------------------------------------------------------
+ // Set semantics x delete handling matrix
+ // --------------------------------------------------------------------------------------------
+
+ public static final TableTestProgram SET_SEM_FORCE_PARTIAL_DELETES =
+ TableTestProgram.of(
+ "to-changelog-set-sem-force-partial-deletes",
+ "set semantics: produces_full_deletes=false nulls non-partition-key columns on DELETE even when the input row is fully populated")
+ .setupTableSource(
+ SourceTestStep.newBuilder("t")
+ .addSchema(
+ "name STRING PRIMARY KEY NOT ENFORCED", "score BIGINT")
+ .addMode(ChangelogMode.all())
+ .producedValues(
+ Row.ofKind(RowKind.INSERT, "Alice", 10L),
+ Row.ofKind(RowKind.INSERT, "Bob", 20L),
+ Row.ofKind(RowKind.UPDATE_AFTER, "Alice", 30L),
+ Row.ofKind(RowKind.DELETE, "Bob", 20L))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema("name STRING", "op STRING", "score BIGINT")
+ .consumedValues(
+ "+I[Alice, INSERT, 10]",
+ "+I[Bob, INSERT, 20]",
+ "+I[Alice, UPDATE_AFTER, 30]",
+ "+I[Bob, DELETE, null]")
+ .build())
+ .runSql(
+ "INSERT INTO sink SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE t PARTITION BY name,"
+ + "produces_full_deletes => false)")
+ .build();
+
+ public static final TableTestProgram SET_SEM_PARTIAL_DELETES =
+ TableTestProgram.of(
+ "to-changelog-set-sem-partial-deletes",
+ "set semantics: produces_full_deletes=false (default) lets a partial DELETE row from the input pass through with non-partition-key columns null")
+ .setupTableSource(
+ SourceTestStep.newBuilder("t")
+ .addSchema(
+ "name STRING PRIMARY KEY NOT ENFORCED", "score BIGINT")
+ .addMode(ChangelogMode.all())
+ .producedValues(
+ Row.ofKind(RowKind.INSERT, "Alice", 10L),
+ Row.ofKind(RowKind.DELETE, "Alice", null))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema("name STRING", "op STRING", "score BIGINT")
+ .consumedValues(
+ "+I[Alice, INSERT, 10]", "+I[Alice, DELETE, null]")
+ .build())
+ .runSql(
+ "INSERT INTO sink SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE t PARTITION BY name)")
+ .build();
+
+ public static final TableTestProgram SET_SEM_FULL_DELETES =
+ TableTestProgram.of(
+ "to-changelog-set-sem-full-deletes",
+ "set semantics: produces_full_deletes=true on an input that already emits full deletes is a no-op for the planner and the full DELETE row reaches the output")
+ .setupTableSource(
+ SourceTestStep.newBuilder("t")
+ .addSchema(
+ "name STRING PRIMARY KEY NOT ENFORCED", "score BIGINT")
+ .addMode(ChangelogMode.all())
+ .producedValues(
+ Row.ofKind(RowKind.INSERT, "Alice", 10L),
+ Row.ofKind(RowKind.DELETE, "Alice", 10L))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema("name STRING", "op STRING", "score BIGINT")
+ .consumedValues(
+ "+I[Alice, INSERT, 10]", "+I[Alice, DELETE, 10]")
+ .build())
+ .runSql(
+ "INSERT INTO sink SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE t PARTITION BY name, "
+ + "produces_full_deletes => true)")
+ .build();
+
+ public static final TableTestProgram SET_SEM_FORCE_FULL_DELETES =
+ TableTestProgram.of(
+ "to-changelog-set-sem-force-full-deletes",
+ "set semantics: produces_full_deletes=true forces ChangelogNormalize to materialize the full DELETE row from an upsert source emitting key-only deletes")
+ .setupTableSource(
+ SourceTestStep.newBuilder("t")
+ .addSchema(
+ "name STRING PRIMARY KEY NOT ENFORCED", "score BIGINT")
+ .addMode(ChangelogMode.upsert())
+ .producedValues(
+ Row.ofKind(RowKind.INSERT, "Alice", 10L),
+ Row.ofKind(RowKind.INSERT, "Bob", 20L),
+ Row.ofKind(RowKind.UPDATE_AFTER, "Alice", 30L),
+ Row.ofKind(RowKind.DELETE, "Bob", null))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema("name STRING", "op STRING", "score BIGINT")
+ .consumedValues(
+ "+I[Alice, INSERT, 10]",
+ "+I[Bob, INSERT, 20]",
+ "+I[Alice, UPDATE_BEFORE, 10]",
+ "+I[Alice, UPDATE_AFTER, 30]",
+ "+I[Bob, DELETE, 20]")
+ .build())
+ .runSql(
+ "INSERT INTO sink SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE t PARTITION BY name, "
+ + "produces_full_deletes => true)")
+ .build();
}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ToChangelogTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ToChangelogTest.java
index 7f6f93849c2..1f7347b027e 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ToChangelogTest.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ToChangelogTest.java
@@ -144,4 +144,42 @@ public class ToChangelogTest extends TableTestBase {
+ "op_mapping => MAP['INSERT,UPDATE_AFTER', 'C'])",
CHANGELOG_MODE);
}
+
+ @Test
+ void testUpsertSourceProducesFullDeletes() {
+ util.tableEnv()
+ .executeSql(
+ "CREATE TABLE upsert_source ("
+ + " id INT,"
+ + " name STRING,"
+ + " PRIMARY KEY (id) NOT ENFORCED"
+ + ") WITH ("
+ + " 'connector' = 'values',"
+ + " 'changelog-mode' = 'I,UA,D'"
+ + ")");
+ util.verifyRelPlan(
+ "SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE upsert_source, "
+ + "produces_full_deletes => true)",
+ CHANGELOG_MODE);
+ }
+
+ @Test
+ void testUpsertSourceKeyOnlyDeletes() {
+ util.tableEnv()
+ .executeSql(
+ "CREATE TABLE upsert_source ("
+ + " id INT,"
+ + " name STRING,"
+ + " PRIMARY KEY (id) NOT ENFORCED"
+ + ") WITH ("
+ + " 'connector' = 'values',"
+ + " 'changelog-mode' = 'I,UA,D'"
+ + ")");
+ util.verifyRelPlan(
+ "SELECT * FROM TO_CHANGELOG("
+ + "input => TABLE upsert_source, "
+ + "produces_full_deletes => false)",
+ CHANGELOG_MODE);
+ }
}
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/ToChangelogTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/ToChangelogTest.xml
index 7cea1058e5c..174addbf418 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/ToChangelogTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/ToChangelogTest.xml
@@ -23,14 +23,14 @@ limitations under the License.
<Resource name="ast">
<![CDATA[
LogicalProject(op=[$0], id=[$1], name=[$2])
-+- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)])
++- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)])
+- LogicalProject(id=[$0], name=[$1])
+- LogicalTableScan(table=[[default_catalog, default_database, insert_only_source]])
]]>
</Resource>
<Resource name="optimized rel plan">
<![CDATA[
-ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[null], select=[op,id,name], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)], changelogMode=[I])
+ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[null], select=[op,id,name], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)], changelogMode=[I])
+- TableSourceScan(table=[[default_catalog, default_database, insert_only_source]], fields=[id, name], changelogMode=[I])
]]>
</Resource>
@@ -42,14 +42,14 @@ ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), D
<Resource name="ast">
<![CDATA[
LogicalProject(id=[$0], op=[$1], name=[$2])
-+- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)])
++- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)])
+- LogicalProject(id=[$0], name=[$1])
+- LogicalTableScan(table=[[default_catalog, default_database, retract_source]])
]]>
</Resource>
<Resource name="optimized rel plan">
<![CDATA[
-ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[TO_CHANGELOG], select=[id,op,name], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)], changelogMode=[I])
+ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[TO_CHANGELOG], select=[id,op,name], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)], changelogMode=[I])
+- Exchange(distribution=[hash[id]], changelogMode=[I,UB,UA,D])
+- TableSourceScan(table=[[default_catalog, default_database, retract_source]], fields=[id, name], changelogMode=[I,UB,UA,D])
]]>
@@ -62,14 +62,14 @@ ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DEFAUL
<Resource name="ast">
<![CDATA[
LogicalProject(op=[$0], id=[$1], name=[$2])
-+- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)])
++- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)])
+- LogicalProject(id=[$0], name=[$1])
+- LogicalTableScan(table=[[default_catalog, default_database, retract_source]])
]]>
</Resource>
<Resource name="optimized rel plan">
<![CDATA[
-ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[null], select=[op,id,name], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)], changelogMode=[I])
+ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[null], select=[op,id,name], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)], changelogMode=[I])
+- TableSourceScan(table=[[default_catalog, default_database, retract_source]], fields=[id, name], changelogMode=[I,UB,UA,D])
]]>
</Resource>
@@ -81,14 +81,14 @@ ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), D
<Resource name="ast">
<![CDATA[
LogicalProject(id=[$0], op=[$1], name=[$2])
-+- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRIPTOR(_UTF-16LE'op'), MAP(_UTF-16LE'INSERT,UPDATE_AFTER':VARCHAR(19) CHARACTER SET "UTF-16LE", _UTF-16LE'C', _UTF-16LE'DELETE':VARCHAR(19) CHARACTER SET "UTF-16LE", _UTF-16LE'D'), DEFAULT(), DEFAULT())], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)])
++- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRIPTOR(_UTF-16LE'op'), MAP(_UTF-16LE'INSERT,UPDATE_AFTER':VARCHAR(19) CHARACTER SET "UTF-16LE", _UTF-16LE'C', _UTF-16LE'DELETE':VARCHAR(19) CHARACTER SET "UTF-16LE", _UTF-16LE'D'), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)])
+- LogicalProject(id=[$0], name=[$1])
+- LogicalTableScan(table=[[default_catalog, default_database, upsert_source]])
]]>
</Resource>
<Resource name="optimized rel plan">
<![CDATA[
-ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRIPTOR(_UTF-16LE'op'), MAP(_UTF-16LE'INSERT,UPDATE_AFTER':VARCHAR(19) CHARACTER SET "UTF-16LE", _UTF-16LE'C', _UTF-16LE'DELETE':VARCHAR(19) CHARACTER SET "UTF-16LE", _UTF-16LE'D'), DEFAULT(), DEFAULT())], uid=[TO_CHANGELOG], select=[id,op,name], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)], changelogMode=[I])
+ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRIPTOR(_UTF-16LE'op'), MAP(_UTF-16LE'INSERT,UPDATE_AFTER':VARCHAR(19) CHARACTER SET "UTF-16LE", _UTF-16LE'C', _UTF-16LE'DELETE':VARCHAR(19) CHARACTER SET "UTF-16LE", _UTF-16LE'D'), DEFAULT(), DEFAULT(), DEFAULT())], uid=[TO_CHANGELOG], select=[id,op,name], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)], changelogMode=[I])
+- Exchange(distribution=[hash[id]], changelogMode=[I,UA,D])
+- TableSourceScan(table=[[default_catalog, default_database, upsert_source]], fields=[id, name], changelogMode=[I,UA,D])
]]>
@@ -101,14 +101,14 @@ ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRI
<Resource name="ast">
<![CDATA[
LogicalProject(id=[$0], op=[$1], name=[$2])
-+- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRIPTOR(_UTF-16LE'op'), MAP(_UTF-16LE'INSERT,UPDATE_AFTER', _UTF-16LE'C'), DEFAULT(), DEFAULT())], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)])
++- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRIPTOR(_UTF-16LE'op'), MAP(_UTF-16LE'INSERT,UPDATE_AFTER', _UTF-16LE'C'), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)])
+- LogicalProject(id=[$0], name=[$1])
+- LogicalTableScan(table=[[default_catalog, default_database, upsert_source]])
]]>
</Resource>
<Resource name="optimized rel plan">
<![CDATA[
-ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRIPTOR(_UTF-16LE'op'), MAP(_UTF-16LE'INSERT,UPDATE_AFTER', _UTF-16LE'C'), DEFAULT(), DEFAULT())], uid=[TO_CHANGELOG], select=[id,op,name], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)], changelogMode=[I])
+ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRIPTOR(_UTF-16LE'op'), MAP(_UTF-16LE'INSERT,UPDATE_AFTER', _UTF-16LE'C'), DEFAULT(), DEFAULT(), DEFAULT())], uid=[TO_CHANGELOG], select=[id,op,name], rowType=[RecordType(INTEGER id, VARCHAR(2147483647) op, VARCHAR(2147483647) name)], changelogMode=[I])
+- Exchange(distribution=[hash[id]], changelogMode=[I,UA,D])
+- TableSourceScan(table=[[default_catalog, default_database, upsert_source]], fields=[id, name], changelogMode=[I,UA,D])
]]>
@@ -121,14 +121,56 @@ ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0) PARTITION BY($0), DESCRI
<Resource name="ast">
<![CDATA[
LogicalProject(op=[$0], id=[$1], name=[$2])
-+- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)])
++- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)])
+- LogicalProject(id=[$0], name=[$1])
+- LogicalTableScan(table=[[default_catalog, default_database, upsert_source]])
]]>
</Resource>
<Resource name="optimized rel plan">
<![CDATA[
-ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[null], select=[op,id,name], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)], changelogMode=[I])
+ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[null], select=[op,id,name], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)], changelogMode=[I])
++- ChangelogNormalize(key=[id], changelogMode=[I,UB,UA,D])
+ +- Exchange(distribution=[hash[id]], changelogMode=[I,UA,D])
+ +- TableSourceScan(table=[[default_catalog, default_database, upsert_source]], fields=[id, name], changelogMode=[I,UA,D])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testUpsertSourceKeyOnlyDeletes">
+ <Resource name="sql">
+ <![CDATA[SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source, produces_full_deletes => false)]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(op=[$0], id=[$1], name=[$2])
++- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), false, DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)])
+ +- LogicalProject(id=[$0], name=[$1])
+ +- LogicalTableScan(table=[[default_catalog, default_database, upsert_source]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), false, DEFAULT(), DEFAULT())], uid=[null], select=[op,id,name], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)], changelogMode=[I])
++- ChangelogNormalize(key=[id], changelogMode=[I,UB,UA,D])
+ +- Exchange(distribution=[hash[id]], changelogMode=[I,UA,D])
+ +- TableSourceScan(table=[[default_catalog, default_database, upsert_source]], fields=[id, name], changelogMode=[I,UA,D])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testUpsertSourceProducesFullDeletes">
+ <Resource name="sql">
+ <![CDATA[SELECT * FROM TO_CHANGELOG(input => TABLE upsert_source, produces_full_deletes => true)]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(op=[$0], id=[$1], name=[$2])
++- LogicalTableFunctionScan(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), true, DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)])
+ +- LogicalProject(id=[$0], name=[$1])
+ +- LogicalTableScan(table=[[default_catalog, default_database, upsert_source]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), true, DEFAULT(), DEFAULT())], uid=[null], select=[op,id,name], rowType=[RecordType(VARCHAR(2147483647) op, INTEGER id, VARCHAR(2147483647) name)], changelogMode=[I])
+- ChangelogNormalize(key=[id], changelogMode=[I,UB,UA,D])
+- Exchange(distribution=[hash[id]], changelogMode=[I,UA,D])
+- TableSourceScan(table=[[default_catalog, default_database, upsert_source]], fields=[id, name], changelogMode=[I,UA,D])
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/to-changelog-retract-restore/plan/to-changelog-retract-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/to-changelog-retract-restore/plan/to-changelog-retract-restore.json
index 959c9473118..ea961c665ca 100644
--- a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/to-changelog-retract-restore/plan/to-changelog-retract-restore.json
+++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/to-changelog-retract-restore/plan/to-changelog-retract-restore.json
@@ -37,7 +37,7 @@
"priority" : 0
} ],
"outputType" : "ROW<`op` VARCHAR(2147483647), `name` VARCHAR(2147483647) NOT NULL, `score` BIGINT>",
- "description" : "ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[null], select=[op,name,score], rowType=[RecordType(VARCHAR(2147483647) op, VARCHAR(2147483647) name, BIGINT score)])",
+ "description" : "ProcessTableFunction(invocation=[TO_CHANGELOG(TABLE(#0), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT(), DEFAULT())], uid=[null], select=[op,name,score], rowType=[RecordType(VARCHAR(2147483647) op, VARCHAR(2147483647) name, BIGINT score)])",
"uid" : null,
"functionCall" : {
"kind" : "CALL",
@@ -59,6 +59,11 @@
"syntax" : "SPECIAL",
"internalName" : "$DEFAULT$1",
"type" : "MAP<VARCHAR(2147483647), VARCHAR(2147483647)>"
+ }, {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$DEFAULT$1",
+ "type" : "BOOLEAN"
}, {
"kind" : "CALL",
"syntax" : "SPECIAL",
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
index 0b6c6ca55b8..0110475ef8d 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
@@ -65,11 +65,13 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
private final Map<RowKind, String> rawOpMap;
private final int[] outputIndices;
+ private final boolean producesFullDelete;
private transient Map<RowKind, StringData> opMap;
private transient GenericRowData opRow;
private transient JoinedRowData output;
private transient ProjectedRowData projectedOutput;
+ private transient GenericRowData nullPayloadRow;
@SuppressWarnings("unchecked")
public ToChangelogFunction(final SpecializedContext context) {
@@ -84,7 +86,30 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
if (opMapping != null) {
validateOpMap(this.rawOpMap, tableSemantics);
}
+ final boolean producesFullDeletesArg =
+ callContext.getArgumentValue(3, Boolean.class).orElse(false);
+ validateProducesFullDeletes(producesFullDeletesArg, this.rawOpMap, tableSemantics);
+
this.outputIndices = ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics);
+ this.producesFullDelete = resolveProducesFullDelete(producesFullDeletesArg, tableSemantics);
+ }
+
+ /**
+ * Decides whether this function emits full DELETE rows (input passed through unchanged) or
+ * partial DELETE rows (only identifying columns preserved, rest nulled).
+ *
+ * <p>The framework prepends partition-key columns to the output without consulting this
+ * function, so in set semantics partition keys are preserved on DELETE rows for free. In row
+ * semantics there is no key column to preserve, so the function passes the input through
+ * unchanged regardless of {@code produces_full_deletes}.
+ */
+ private static boolean resolveProducesFullDelete(
+ final boolean producesFullDeletesArg, final TableSemantics tableSemantics) {
+ if (producesFullDeletesArg) {
+ return true;
+ }
+ final boolean hasPartitionBy = tableSemantics.partitionByColumns().length > 0;
+ return !hasPartitionBy;
}
@Override
@@ -95,6 +120,7 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
opRow = new GenericRowData(1);
output = new JoinedRowData();
projectedOutput = ProjectedRowData.from(outputIndices);
+ nullPayloadRow = new GenericRowData(outputIndices.length);
}
/**
@@ -145,17 +171,59 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
}
}
+ /**
+ * Rejects {@code produces_full_deletes=true} when the input changelog cannot produce DELETE
+ * rows (either the input mode does not contain DELETE, or the active {@code op_mapping} strips
+ * it). The parameter is then dead and likely a user mistake.
+ *
+ * <p>Lives here rather than in the input type strategy because {@link
+ * TableSemantics#changelogMode()} returns empty during type inference and is only populated at
+ * specialization time, which is when this constructor runs.
+ */
+ private static void validateProducesFullDeletes(
+ final boolean producesFullDeletesArg,
+ final Map<RowKind, String> mapping,
+ final TableSemantics tableSemantics) {
+ if (!producesFullDeletesArg) {
+ return;
+ }
+ final ChangelogMode inputMode = tableSemantics.changelogMode().orElse(null);
+ if (inputMode == null) {
+ return;
+ }
+ if (!inputMode.contains(RowKind.DELETE)) {
+ throw new ValidationException(
+ String.format(
+ "Invalid 'produces_full_deletes' for TO_CHANGELOG: the input table "
+ + "only produces %s and never emits DELETE rows. Remove the "
+ + "'produces_full_deletes' argument.",
+ inputMode.getContainedKinds()));
+ }
+ if (!mapping.containsKey(RowKind.DELETE)) {
+ throw new ValidationException(
+ "Invalid 'produces_full_deletes' for TO_CHANGELOG: the active 'op_mapping' "
+ + "does not map DELETE rows, so no DELETE rows are emitted. Remove "
+ + "the 'produces_full_deletes' argument or add a DELETE entry to "
+ + "'op_mapping'.");
+ }
+ }
+
public void eval(
final Context ctx,
final RowData input,
@Nullable final ColumnList op,
- @Nullable final MapData opMapping) {
+ @Nullable final MapData opMapping,
+ @Nullable final Boolean producesFullDeletes) {
final StringData opCode = opMap.get(input.getRowKind());
if (opCode == null) {
return;
}
opRow.setField(0, opCode);
- collect(output.replace(opRow, projectedOutput.replaceRow(input)));
+ final RowData payload =
+ (input.getRowKind() == RowKind.DELETE && !producesFullDelete)
+ ? nullPayloadRow
+ : projectedOutput.replaceRow(input);
+ collect(output.replace(opRow, payload));
}
}
(flink) 03/06: [FLINK-39735][table] Expose input upsert key on TableSemantics
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gustavodemorais pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit d763453e045ad445c7d5a8382559d34a9429a1e9
Author: Ramin Gharib <ra...@gmail.com>
AuthorDate: Tue May 26 10:56:29 2026 +0200
[FLINK-39735][table] Expose input upsert key on TableSemantics
Adds upsertKeyColumns() to TableSemantics so ProcessTableFunctions can read the planner-derived upsert key candidates of each table input at specialization time. The planner exposes all candidates; picking one is the function's responsibility. UpsertKeyUtils provides a stable smallestKey helper for that choice.
Co-Authored-By: Jubin Soni <ju...@gmail.com>
---
.../resolver/rules/ResolveCallByArgumentsRule.java | 5 ++
.../flink/table/functions/TableSemantics.java | 24 ++++++++
.../apache/flink/table/utils/UpsertKeyUtils.java | 67 ++++++++++++++++++++++
.../types/inference/utils/TableSemanticsMock.java | 25 ++++++++
.../functions/bridging/BridgingSqlFunction.java | 19 +++++-
.../inference/CallBindingCallContext.java | 6 ++
.../inference/OperatorBindingCallContext.java | 47 +++++++++++++--
.../stream/StreamExecProcessTableFunction.java | 32 +++++++++--
.../stream/StreamPhysicalProcessTableFunction.java | 19 +++++-
.../table/planner/plan/utils/UpsertKeyUtil.java | 23 +++-----
.../codegen/ProcessTableRunnerGenerator.scala | 10 +++-
.../plan/to-changelog-retract-restore.json | 3 +-
.../operators/process/RuntimeTableSemantics.java | 11 +++-
...essSetTableOperatorInterruptibleTimersTest.java | 3 +-
.../functions/TestHarnessTableSemantics.java | 14 +++++
15 files changed, 273 insertions(+), 35 deletions(-)
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/resolver/rules/ResolveCallByArgumentsRule.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/resolver/rules/ResolveCallByArgumentsRule.java
index be8e578deea..5e4a02cb9fb 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/resolver/rules/ResolveCallByArgumentsRule.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/resolver/rules/ResolveCallByArgumentsRule.java
@@ -781,6 +781,11 @@ final class ResolveCallByArgumentsRule implements ResolverRule {
return Optional.empty();
}
+ @Override
+ public List<int[]> upsertKeyColumns() {
+ return Collections.emptyList();
+ }
+
private PartitionQueryOperation findPartitionOperation(QueryOperation op) {
if (op instanceof PartitionQueryOperation) {
return (PartitionQueryOperation) op;
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/TableSemantics.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/TableSemantics.java
index f63566befce..3d17401c5ef 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/TableSemantics.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/TableSemantics.java
@@ -24,6 +24,7 @@ import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.connector.ChangelogMode;
import org.apache.flink.table.types.DataType;
+import java.util.List;
import java.util.Optional;
/**
@@ -128,6 +129,29 @@ public interface TableSemantics {
*/
Optional<ChangelogMode> changelogMode();
+ /**
+ * Upsert key candidates derived from the passed table's metadata.
+ *
+ * <p>Returns a list of 0-based column index arrays that uniquely identify a row for upsert
+ * semantics. This is distinct from {@link #partitionByColumns()}: partition keys describe
+ * distribution and co-location, upsert keys describe row identity. Useful for functions that
+ * need to emit key-only deletes, match UPDATE_BEFORE / UPDATE_AFTER pairs, or want to have a
+ * unique identifier to interact with state.
+ *
+ * <p>Returns an empty list when no upsert key is derivable, or when the planner has not yet
+ * computed metadata (during type inference).
+ *
+ * <p>When the planner derives multiple candidate upsert keys for the same input (e.g., a table
+ * with several primary key constraints), all of them are returned. Picking which candidate to
+ * use is the function's responsibility, and the choice must be stable across releases to keep
+ * PTF state consistent after job restarts and upgrades. The order of the returned list is not
+ * part of the contract; PTF authors should not depend on it. A typical choice is the smallest
+ * candidate by cardinality, with ties broken by the column indices in ascending order.
+ *
+ * @return Candidate upsert keys of the passed table, or an empty list if none.
+ */
+ List<int[]> upsertKeyColumns();
+
/** The sort direction for ORDER BY columns in table arguments with set semantics. */
@PublicEvolving
enum SortDirection {
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/UpsertKeyUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/UpsertKeyUtils.java
new file mode 100644
index 00000000000..528a977d037
--- /dev/null
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/UpsertKeyUtils.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.utils;
+
+import org.apache.flink.annotation.Internal;
+
+import java.util.Comparator;
+import java.util.List;
+
+/** Helpers for working with upsert key candidates. */
+@Internal
+public final class UpsertKeyUtils {
+
+ /**
+ * Comparator that orders upsert-key candidates deterministically and stably across releases:
+ * candidates with fewer columns come first; ties between equal-cardinality candidates are
+ * broken by the column indices in ascending order, leading column first.
+ */
+ private static final Comparator<int[]> SMALLEST_FIRST =
+ Comparator.<int[]>comparingInt(a -> a.length)
+ .thenComparing(
+ (a, b) -> {
+ for (int i = 0; i < a.length; i++) {
+ final int cmp = Integer.compare(a[i], b[i]);
+ if (cmp != 0) {
+ return cmp;
+ }
+ }
+ return 0;
+ });
+
+ /**
+ * Picks the smallest upsert key from the given candidates using {@link #SMALLEST_FIRST}.
+ * Returns an empty array when the candidate list is empty. The returned reference is one of the
+ * input arrays; callers must not mutate it.
+ */
+ public static int[] smallestKey(final List<int[]> candidates) {
+ if (candidates.isEmpty()) {
+ return new int[0];
+ }
+ int[] smallest = candidates.get(0);
+ for (int i = 1; i < candidates.size(); i++) {
+ if (SMALLEST_FIRST.compare(candidates.get(i), smallest) < 0) {
+ smallest = candidates.get(i);
+ }
+ }
+ return smallest;
+ }
+
+ private UpsertKeyUtils() {}
+}
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/utils/TableSemanticsMock.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/utils/TableSemanticsMock.java
index fe881f8fd1f..9b9830870fd 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/utils/TableSemanticsMock.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/utils/TableSemanticsMock.java
@@ -24,6 +24,8 @@ import org.apache.flink.table.types.DataType;
import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
import java.util.Optional;
/** Mock implementation of {@link TableSemantics} for testing purposes. */
@@ -35,6 +37,7 @@ public class TableSemanticsMock implements TableSemantics {
private final SortDirection[] orderByDirections;
private final int timeColumn;
private final ChangelogMode changelogMode;
+ private final List<int[]> upsertKeyColumns;
public TableSemanticsMock(DataType dataType) {
this(dataType, new int[0], new int[0], -1, null);
@@ -46,6 +49,22 @@ public class TableSemanticsMock implements TableSemantics {
int[] orderByColumns,
int timeColumn,
@Nullable ChangelogMode changelogMode) {
+ this(
+ dataType,
+ partitionByColumns,
+ orderByColumns,
+ timeColumn,
+ changelogMode,
+ Collections.emptyList());
+ }
+
+ public TableSemanticsMock(
+ DataType dataType,
+ int[] partitionByColumns,
+ int[] orderByColumns,
+ int timeColumn,
+ @Nullable ChangelogMode changelogMode,
+ List<int[]> upsertKeyColumns) {
this.dataType = dataType;
this.partitionByColumns = partitionByColumns;
this.orderByColumns = orderByColumns;
@@ -55,6 +74,7 @@ public class TableSemanticsMock implements TableSemantics {
}
this.timeColumn = timeColumn;
this.changelogMode = changelogMode;
+ this.upsertKeyColumns = upsertKeyColumns;
}
@Override
@@ -86,4 +106,9 @@ public class TableSemanticsMock implements TableSemantics {
public Optional<ChangelogMode> changelogMode() {
return Optional.ofNullable(changelogMode);
}
+
+ @Override
+ public List<int[]> upsertKeyColumns() {
+ return upsertKeyColumns;
+ }
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/bridging/BridgingSqlFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/bridging/BridgingSqlFunction.java
index 2c393adf875..900d8b6a0bd 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/bridging/BridgingSqlFunction.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/bridging/BridgingSqlFunction.java
@@ -335,7 +335,7 @@ public class BridgingSqlFunction extends SqlFunction {
* scalar arguments through the same coercion path as validation.
*/
public CallContext toCallContext(RexCall call) {
- return toCallContext(call, null, null, null);
+ return toCallContext(call, null, null, null, null);
}
/**
@@ -348,6 +348,20 @@ public class BridgingSqlFunction extends SqlFunction {
@Nullable List<Integer> inputTimeColumns,
@Nullable List<ChangelogMode> inputChangelogModes,
@Nullable ChangelogMode outputChangelogMode) {
+ return toCallContext(
+ call, inputTimeColumns, inputChangelogModes, outputChangelogMode, null);
+ }
+
+ /**
+ * Variant that additionally exposes the call's input upsert keys. Used by the streaming codegen
+ * path so PTFs can specialize themselves on the input's row-identity information.
+ */
+ public CallContext toCallContext(
+ RexCall call,
+ @Nullable List<Integer> inputTimeColumns,
+ @Nullable List<ChangelogMode> inputChangelogModes,
+ @Nullable ChangelogMode outputChangelogMode,
+ @Nullable List<List<int[]>> inputUpsertKeys) {
return new OperatorBindingCallContext(
dataTypeFactory,
getDefinition(),
@@ -355,7 +369,8 @@ public class BridgingSqlFunction extends SqlFunction {
call.getType(),
inputTimeColumns,
inputChangelogModes,
- outputChangelogMode);
+ outputChangelogMode,
+ inputUpsertKeys);
}
/**
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/CallBindingCallContext.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/CallBindingCallContext.java
index 065a3033545..b86fe9b4175 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/CallBindingCallContext.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/CallBindingCallContext.java
@@ -50,6 +50,7 @@ import org.apache.calcite.sql.type.SqlTypeName;
import javax.annotation.Nullable;
import java.util.AbstractList;
+import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -312,6 +313,11 @@ public final class CallBindingCallContext extends AbstractSqlCallContext {
public Optional<ChangelogMode> changelogMode() {
return Optional.empty();
}
+
+ @Override
+ public List<int[]> upsertKeyColumns() {
+ return Collections.emptyList();
+ }
}
// --------------------------------------------------------------------------------------------
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/OperatorBindingCallContext.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/OperatorBindingCallContext.java
index f31406dad19..7d1d3940899 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/OperatorBindingCallContext.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/inference/OperatorBindingCallContext.java
@@ -64,13 +64,14 @@ public final class OperatorBindingCallContext extends AbstractSqlCallContext {
private final @Nullable List<Integer> inputTimeColumns;
private final @Nullable List<ChangelogMode> inputChangelogModes;
private final @Nullable ChangelogMode outputChangelogMode;
+ private final @Nullable List<List<int[]>> inputUpsertKeys;
public OperatorBindingCallContext(
DataTypeFactory dataTypeFactory,
FunctionDefinition definition,
SqlOperatorBinding binding,
RelDataType returnRelDataType) {
- this(dataTypeFactory, definition, binding, returnRelDataType, null, null, null);
+ this(dataTypeFactory, definition, binding, returnRelDataType, null, null, null, null);
}
public OperatorBindingCallContext(
@@ -81,6 +82,26 @@ public final class OperatorBindingCallContext extends AbstractSqlCallContext {
@Nullable List<Integer> inputTimeColumns,
@Nullable List<ChangelogMode> inputChangelogModes,
@Nullable ChangelogMode outputChangelogMode) {
+ this(
+ dataTypeFactory,
+ definition,
+ binding,
+ returnRelDataType,
+ inputTimeColumns,
+ inputChangelogModes,
+ outputChangelogMode,
+ null);
+ }
+
+ public OperatorBindingCallContext(
+ DataTypeFactory dataTypeFactory,
+ FunctionDefinition definition,
+ SqlOperatorBinding binding,
+ RelDataType returnRelDataType,
+ @Nullable List<Integer> inputTimeColumns,
+ @Nullable List<ChangelogMode> inputChangelogModes,
+ @Nullable ChangelogMode outputChangelogMode,
+ @Nullable List<List<int[]>> inputUpsertKeys) {
super(
dataTypeFactory,
definition,
@@ -109,6 +130,7 @@ public final class OperatorBindingCallContext extends AbstractSqlCallContext {
this.inputTimeColumns = inputTimeColumns;
this.inputChangelogModes = inputChangelogModes;
this.outputChangelogMode = outputChangelogMode;
+ this.inputUpsertKeys = inputUpsertKeys;
}
@Override
@@ -173,13 +195,18 @@ public final class OperatorBindingCallContext extends AbstractSqlCallContext {
Optional.ofNullable(inputChangelogModes)
.map(m -> m.get(tableArgCall.getInputIndex()))
.orElse(null);
+ final List<int[]> upsertKeys =
+ Optional.ofNullable(inputUpsertKeys)
+ .map(m -> m.get(tableArgCall.getInputIndex()))
+ .orElse(List.of());
return Optional.of(
OperatorBindingTableSemantics.create(
argumentDataTypes.get(pos),
staticArg,
tableArgCall,
timeColumn,
- changelogMode));
+ changelogMode,
+ upsertKeys));
}
@Override
@@ -283,20 +310,23 @@ public final class OperatorBindingCallContext extends AbstractSqlCallContext {
private final SortDirection[] orderByDirections;
private final int timeColumn;
private final @Nullable ChangelogMode changelogMode;
+ private final List<int[]> upsertKeyColumns;
public static OperatorBindingTableSemantics create(
DataType tableDataType,
StaticArgument staticArg,
RexTableArgCall tableArgCall,
int timeColumn,
- @Nullable ChangelogMode changelogMode) {
+ @Nullable ChangelogMode changelogMode,
+ List<int[]> upsertKeyColumns) {
return new OperatorBindingTableSemantics(
createDataType(tableDataType, staticArg),
tableArgCall.getPartitionKeys(),
tableArgCall.getOrderKeys(),
RexTableArgCall.toSortDirections(tableArgCall.getSortOrder()),
timeColumn,
- changelogMode);
+ changelogMode,
+ upsertKeyColumns);
}
private OperatorBindingTableSemantics(
@@ -305,13 +335,15 @@ public final class OperatorBindingCallContext extends AbstractSqlCallContext {
int[] orderByColumns,
SortDirection[] orderByDirections,
int timeColumn,
- @Nullable ChangelogMode changelogMode) {
+ @Nullable ChangelogMode changelogMode,
+ List<int[]> upsertKeyColumns) {
this.dataType = dataType;
this.partitionByColumns = partitionByColumns;
this.orderByColumns = orderByColumns;
this.orderByDirections = orderByDirections;
this.timeColumn = timeColumn;
this.changelogMode = changelogMode;
+ this.upsertKeyColumns = upsertKeyColumns;
}
private static DataType createDataType(DataType tableDataType, StaticArgument staticArg) {
@@ -353,5 +385,10 @@ public final class OperatorBindingCallContext extends AbstractSqlCallContext {
public Optional<ChangelogMode> changelogMode() {
return Optional.ofNullable(changelogMode);
}
+
+ @Override
+ public List<int[]> upsertKeyColumns() {
+ return upsertKeyColumns;
+ }
}
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecProcessTableFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecProcessTableFunction.java
index 3973329af74..e0cf5f09187 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecProcessTableFunction.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecProcessTableFunction.java
@@ -69,6 +69,7 @@ import org.apache.flink.table.types.inference.TypeInferenceUtil.StateInfo;
import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.calcite.linq4j.Ord;
@@ -108,6 +109,7 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
public static final String FIELD_NAME_FUNCTION_CALL = "functionCall";
public static final String FIELD_NAME_INPUT_CHANGELOG_MODES = "inputChangelogModes";
public static final String FIELD_NAME_OUTPUT_CHANGELOG_MODE = "outputChangelogMode";
+ public static final String FIELD_NAME_INPUT_UPSERT_KEYS = "inputUpsertKeys";
@JsonProperty(FIELD_NAME_UID)
private final @Nullable String uid;
@@ -121,6 +123,10 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
@JsonProperty(FIELD_NAME_OUTPUT_CHANGELOG_MODE)
private final ChangelogMode outputChangelogMode;
+ @JsonProperty(FIELD_NAME_INPUT_UPSERT_KEYS)
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private final List<List<int[]>> inputUpsertKeys;
+
public StreamExecProcessTableFunction(
ReadableConfig tableConfig,
List<InputProperty> inputProperties,
@@ -129,7 +135,8 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
@Nullable String uid,
RexCall invocation,
List<ChangelogMode> inputChangelogModes,
- ChangelogMode outputChangelogMode) {
+ ChangelogMode outputChangelogMode,
+ List<List<int[]>> inputUpsertKeys) {
this(
ExecNodeContext.newNodeId(),
ExecNodeContext.newContext(StreamExecProcessTableFunction.class),
@@ -141,7 +148,8 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
uid,
invocation,
inputChangelogModes,
- outputChangelogMode);
+ outputChangelogMode,
+ inputUpsertKeys);
}
@JsonCreator
@@ -155,7 +163,9 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
@JsonProperty(FIELD_NAME_UID) @Nullable String uid,
@JsonProperty(FIELD_NAME_FUNCTION_CALL) RexNode invocation,
@JsonProperty(FIELD_NAME_INPUT_CHANGELOG_MODES) List<ChangelogMode> inputChangelogModes,
- @JsonProperty(FIELD_NAME_OUTPUT_CHANGELOG_MODE) ChangelogMode outputChangelogMode) {
+ @JsonProperty(FIELD_NAME_OUTPUT_CHANGELOG_MODE) ChangelogMode outputChangelogMode,
+ @JsonProperty(FIELD_NAME_INPUT_UPSERT_KEYS)
+ @Nullable List<List<int[]>> inputUpsertKeys) {
super(id, context, persistedConfig, inputProperties, outputType, description);
this.uid = uid;
// Mirror the FlinkLogicalTableFunctionScan converter for the compiled-plan restore path:
@@ -164,6 +174,7 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
this.invocation = BridgingSqlFunction.resolveCallTraits((RexCall) invocation);
this.inputChangelogModes = inputChangelogModes;
this.outputChangelogMode = outputChangelogMode;
+ this.inputUpsertKeys = inputUpsertKeys != null ? inputUpsertKeys : List.of();
}
public @Nullable String getUid() {
@@ -202,7 +213,12 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
final RexCall udfCall = StreamPhysicalProcessTableFunction.toUdfCall(invocation);
final GeneratedRunnerResult generated =
ProcessTableRunnerGenerator.generate(
- ctx, udfCall, inputTimeColumns, inputChangelogModes, outputChangelogMode);
+ ctx,
+ udfCall,
+ inputTimeColumns,
+ inputChangelogModes,
+ outputChangelogMode,
+ inputUpsertKeys);
final GeneratedProcessTableRunner generatedRunner = generated.runner();
final LinkedHashMap<String, StateInfo> stateInfos = generated.stateInfos();
@@ -310,9 +326,12 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
final int timeColumn = inputTimeColumns.get(tableArgCall.getInputIndex());
+ final int inputIndex = tableArgCall.getInputIndex();
+ final List<int[]> upsertKeys =
+ inputIndex < inputUpsertKeys.size() ? inputUpsertKeys.get(inputIndex) : List.of();
return new RuntimeTableSemantics(
tableArg.getName(),
- tableArgCall.getInputIndex(),
+ inputIndex,
dataType,
tableArgCall.getPartitionKeys(),
tableArgCall.getOrderKeys(),
@@ -320,7 +339,8 @@ public class StreamExecProcessTableFunction extends ExecNodeBase<RowData>
consumedChangelogMode,
tableArg.is(StaticArgumentTrait.PASS_COLUMNS_THROUGH),
tableArg.is(StaticArgumentTrait.SET_SEMANTIC_TABLE),
- timeColumn);
+ timeColumn,
+ upsertKeys);
}
private Transformation<RowData> createKeyedTransformation(
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalProcessTableFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalProcessTableFunction.java
index 5ccecf18e71..a337c250860 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalProcessTableFunction.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalProcessTableFunction.java
@@ -27,6 +27,7 @@ import org.apache.flink.table.functions.ProcessTableFunction;
import org.apache.flink.table.planner.calcite.FlinkTypeFactory;
import org.apache.flink.table.planner.calcite.RexTableArgCall;
import org.apache.flink.table.planner.functions.bridging.BridgingSqlFunction;
+import org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery;
import org.apache.flink.table.planner.plan.nodes.exec.ExecNode;
import org.apache.flink.table.planner.plan.nodes.exec.InputProperty;
import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecProcessTableFunction;
@@ -62,6 +63,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
@@ -165,6 +167,20 @@ public class StreamPhysicalProcessTableFunction extends AbstractRelNode
verifyTimeAttributes(getInputs(), call, inputChangelogModes, outputChangelogMode);
final List<Ord<StaticArgument>> providedInputArgs = getProvidedInputArgs(call);
verifyPassThroughColumnsForUpdates(providedInputArgs, outputChangelogMode);
+ final FlinkRelMetadataQuery fmq =
+ FlinkRelMetadataQuery.reuseOrCreate(getCluster().getMetadataQuery());
+ final List<List<int[]>> inputUpsertKeys =
+ getInputs().stream()
+ .map(
+ input -> {
+ final Set<ImmutableBitSet> keys = fmq.getUpsertKeys(input);
+ return keys == null
+ ? Collections.<int[]>emptyList()
+ : keys.stream()
+ .map(ImmutableBitSet::toArray)
+ .collect(Collectors.toList());
+ })
+ .collect(Collectors.toList());
return new StreamExecProcessTableFunction(
unwrapTableConfig(this),
getInputs().stream().map(i -> InputProperty.DEFAULT).collect(Collectors.toList()),
@@ -173,7 +189,8 @@ public class StreamPhysicalProcessTableFunction extends AbstractRelNode
uid,
call,
inputChangelogModes,
- outputChangelogMode);
+ outputChangelogMode,
+ inputUpsertKeys);
}
@Override
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/UpsertKeyUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/UpsertKeyUtil.java
index cc301118b29..af99d92e9ed 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/UpsertKeyUtil.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/UpsertKeyUtil.java
@@ -18,13 +18,17 @@
package org.apache.flink.table.planner.plan.utils;
+import org.apache.flink.table.utils.UpsertKeyUtils;
+
import org.apache.calcite.util.ImmutableBitSet;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
+import java.util.List;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* Utility for upsertKey which represented as a Set of {@link
@@ -55,21 +59,8 @@ public class UpsertKeyUtil {
if (null == upsertKeys || upsertKeys.isEmpty()) {
return Optional.empty();
}
- return upsertKeys.stream()
- .map(ImmutableBitSet::toArray)
- .reduce(
- (k1, k2) -> {
- if (k1.length < k2.length) {
- return k1;
- }
- if (k1.length == k2.length) {
- for (int index = 0; index < k1.length; index++) {
- if (k1[index] < k2[index]) {
- return k1;
- }
- }
- }
- return k2;
- });
+ final List<int[]> asArrays =
+ upsertKeys.stream().map(ImmutableBitSet::toArray).collect(Collectors.toList());
+ return Optional.of(UpsertKeyUtils.smallestKey(asArrays));
}
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProcessTableRunnerGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProcessTableRunnerGenerator.scala
index 52df803d5c8..ea9f5ca4c77 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProcessTableRunnerGenerator.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProcessTableRunnerGenerator.scala
@@ -65,7 +65,8 @@ object ProcessTableRunnerGenerator {
udfCall: RexCall,
inputTimeColumns: java.util.List[Integer],
inputChangelogModes: java.util.List[ChangelogMode],
- outputChangelogMode: ChangelogMode): GeneratedRunnerResult = {
+ outputChangelogMode: ChangelogMode,
+ inputUpsertKeys: java.util.List[java.util.List[Array[Int]]]): GeneratedRunnerResult = {
val function: BridgingSqlFunction = udfCall.getOperator.asInstanceOf[BridgingSqlFunction]
val definition: FunctionDefinition = function.getDefinition
val dataTypeFactory = function.getDataTypeFactory
@@ -77,7 +78,12 @@ object ProcessTableRunnerGenerator {
// Thus, functions can reconfigure themselves for the exact use case.
// Including updating their state layout.
val callContext =
- function.toCallContext(udfCall, inputTimeColumns, inputChangelogModes, outputChangelogMode)
+ function.toCallContext(
+ udfCall,
+ inputTimeColumns,
+ inputChangelogModes,
+ outputChangelogMode,
+ inputUpsertKeys)
// Create the final UDF for runtime
val udf = UserDefinedFunctionHelper.createSpecializedFunction(
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/to-changelog-retract-restore/plan/to-changelog-retract-restore.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/to-changelog-retract-restore/plan/to-changelog-retract-restore.json
index ea961c665ca..324327cad4b 100644
--- a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/to-changelog-retract-restore/plan/to-changelog-retract-restore.json
+++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-process-table-function_1/to-changelog-retract-restore/plan/to-changelog-retract-restore.json
@@ -78,7 +78,8 @@
"type" : "ROW<`op` VARCHAR(2147483647), `name` VARCHAR(2147483647) NOT NULL, `score` BIGINT> NOT NULL"
},
"inputChangelogModes" : [ [ "INSERT", "UPDATE_BEFORE", "UPDATE_AFTER", "DELETE" ] ],
- "outputChangelogMode" : [ "INSERT" ]
+ "outputChangelogMode" : [ "INSERT" ],
+ "inputUpsertKeys" : [ [ [ 0 ] ] ]
}, {
"id" : 3,
"type" : "stream-exec-sink_2",
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/RuntimeTableSemantics.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/RuntimeTableSemantics.java
index cabab4c6131..7f0fbf0c3d3 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/RuntimeTableSemantics.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/RuntimeTableSemantics.java
@@ -24,6 +24,7 @@ import org.apache.flink.table.functions.TableSemantics;
import org.apache.flink.table.types.DataType;
import java.io.Serializable;
+import java.util.List;
import java.util.Optional;
/**
@@ -44,6 +45,7 @@ public class RuntimeTableSemantics implements TableSemantics, Serializable {
private final boolean passColumnsThrough;
private final boolean hasSetSemantics;
private final int timeColumn;
+ private final List<int[]> upsertKeyColumns;
private transient ChangelogMode changelogMode;
@@ -57,7 +59,8 @@ public class RuntimeTableSemantics implements TableSemantics, Serializable {
RuntimeChangelogMode consumedChangelogMode,
boolean passColumnsThrough,
boolean hasSetSemantics,
- int timeColumn) {
+ int timeColumn,
+ List<int[]> upsertKeyColumns) {
this.argName = argName;
this.inputIndex = inputIndex;
this.dataType = dataType;
@@ -68,6 +71,7 @@ public class RuntimeTableSemantics implements TableSemantics, Serializable {
this.passColumnsThrough = passColumnsThrough;
this.hasSetSemantics = hasSetSemantics;
this.timeColumn = timeColumn;
+ this.upsertKeyColumns = upsertKeyColumns;
}
public String getArgName() {
@@ -122,4 +126,9 @@ public class RuntimeTableSemantics implements TableSemantics, Serializable {
public Optional<ChangelogMode> changelogMode() {
return Optional.of(getChangelogMode());
}
+
+ @Override
+ public List<int[]> upsertKeyColumns() {
+ return upsertKeyColumns;
+ }
}
diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/ProcessSetTableOperatorInterruptibleTimersTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/ProcessSetTableOperatorInterruptibleTimersTest.java
index be390ab5f55..15258ff24cb 100644
--- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/ProcessSetTableOperatorInterruptibleTimersTest.java
+++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/ProcessSetTableOperatorInterruptibleTimersTest.java
@@ -246,7 +246,8 @@ class ProcessSetTableOperatorInterruptibleTimersTest {
RuntimeChangelogMode.serialize(ChangelogMode.insertOnly()),
/* passColumnsThrough */ false,
/* hasSetSemantics */ true,
- /* timeColumn */ 1);
+ /* timeColumn */ 1,
+ /* upsertKeyColumns */ List.of());
}
// --------------------------------------------------------------------------------------------
diff --git a/flink-table/flink-table-test-utils/src/main/java/org/apache/flink/table/runtime/functions/TestHarnessTableSemantics.java b/flink-table/flink-table-test-utils/src/main/java/org/apache/flink/table/runtime/functions/TestHarnessTableSemantics.java
index fadf21d7dd9..91edd9a059d 100644
--- a/flink-table/flink-table-test-utils/src/main/java/org/apache/flink/table/runtime/functions/TestHarnessTableSemantics.java
+++ b/flink-table/flink-table-test-utils/src/main/java/org/apache/flink/table/runtime/functions/TestHarnessTableSemantics.java
@@ -23,6 +23,8 @@ import org.apache.flink.table.connector.ChangelogMode;
import org.apache.flink.table.functions.TableSemantics;
import org.apache.flink.table.types.DataType;
+import java.util.Collections;
+import java.util.List;
import java.util.Optional;
/** {@link TableSemantics} implementation for {@link ProcessTableFunctionTestHarness}. */
@@ -30,10 +32,17 @@ import java.util.Optional;
class TestHarnessTableSemantics implements TableSemantics {
private final DataType dataType;
private final int[] partitionByColumns;
+ private final List<int[]> upsertKeyColumns;
TestHarnessTableSemantics(DataType dataType, int[] partitionByColumns) {
+ this(dataType, partitionByColumns, Collections.emptyList());
+ }
+
+ TestHarnessTableSemantics(
+ DataType dataType, int[] partitionByColumns, List<int[]> upsertKeyColumns) {
this.dataType = dataType;
this.partitionByColumns = partitionByColumns;
+ this.upsertKeyColumns = upsertKeyColumns;
}
@Override
@@ -65,4 +74,9 @@ class TestHarnessTableSemantics implements TableSemantics {
public Optional<ChangelogMode> changelogMode() {
return Optional.empty();
}
+
+ @Override
+ public List<int[]> upsertKeyColumns() {
+ return upsertKeyColumns;
+ }
}
(flink) 06/06: [FLINK-39636][table] Widen output column nullability for partial deletes
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gustavodemorais pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit c033b17fae27117fbfc6b007348e9499d7830023
Author: Ramin Gharib <rg...@confluent.io>
AuthorDate: Thu May 28 16:00:22 2026 +0200
[FLINK-39636][table] Widen output column nullability for partial deletes
When produces_full_deletes=false, the operator nulls out columns that are not part of the upsert key or PARTITION BY on DELETE rows. The output schema previously declared those columns as NOT NULL, mirroring the input, which conflicts with the values the operator emits at runtime.
Read produces_full_deletes in OUTPUT_TYPE_STRATEGY and widen non-preserved columns to nullable. Preserved columns (upsert key plus partition keys) keep their declared nullability. The new ChangelogTypeStrategyUtils#computePreservedColumnIndices helper centralizes the index computation so the runtime and the type strategy stay
in sync.
Add ToChangelogOutputTypeStrategyTest covering all three cases: full deletes preserves NOT NULL, partial deletes with an upsert key widens only the non-key columns, and partial deletes without an upsert key widens everything.
---
.../docs/sql/reference/queries/changelog.md | 4 +
.../strategies/ChangelogTypeStrategyUtils.java | 24 +++++
.../strategies/ToChangelogTypeStrategy.java | 39 +++++++-
.../ToChangelogOutputTypeStrategyTest.java | 110 +++++++++++++++++++++
4 files changed, 174 insertions(+), 3 deletions(-)
diff --git a/docs/content/docs/sql/reference/queries/changelog.md b/docs/content/docs/sql/reference/queries/changelog.md
index 641bb7245f7..71539a9b8cb 100644
--- a/docs/content/docs/sql/reference/queries/changelog.md
+++ b/docs/content/docs/sql/reference/queries/changelog.md
@@ -449,6 +449,10 @@ SELECT * FROM TO_CHANGELOG(input => TABLE retract_source PARTITION BY id)
The planner skips `ChangelogNormalize` and the function emits partial DELETE rows. This avoids the stateful normalization operator for upsert sources (e.g. Kafka compacted topics) where the full pre-image is not needed downstream. This requires an [upsert key](#upsert-key) to be present for the input table (row semantics) or `PARTITION BY` (set semantics); otherwise the call is rejected with a validation error.
+{{< hint warning >}}
+**Output nullability changes when `produces_full_deletes => false`.** Columns that are not part of the upsert key (or `PARTITION BY`) are nulled on DELETE rows at runtime, so the type system widens them to nullable in the output schema. Columns declared `NOT NULL` on the input therefore appear as nullable in the output of a `TO_CHANGELOG(..., produces_full_deletes => false)` call. The upsert-key (or partition-key) columns keep their declared nullability. Use the default `produces_full_de [...]
+{{< /hint >}}
+
**Row semantics** (no `PARTITION BY`): the function preserves the planner-derived upsert key columns on DELETE rows and nulls the rest. The upsert key is typically a declared `PRIMARY KEY` when directly reading from a source or the key provided in a `GROUP BY <key>`.
```sql
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ChangelogTypeStrategyUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ChangelogTypeStrategyUtils.java
index d4e20c30210..6adc885dd78 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ChangelogTypeStrategyUtils.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ChangelogTypeStrategyUtils.java
@@ -22,9 +22,11 @@ import org.apache.flink.annotation.Internal;
import org.apache.flink.table.functions.TableSemantics;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.inference.CallContext;
+import org.apache.flink.table.utils.UpsertKeyUtils;
import org.apache.flink.types.ColumnList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.OptionalInt;
import java.util.Set;
@@ -80,6 +82,28 @@ public final class ChangelogTypeStrategyUtils {
return computeOutputIndices(tableSemantics, opIndex);
}
+ /**
+ * Returns the set of input column indices whose values the function preserves on partial DELETE
+ * rows (i.e. those that {@code TO_CHANGELOG} keeps when {@code produces_full_deletes=false}).
+ * The set is the union of {@code PARTITION BY} columns (the framework prepends them outside the
+ * projected output) and the smallest upsert-key candidate exposed by {@link
+ * TableSemantics#upsertKeyColumns()}.
+ *
+ * <p>Returns only the partition keys when no upsert key candidate is exposed yet, for example
+ * during type inference where planner-derived metadata is not yet populated.
+ */
+ public static Set<Integer> computePreservedColumnIndices(final TableSemantics tableSemantics) {
+ final Set<Integer> preserved = new HashSet<>(collectPartitionKeyIndices(tableSemantics));
+ final List<int[]> upsertKeys = tableSemantics.upsertKeyColumns();
+ if (upsertKeys == null || upsertKeys.isEmpty()) {
+ return preserved;
+ }
+ for (final int column : UpsertKeyUtils.smallestKey(upsertKeys)) {
+ preserved.add(column);
+ }
+ return preserved;
+ }
+
private static int[] computeOutputIndices(
final TableSemantics tableSemantics, final int extraExcludedIndex) {
final Set<Integer> excluded = collectPartitionKeyIndices(tableSemantics);
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
index 1434b221077..a2b8e13681b 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
@@ -32,6 +32,7 @@ import org.apache.flink.types.RowKind;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -84,8 +85,13 @@ public final class ToChangelogTypeStrategy {
final String opColumnName =
ChangelogTypeStrategyUtils.resolveOpColumnName(callContext);
+ final boolean producesFullDeletes =
+ callContext
+ .getArgumentValue(ARG_PRODUCES_FULL_DELETES, Boolean.class)
+ .orElse(true);
- final List<Field> outputFields = buildOutputFields(semantics, opColumnName);
+ final List<Field> outputFields =
+ buildOutputFields(semantics, opColumnName, producesFullDeletes);
return Optional.of(DataTypes.ROW(outputFields).notNull());
};
@@ -208,15 +214,42 @@ public final class ToChangelogTypeStrategy {
return false;
}
+ /**
+ * Builds the output {@link Field}s for the {@code TO_CHANGELOG} call.
+ *
+ * <p>When the function emits partial DELETE rows ({@code produces_full_deletes=false}), columns
+ * that are not part of the upsert key (or {@code PARTITION BY}) are nulled out at runtime.
+ * Those columns are widened to nullable here so the output schema matches what the operator can
+ * emit; preserved columns keep their declared nullability.
+ *
+ * <p>At type inference time the upsert key may not be exposed yet; in that case the helper
+ * returns only the partition keys and all projected columns are widened conservatively.
+ */
private static List<Field> buildOutputFields(
- final TableSemantics semantics, final String opColumnName) {
+ final TableSemantics semantics,
+ final String opColumnName,
+ final boolean producesFullDeletes) {
final List<Field> inputFields = DataType.getFields(semantics.dataType());
final int[] outputIndices = ChangelogTypeStrategyUtils.computeOutputIndices(semantics);
final List<Field> outputFields = new ArrayList<>();
outputFields.add(DataTypes.FIELD(opColumnName, DataTypes.STRING()));
- Arrays.stream(outputIndices).mapToObj(inputFields::get).forEach(outputFields::add);
+ final Set<Integer> preserved =
+ producesFullDeletes
+ ? Collections.emptySet()
+ : ChangelogTypeStrategyUtils.computePreservedColumnIndices(semantics);
+ Arrays.stream(outputIndices)
+ .mapToObj(
+ idx ->
+ producesFullDeletes || preserved.contains(idx)
+ ? inputFields.get(idx)
+ : asNullable(inputFields.get(idx)))
+ .forEach(outputFields::add);
return outputFields;
}
+ private static Field asNullable(final Field field) {
+ return DataTypes.FIELD(field.getName(), field.getDataType().nullable());
+ }
+
private ToChangelogTypeStrategy() {}
}
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogOutputTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogOutputTypeStrategyTest.java
new file mode 100644
index 00000000000..e912370ef00
--- /dev/null
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogOutputTypeStrategyTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.types.inference.strategies;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.inference.TypeStrategiesTestBase;
+import org.apache.flink.table.types.inference.utils.TableSemanticsMock;
+import org.apache.flink.types.ColumnList;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.apache.flink.table.types.inference.strategies.SpecificTypeStrategies.TO_CHANGELOG_OUTPUT_TYPE_STRATEGY;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_OP;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_OP_MAPPING;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_PRODUCES_FULL_DELETES;
+import static org.apache.flink.table.types.inference.strategies.ToChangelogTypeStrategy.ARG_TABLE;
+
+/** Tests for {@link ToChangelogTypeStrategy#OUTPUT_TYPE_STRATEGY}. */
+class ToChangelogOutputTypeStrategyTest extends TypeStrategiesTestBase {
+
+ private static final DataType TABLE_TYPE_NOT_NULL_SCORE =
+ DataTypes.ROW(
+ DataTypes.FIELD("name", DataTypes.STRING().notNull()),
+ DataTypes.FIELD("score", DataTypes.BIGINT().notNull()));
+
+ private static final DataType DESCRIPTOR_TYPE = DataTypes.DESCRIPTOR();
+ private static final DataType MAP_TYPE = DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING());
+ private static final DataType BOOLEAN_TYPE = DataTypes.BOOLEAN();
+
+ @Override
+ protected Stream<TestSpec> testData() {
+ return Stream.of(
+ TestSpec.forStrategy(
+ "produces_full_deletes=true preserves NOT NULL on input columns",
+ TO_CHANGELOG_OUTPUT_TYPE_STRATEGY)
+ .inputTypes(
+ TABLE_TYPE_NOT_NULL_SCORE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(
+ ARG_TABLE, new TableSemanticsMock(TABLE_TYPE_NOT_NULL_SCORE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(ARG_OP_MAPPING, null)
+ .calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, true)
+ .expectDataType(
+ DataTypes.ROW(
+ DataTypes.FIELD("op", DataTypes.STRING()),
+ DataTypes.FIELD(
+ "name", DataTypes.STRING().notNull()),
+ DataTypes.FIELD(
+ "score", DataTypes.BIGINT().notNull()))
+ .notNull()),
+ TestSpec.forStrategy(
+ "produces_full_deletes=false widens non-upsert-key columns to nullable",
+ TO_CHANGELOG_OUTPUT_TYPE_STRATEGY)
+ .inputTypes(
+ TABLE_TYPE_NOT_NULL_SCORE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(
+ ARG_TABLE,
+ new TableSemanticsMock(
+ TABLE_TYPE_NOT_NULL_SCORE,
+ new int[0],
+ new int[0],
+ -1,
+ null,
+ List.of(new int[] {0})))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(ARG_OP_MAPPING, null)
+ .calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, false)
+ .expectDataType(
+ DataTypes.ROW(
+ DataTypes.FIELD("op", DataTypes.STRING()),
+ DataTypes.FIELD(
+ "name", DataTypes.STRING().notNull()),
+ DataTypes.FIELD("score", DataTypes.BIGINT()))
+ .notNull()),
+ TestSpec.forStrategy(
+ "produces_full_deletes=false without upsert key widens all columns",
+ TO_CHANGELOG_OUTPUT_TYPE_STRATEGY)
+ .inputTypes(
+ TABLE_TYPE_NOT_NULL_SCORE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(
+ ARG_TABLE, new TableSemanticsMock(TABLE_TYPE_NOT_NULL_SCORE))
+ .calledWithLiteralAt(ARG_OP, ColumnList.of("op"))
+ .calledWithLiteralAt(ARG_OP_MAPPING, null)
+ .calledWithLiteralAt(ARG_PRODUCES_FULL_DELETES, false)
+ .expectDataType(
+ DataTypes.ROW(
+ DataTypes.FIELD("op", DataTypes.STRING()),
+ DataTypes.FIELD("name", DataTypes.STRING()),
+ DataTypes.FIELD("score", DataTypes.BIGINT()))
+ .notNull()));
+ }
+}
(flink) 02/06: [FLINK-39636][table] Refactor ToChangelogTypeStrategy
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gustavodemorais pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 340fbf879fd3e03dea7b37f7c28c6c404db21f65
Author: Ramin Gharib <ra...@gmail.com>
AuthorDate: Fri May 22 17:06:54 2026 +0200
[FLINK-39636][table] Refactor ToChangelogTypeStrategy
Aligns the structure of ToChangelogTypeStrategy with FromChangelogTypeStrategy. Pure refactor, no behavior change.
---
.../strategies/ToChangelogTypeStrategy.java | 182 +++++++++++++--------
.../ToChangelogInputTypeStrategyTest.java | 146 +++++++++++++++++
.../exec/stream/ToChangelogSemanticTests.java | 1 -
.../nodes/exec/stream/ToChangelogTestPrograms.java | 19 ---
.../runtime/functions/ptf/ToChangelogFunction.java | 20 +--
5 files changed, 267 insertions(+), 101 deletions(-)
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
index 8ff8101dc5e..d406875aa2f 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/ToChangelogTypeStrategy.java
@@ -22,15 +22,10 @@ import org.apache.flink.annotation.Internal;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.DataTypes.Field;
import org.apache.flink.table.api.ValidationException;
-import org.apache.flink.table.functions.FunctionDefinition;
import org.apache.flink.table.functions.TableSemantics;
import org.apache.flink.table.types.DataType;
-import org.apache.flink.table.types.inference.ArgumentCount;
import org.apache.flink.table.types.inference.CallContext;
-import org.apache.flink.table.types.inference.ConstantArgumentCount;
import org.apache.flink.table.types.inference.InputTypeStrategy;
-import org.apache.flink.table.types.inference.Signature;
-import org.apache.flink.table.types.inference.Signature.Argument;
import org.apache.flink.table.types.inference.TypeStrategy;
import org.apache.flink.types.ColumnList;
@@ -46,7 +41,13 @@ import java.util.Set;
@Internal
public final class ToChangelogTypeStrategy {
- private static final String DEFAULT_OP_COLUMN_NAME = "op";
+ // Positional argument indexes for TO_CHANGELOG. Must match the order of StaticArguments
+ // registered in BuiltInFunctionDefinitions#TO_CHANGELOG; changing one without the other
+ // silently breaks argument resolution.
+ public static final int ARG_TABLE = 0;
+ public static final int ARG_OP = 1;
+ public static final int ARG_OP_MAPPING = 2;
+ public static final int ARG_PRODUCES_FULL_DELETES = 3;
private static final Set<String> VALID_ROW_KIND_NAMES =
Set.of("INSERT", "UPDATE_BEFORE", "UPDATE_AFTER", "DELETE");
@@ -56,34 +57,12 @@ public final class ToChangelogTypeStrategy {
// --------------------------------------------------------------------------------------------
public static final InputTypeStrategy INPUT_TYPE_STRATEGY =
- new InputTypeStrategy() {
- @Override
- public ArgumentCount getArgumentCount() {
- return ConstantArgumentCount.between(1, 4);
- }
-
+ new ValidationOnlyInputTypeStrategy() {
@Override
public Optional<List<DataType>> inferInputTypes(
final CallContext callContext, final boolean throwOnFailure) {
return validateInputs(callContext, throwOnFailure);
}
-
- @Override
- public List<Signature> getExpectedSignatures(final FunctionDefinition definition) {
- return List.of(
- Signature.of(Argument.of("input", "TABLE")),
- Signature.of(
- Argument.of("input", "TABLE"), Argument.of("op", "DESCRIPTOR")),
- Signature.of(
- Argument.of("input", "TABLE"),
- Argument.of("op", "DESCRIPTOR"),
- Argument.of("op_mapping", "MAP<STRING, STRING>")),
- Signature.of(
- Argument.of("input", "TABLE"),
- Argument.of("op", "DESCRIPTOR"),
- Argument.of("op_mapping", "MAP<STRING, STRING>"),
- Argument.of("produces_full_deletes", "BOOLEAN")));
- }
};
// --------------------------------------------------------------------------------------------
@@ -94,20 +73,16 @@ public final class ToChangelogTypeStrategy {
callContext -> {
final TableSemantics semantics =
callContext
- .getTableSemantics(0)
+ .getTableSemantics(ARG_TABLE)
.orElseThrow(
() ->
new ValidationException(
"First argument must be a table for TO_CHANGELOG."));
- final String opColumnName = resolveOpColumnName(callContext);
- final List<Field> inputFields = DataType.getFields(semantics.dataType());
- final int[] outputIndices =
- ChangelogTypeStrategyUtils.computeOutputIndices(semantics);
+ final String opColumnName =
+ ChangelogTypeStrategyUtils.resolveOpColumnName(callContext);
- final List<Field> outputFields = new ArrayList<>();
- outputFields.add(DataTypes.FIELD(opColumnName, DataTypes.STRING()));
- Arrays.stream(outputIndices).mapToObj(inputFields::get).forEach(outputFields::add);
+ final List<Field> outputFields = buildOutputFields(semantics, opColumnName);
return Optional.of(DataTypes.ROW(outputFields).notNull());
};
@@ -118,45 +93,68 @@ public final class ToChangelogTypeStrategy {
private static Optional<List<DataType>> validateInputs(
final CallContext callContext, final boolean throwOnFailure) {
- final boolean isMissingTableArg = callContext.getTableSemantics(0).isEmpty();
- if (isMissingTableArg) {
+ Optional<List<DataType>> error;
+
+ error = validateTableArg(callContext, throwOnFailure);
+ if (error.isPresent()) {
+ return error;
+ }
+
+ error = validateOpDescriptor(callContext, throwOnFailure);
+ if (error.isPresent()) {
+ return error;
+ }
+
+ error = validateOpMapping(callContext, throwOnFailure);
+ if (error.isPresent()) {
+ return error;
+ }
+
+ error = validateProducesFullDeletes(callContext, throwOnFailure);
+ if (error.isPresent()) {
+ return error;
+ }
+
+ return Optional.of(callContext.getArgumentDataTypes());
+ }
+
+ private static Optional<List<DataType>> validateTableArg(
+ final CallContext callContext, final boolean throwOnFailure) {
+ if (callContext.getTableSemantics(ARG_TABLE).isEmpty()) {
return callContext.fail(
throwOnFailure, "First argument must be a table for TO_CHANGELOG.");
}
+ return Optional.empty();
+ }
- final Optional<ColumnList> opDescriptor = callContext.getArgumentValue(1, ColumnList.class);
- final boolean hasInvalidOpDescriptor =
- opDescriptor.isPresent() && opDescriptor.get().getNames().size() != 1;
- if (hasInvalidOpDescriptor) {
+ private static Optional<List<DataType>> validateOpDescriptor(
+ final CallContext callContext, final boolean throwOnFailure) {
+ final Optional<ColumnList> opDescriptor =
+ callContext.getArgumentValue(ARG_OP, ColumnList.class);
+ if (opDescriptor.isPresent() && opDescriptor.get().getNames().size() != 1) {
return callContext.fail(
throwOnFailure,
"The descriptor for argument 'op' must contain exactly one column name.");
}
+ return Optional.empty();
+ }
- final boolean hasMappingArgProvided = !callContext.isArgumentNull(2);
- final boolean isMappingArgLiteral = callContext.isArgumentLiteral(2);
+ /** Validates op_mapping is a constant literal and that its keys are well-formed. */
+ @SuppressWarnings("rawtypes")
+ private static Optional<List<DataType>> validateOpMapping(
+ final CallContext callContext, final boolean throwOnFailure) {
+ final boolean hasMappingArgProvided = !callContext.isArgumentNull(ARG_OP_MAPPING);
+ final boolean isMappingArgLiteral = callContext.isArgumentLiteral(ARG_OP_MAPPING);
if (hasMappingArgProvided && !isMappingArgLiteral) {
return callContext.fail(
throwOnFailure, "The 'op_mapping' argument must be a constant MAP literal.");
}
- final Optional<Map> opMapping = callContext.getArgumentValue(2, Map.class);
+ final Optional<Map> opMapping = callContext.getArgumentValue(ARG_OP_MAPPING, Map.class);
if (opMapping.isPresent()) {
- final Optional<List<DataType>> validationError =
- validateOpMappingKeys(callContext, opMapping.get(), throwOnFailure);
- if (validationError.isPresent()) {
- return validationError;
- }
+ return validateOpMappingKeys(callContext, opMapping.get(), throwOnFailure);
}
-
- final boolean hasProducesFullDeletesArg = !callContext.isArgumentNull(3);
- if (hasProducesFullDeletesArg && !callContext.isArgumentLiteral(3)) {
- return callContext.fail(
- throwOnFailure,
- "The 'produces_full_deletes' argument must be a constant BOOLEAN literal.");
- }
-
- return Optional.of(callContext.getArgumentDataTypes());
+ return Optional.empty();
}
/**
@@ -199,12 +197,64 @@ public final class ToChangelogTypeStrategy {
return Optional.empty();
}
- private static String resolveOpColumnName(final CallContext callContext) {
- return callContext
- .getArgumentValue(1, ColumnList.class)
- .filter(cl -> !cl.getNames().isEmpty())
- .map(cl -> cl.getNames().get(0))
- .orElse(DEFAULT_OP_COLUMN_NAME);
+ @SuppressWarnings("rawtypes")
+ private static Optional<List<DataType>> validateProducesFullDeletes(
+ final CallContext callContext, final boolean throwOnFailure) {
+ final boolean hasArgProvided = !callContext.isArgumentNull(ARG_PRODUCES_FULL_DELETES);
+ if (hasArgProvided && !callContext.isArgumentLiteral(ARG_PRODUCES_FULL_DELETES)) {
+ return callContext.fail(
+ throwOnFailure,
+ "The 'produces_full_deletes' argument must be a constant BOOLEAN literal.");
+ }
+ final boolean producesFullDeletes =
+ callContext
+ .getArgumentValue(ARG_PRODUCES_FULL_DELETES, Boolean.class)
+ .orElse(false);
+ if (!producesFullDeletes) {
+ return Optional.empty();
+ }
+ // The check against the input changelog mode lives in the function constructor since
+ // TableSemantics#changelogMode() returns empty here at type-inference time. The mapping
+ // check below only needs the literal op_mapping argument, so it lives here.
+ final Optional<Map> opMapping = callContext.getArgumentValue(ARG_OP_MAPPING, Map.class);
+ if (opMapping.isPresent() && !mapsDelete(opMapping.get())) {
+ return callContext.fail(
+ throwOnFailure,
+ "Invalid 'produces_full_deletes' for TO_CHANGELOG: the active 'op_mapping' "
+ + "does not map DELETE rows, so no DELETE rows are emitted. Remove "
+ + "the 'produces_full_deletes' argument or add a DELETE entry to "
+ + "'op_mapping'.");
+ }
+ return Optional.empty();
+ }
+
+ /**
+ * Returns {@code true} when at least one {@code op_mapping} key references {@code DELETE}.
+ * Keys may be comma-separated (e.g., {@code "INSERT, DELETE"}) per the user-facing contract.
+ */
+ @SuppressWarnings("rawtypes")
+ private static boolean mapsDelete(final Map opMapping) {
+ for (final Object key : opMapping.keySet()) {
+ if (!(key instanceof String)) {
+ continue;
+ }
+ for (final String rawName : ((String) key).split(",")) {
+ if ("DELETE".equals(rawName.trim())) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static List<Field> buildOutputFields(
+ final TableSemantics semantics, final String opColumnName) {
+ final List<Field> inputFields = DataType.getFields(semantics.dataType());
+ final int[] outputIndices = ChangelogTypeStrategyUtils.computeOutputIndices(semantics);
+ final List<Field> outputFields = new ArrayList<>();
+ outputFields.add(DataTypes.FIELD(opColumnName, DataTypes.STRING()));
+ Arrays.stream(outputIndices).mapToObj(inputFields::get).forEach(outputFields::add);
+ return outputFields;
}
private ToChangelogTypeStrategy() {}
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java
new file mode 100644
index 00000000000..e9b20a7a5ba
--- /dev/null
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/strategies/ToChangelogInputTypeStrategyTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.types.inference.strategies;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.inference.InputTypeStrategiesTestBase;
+import org.apache.flink.table.types.inference.utils.TableSemanticsMock;
+import org.apache.flink.types.ColumnList;
+
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.apache.flink.table.types.inference.strategies.SpecificInputTypeStrategies.TO_CHANGELOG_INPUT_TYPE_STRATEGY;
+
+/** Tests for {@link ToChangelogTypeStrategy#INPUT_TYPE_STRATEGY}. */
+class ToChangelogInputTypeStrategyTest extends InputTypeStrategiesTestBase {
+
+ private static final DataType TABLE_TYPE =
+ DataTypes.ROW(
+ DataTypes.FIELD("name", DataTypes.STRING()),
+ DataTypes.FIELD("score", DataTypes.BIGINT()));
+
+ private static final DataType DESCRIPTOR_TYPE = DataTypes.DESCRIPTOR();
+
+ private static final DataType MAP_TYPE = DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING());
+
+ private static final DataType BOOLEAN_TYPE = DataTypes.BOOLEAN();
+
+ @Override
+ protected Stream<TestSpec> testData() {
+ return Stream.of(
+ // Valid: produces_full_deletes=true with default op_mapping (includes DELETE)
+ TestSpec.forStrategy(
+ "Valid produces_full_deletes=true with default mapping",
+ TO_CHANGELOG_INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(1, ColumnList.of("op"))
+ .calledWithLiteralAt(2, null)
+ .calledWithLiteralAt(3, true)
+ .expectArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
+
+ // Valid: produces_full_deletes=true with op_mapping that includes DELETE
+ TestSpec.forStrategy(
+ "Valid produces_full_deletes=true with explicit DELETE mapping",
+ TO_CHANGELOG_INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(1, ColumnList.of("op"))
+ .calledWithLiteralAt(2, Map.of("INSERT", "I", "DELETE", "D"))
+ .calledWithLiteralAt(3, true)
+ .expectArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
+
+ // Valid: produces_full_deletes=true with comma-separated DELETE key
+ TestSpec.forStrategy(
+ "Valid produces_full_deletes=true with comma-separated DELETE",
+ TO_CHANGELOG_INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(1, ColumnList.of("op"))
+ .calledWithLiteralAt(2, Map.of("INSERT, DELETE", "X"))
+ .calledWithLiteralAt(3, true)
+ .expectArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
+
+ // Valid: produces_full_deletes=false with op_mapping that omits DELETE
+ TestSpec.forStrategy(
+ "Valid produces_full_deletes=false with no DELETE in mapping",
+ TO_CHANGELOG_INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(1, ColumnList.of("op"))
+ .calledWithLiteralAt(2, Map.of("INSERT, UPDATE_AFTER", "X"))
+ .calledWithLiteralAt(3, false)
+ .expectArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE),
+
+ // Error: produces_full_deletes=true with op_mapping that strips DELETE
+ TestSpec.forStrategy(
+ "produces_full_deletes=true rejected when op_mapping omits DELETE",
+ TO_CHANGELOG_INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(1, ColumnList.of("op"))
+ .calledWithLiteralAt(2, Map.of("INSERT, UPDATE_AFTER", "X"))
+ .calledWithLiteralAt(3, true)
+ .expectErrorMessage(
+ "Invalid 'produces_full_deletes' for TO_CHANGELOG: the active "
+ + "'op_mapping' does not map DELETE rows"),
+
+ // Error: multi-column descriptor
+ TestSpec.forStrategy(
+ "Descriptor with multiple columns",
+ TO_CHANGELOG_INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(1, ColumnList.of("a", "b"))
+ .expectErrorMessage("must contain exactly one column name"),
+
+ // Error: invalid RowKind in op_mapping key
+ TestSpec.forStrategy(
+ "Invalid RowKind in mapping key", TO_CHANGELOG_INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(1, ColumnList.of("op"))
+ .calledWithLiteralAt(2, Map.of("INVALID_KIND", "X"))
+ .expectErrorMessage("Unknown change operation: 'INVALID_KIND'"),
+
+ // Error: duplicate RowKind across entries
+ TestSpec.forStrategy(
+ "Duplicate RowKind in mapping keys",
+ TO_CHANGELOG_INPUT_TYPE_STRATEGY)
+ .calledWithArgumentTypes(
+ TABLE_TYPE, DESCRIPTOR_TYPE, MAP_TYPE, BOOLEAN_TYPE)
+ .calledWithTableSemanticsAt(0, new TableSemanticsMock(TABLE_TYPE))
+ .calledWithLiteralAt(1, ColumnList.of("op"))
+ .calledWithLiteralAt(2, Map.of("INSERT, DELETE", "A", "DELETE", "B"))
+ .expectErrorMessage("Duplicate change operation: 'DELETE'"));
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
index 762e08b3538..b3376b3a346 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogSemanticTests.java
@@ -57,7 +57,6 @@ public class ToChangelogSemanticTests extends SemanticTestBase {
ToChangelogTestPrograms.OP_MAPPING_REFERENCES_UNSUPPORTED_KIND,
ToChangelogTestPrograms.DUPLICATE_ROW_KIND,
ToChangelogTestPrograms.PRODUCES_FULL_DELETES_ON_APPEND_ONLY_INPUT,
- ToChangelogTestPrograms.PRODUCES_FULL_DELETES_WITHOUT_DELETE_IN_OP_MAPPING,
ToChangelogTestPrograms.ROW_SEM_PARTIAL_DELETES,
ToChangelogTestPrograms.ROW_SEM_FORCE_FULL_DELETES,
ToChangelogTestPrograms.SET_SEM_PARTIAL_DELETES,
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
index f04d30d1d99..5126014b9a3 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ToChangelogTestPrograms.java
@@ -613,25 +613,6 @@ public class ToChangelogTestPrograms {
"the input table only produces [INSERT] and never emits DELETE rows")
.build();
- public static final TableTestProgram PRODUCES_FULL_DELETES_WITHOUT_DELETE_IN_OP_MAPPING =
- TableTestProgram.of(
- "to-changelog-produces-full-deletes-without-delete-in-op-mapping",
- "fails when produces_full_deletes=true but the active op_mapping strips DELETE")
- .setupTableSource(
- SourceTestStep.newBuilder("t")
- .addSchema(
- "name STRING PRIMARY KEY NOT ENFORCED", "score BIGINT")
- .addMode(ChangelogMode.all())
- .producedValues(Row.ofKind(RowKind.INSERT, "Alice", 10L))
- .build())
- .runFailingSql(
- "SELECT * FROM TO_CHANGELOG("
- + "input => TABLE t, "
- + "op_mapping => MAP['INSERT, UPDATE_AFTER', 'X'], "
- + "produces_full_deletes => true)",
- ValidationException.class,
- "the active 'op_mapping' does not map DELETE rows")
- .build();
// --------------------------------------------------------------------------------------------
// Row semantics x delete handling matrix
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
index 0110475ef8d..d8874a2be16 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/ptf/ToChangelogFunction.java
@@ -88,7 +88,7 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
}
final boolean producesFullDeletesArg =
callContext.getArgumentValue(3, Boolean.class).orElse(false);
- validateProducesFullDeletes(producesFullDeletesArg, this.rawOpMap, tableSemantics);
+ validateProducesFullDeletes(producesFullDeletesArg, tableSemantics);
this.outputIndices = ChangelogTypeStrategyUtils.computeOutputIndices(tableSemantics);
this.producesFullDelete = resolveProducesFullDelete(producesFullDeletesArg, tableSemantics);
@@ -172,18 +172,15 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
}
/**
- * Rejects {@code produces_full_deletes=true} when the input changelog cannot produce DELETE
- * rows (either the input mode does not contain DELETE, or the active {@code op_mapping} strips
- * it). The parameter is then dead and likely a user mistake.
+ * Rejects {@code produces_full_deletes=true} when the input changelog never emits DELETE rows.
*
* <p>Lives here rather than in the input type strategy because {@link
* TableSemantics#changelogMode()} returns empty during type inference and is only populated at
- * specialization time, which is when this constructor runs.
+ * specialization time. The complementary check against the literal {@code op_mapping}
+ * argument runs earlier in {@code ToChangelogTypeStrategy}.
*/
private static void validateProducesFullDeletes(
- final boolean producesFullDeletesArg,
- final Map<RowKind, String> mapping,
- final TableSemantics tableSemantics) {
+ final boolean producesFullDeletesArg, final TableSemantics tableSemantics) {
if (!producesFullDeletesArg) {
return;
}
@@ -199,13 +196,6 @@ public class ToChangelogFunction extends BuiltInProcessTableFunction<RowData> {
+ "'produces_full_deletes' argument.",
inputMode.getContainedKinds()));
}
- if (!mapping.containsKey(RowKind.DELETE)) {
- throw new ValidationException(
- "Invalid 'produces_full_deletes' for TO_CHANGELOG: the active 'op_mapping' "
- + "does not map DELETE rows, so no DELETE rows are emitted. Remove "
- + "the 'produces_full_deletes' argument or add a DELETE entry to "
- + "'op_mapping'.");
- }
}
public void eval(