You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by le...@apache.org on 2024/11/18 09:25:06 UTC
(flink-cdc) branch release-3.2 updated: [FLINK-36509][pipeline-connector][paimon] Add partition columns to primary key
This is an automated email from the ASF dual-hosted git repository.
leonard pushed a commit to branch release-3.2
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
The following commit(s) were added to refs/heads/release-3.2 by this push:
new 40db6d5eb [FLINK-36509][pipeline-connector][paimon] Add partition columns to primary key
40db6d5eb is described below
commit 40db6d5eb7661696797acba30384e37e56cdac5a
Author: Kunni <lv...@alibaba-inc.com>
AuthorDate: Mon Nov 18 17:25:02 2024 +0800
[FLINK-36509][pipeline-connector][paimon] Add partition columns to primary key
This closes #3726.
---
.../paimon/sink/PaimonMetadataApplier.java | 18 +++++++----
.../paimon/sink/PaimonMetadataApplierTest.java | 35 ++++++++++++++++++++--
2 files changed, 45 insertions(+), 8 deletions(-)
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonMetadataApplier.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonMetadataApplier.java
index 3ae3c9c4c..22d5f4922 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonMetadataApplier.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonMetadataApplier.java
@@ -153,14 +153,22 @@ public class PaimonMetadataApplier implements MetadataApplier {
LogicalTypeConversion.toDataType(
DataTypeUtils.toFlinkDataType(column.getType())
.getLogicalType())));
- builder.primaryKey(schema.primaryKeys().toArray(new String[0]));
+ List<String> partitionKeys = new ArrayList<>();
+ List<String> primaryKeys = schema.primaryKeys();
if (partitionMaps.containsKey(event.tableId())) {
- builder.partitionKeys(partitionMaps.get(event.tableId()));
+ partitionKeys.addAll(partitionMaps.get(event.tableId()));
} else if (schema.partitionKeys() != null && !schema.partitionKeys().isEmpty()) {
- builder.partitionKeys(schema.partitionKeys());
+ partitionKeys.addAll(schema.partitionKeys());
}
- builder.options(tableOptions);
- builder.options(schema.options());
+ for (String partitionColumn : partitionKeys) {
+ if (!primaryKeys.contains(partitionColumn)) {
+ primaryKeys.add(partitionColumn);
+ }
+ }
+ builder.partitionKeys(partitionKeys)
+ .primaryKey(primaryKeys)
+ .options(tableOptions)
+ .options(schema.options());
catalog.createTable(
new Identifier(event.tableId().getSchemaName(), event.tableId().getTableName()),
builder.build(),
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonMetadataApplierTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonMetadataApplierTest.java
index 214f2051e..9f3cd806c 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonMetadataApplierTest.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/test/java/org/apache/flink/cdc/connectors/paimon/sink/PaimonMetadataApplierTest.java
@@ -177,6 +177,35 @@ public class PaimonMetadataApplierTest {
new DataField(2, "newcol3", DataTypes.STRING())));
Assertions.assertEquals(
tableSchema, catalog.getTable(Identifier.fromString("test.table1")).rowType());
+
+ // Create table with partition column.
+ createTableEvent =
+ new CreateTableEvent(
+ TableId.parse("test.table_with_partition"),
+ org.apache.flink.cdc.common.schema.Schema.newBuilder()
+ .physicalColumn(
+ "col1",
+ org.apache.flink.cdc.common.types.DataTypes.STRING()
+ .notNull())
+ .physicalColumn(
+ "col2", org.apache.flink.cdc.common.types.DataTypes.INT())
+ .physicalColumn(
+ "dt",
+ org.apache.flink.cdc.common.types.DataTypes.INT().notNull())
+ .primaryKey("col1")
+ .partitionKey("dt")
+ .build());
+ metadataApplier.applySchemaChange(createTableEvent);
+ tableSchema =
+ new RowType(
+ Arrays.asList(
+ new DataField(0, "col1", DataTypes.STRING().notNull()),
+ new DataField(1, "col2", DataTypes.INT()),
+ new DataField(2, "dt", DataTypes.INT().notNull())));
+ Table tableWithPartition =
+ catalog.getTable(Identifier.fromString("test.table_with_partition"));
+ Assertions.assertEquals(tableSchema, tableWithPartition.rowType());
+ Assertions.assertEquals(Arrays.asList("col1", "dt"), tableWithPartition.primaryKeys());
}
@ParameterizedTest
@@ -217,10 +246,10 @@ public class PaimonMetadataApplierTest {
Arrays.asList(
new DataField(0, "col1", DataTypes.STRING().notNull()),
new DataField(1, "col2", DataTypes.STRING()),
- new DataField(2, "col3", DataTypes.STRING()),
- new DataField(3, "col4", DataTypes.STRING())));
+ new DataField(2, "col3", DataTypes.STRING().notNull()),
+ new DataField(3, "col4", DataTypes.STRING().notNull())));
Assertions.assertEquals(tableSchema, table.rowType());
- Assertions.assertEquals(Collections.singletonList("col1"), table.primaryKeys());
+ Assertions.assertEquals(Arrays.asList("col1", "col3", "col4"), table.primaryKeys());
Assertions.assertEquals(Arrays.asList("col3", "col4"), table.partitionKeys());
Assertions.assertEquals("-1", table.options().get("bucket"));
}