You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by le...@apache.org on 2025/04/21 05:51:04 UTC
(flink-cdc) branch master updated: [FLINK-36061][pipeline-connector/iceberg] Introduce Iceberg Sink Connector
This is an automated email from the ASF dual-hosted git repository.
leonard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
The following commit(s) were added to refs/heads/master by this push:
new 241973bc0 [FLINK-36061][pipeline-connector/iceberg] Introduce Iceberg Sink Connector
241973bc0 is described below
commit 241973bc02d3b67914095d97515541a66aed198a
Author: Kunni <lv...@alibaba-inc.com>
AuthorDate: Mon Apr 21 13:51:00 2025 +0800
[FLINK-36061][pipeline-connector/iceberg] Introduce Iceberg Sink Connector
This closes #3904.
---
.github/labeler.yml | 2 +
.../docs/connectors/pipeline-connectors/iceberg.md | 228 ++++++++++++
.../docs/connectors/pipeline-connectors/iceberg.md | 228 ++++++++++++
.../flink-cdc-pipeline-connector-iceberg/pom.xml | 294 +++++++++++++++
.../connectors/iceberg/sink/IcebergDataSink.java | 76 ++++
.../iceberg/sink/IcebergDataSinkFactory.java | 127 +++++++
.../iceberg/sink/IcebergDataSinkOptions.java | 80 ++++
.../iceberg/sink/IcebergMetadataApplier.java | 295 +++++++++++++++
.../iceberg/sink/utils/IcebergTypeUtils.java | 204 ++++++++++
.../connectors/iceberg/sink/utils/OptionUtils.java | 42 +++
.../iceberg/sink/utils/RowDataUtils.java | 61 +++
.../iceberg/sink/v2/IcebergCommitter.java | 151 ++++++++
.../connectors/iceberg/sink/v2/IcebergSink.java | 162 ++++++++
.../connectors/iceberg/sink/v2/IcebergWriter.java | 189 ++++++++++
.../connectors/iceberg/sink/v2/TableMetric.java | 39 ++
.../iceberg/sink/v2/TableSchemaWrapper.java | 46 +++
.../iceberg/sink/v2/WriteResultWrapper.java | 71 ++++
.../sink/v2/WriteResultWrapperSerializer.java | 63 ++++
.../sink/v2/compaction/CompactionOperator.java | 152 ++++++++
.../sink/v2/compaction/CompactionOptions.java | 85 +++++
.../org.apache.flink.cdc.common.factories.Factory | 16 +
.../iceberg/sink/IcebergDataSinkFactoryTest.java | 94 +++++
.../iceberg/sink/IcebergMetadataApplierTest.java | 413 +++++++++++++++++++++
.../iceberg/sink/v2/CompactionOperatorTest.java | 164 ++++++++
.../iceberg/sink/v2/IcebergSinkITCase.java | 195 ++++++++++
.../iceberg/sink/v2/IcebergWriterTest.java | 389 +++++++++++++++++++
.../src/test/resources/log4j2-test.properties | 26 ++
.../flink-cdc-pipeline-connectors/pom.xml | 1 +
.../flink-cdc-pipeline-e2e-tests/pom.xml | 44 ++-
.../pipeline/tests/MySqlToIcebergE2eITCase.java | 383 +++++++++++++++++++
.../tests/utils/PipelineTestEnvironment.java | 2 +-
.../src/test/resources/ddl/iceberg_inventory.sql | 55 +++
.../src/test/resources/docker/peek-iceberg.sql | 29 ++
pom.xml | 3 +
34 files changed, 4402 insertions(+), 7 deletions(-)
diff --git a/.github/labeler.yml b/.github/labeler.yml
index 2ce49823a..0637188cd 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -93,3 +93,5 @@ oceanbase-pipeline-connector:
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase/**/*
maxcompute-pipeline-connector:
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute/**/*
+iceberg-pipeline-connector:
+ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/**/*
diff --git a/docs/content.zh/docs/connectors/pipeline-connectors/iceberg.md b/docs/content.zh/docs/connectors/pipeline-connectors/iceberg.md
new file mode 100644
index 000000000..be43b09b1
--- /dev/null
+++ b/docs/content.zh/docs/connectors/pipeline-connectors/iceberg.md
@@ -0,0 +1,228 @@
+---
+title: "Iceberg"
+weight: 9
+type: docs
+aliases:
+- /connectors/pipeline-connectors/iceberg
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Iceberg Pipeline Connector
+
+The Iceberg Pipeline Connector functions as a *Data Sink* for data pipelines, enabling data writes to Apache Iceberg tables[Iceberg](https://iceberg.apache.org). This document explains how to configure the connector.
+
+## Key Capabilities
+* Automatic Table Creation
+ Creates Iceberg tables dynamically when they do not exist
+* Schema Synchronization
+ Propagates schema changes (e.g., column additions) from source systems to Iceberg
+* Data Replication
+ Supports both batch and streaming data synchronization
+
+How to create Pipeline
+----------------
+
+The pipeline for reading data from MySQL and sink to Iceberg can be defined as follows:
+
+```yaml
+source:
+ type: mysql
+ name: MySQL Source
+ hostname: 127.0.0.1
+ port: 3306
+ username: admin
+ password: pass
+ tables: adb.\.*, bdb.user_table_[0-9]+, [app|web].order_\.*
+ server-id: 5401-5404
+
+sink:
+ type: iceberg
+ name: Iceberg Sink
+ catalog.properties.type: hadoop
+ catalog.properties.warehouse: /path/warehouse
+
+pipeline:
+ name: MySQL to Iceberg Pipeline
+ parallelism: 2
+```
+
+Pipeline Connector Options
+----------------
+<div class="highlight">
+<table class="colwidths-auto docutils">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 25%">Option</th>
+ <th class="text-left" style="width: 8%">Required</th>
+ <th class="text-left" style="width: 7%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 50%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>type</td>
+ <td>required</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Specify what connector to use, here should be <code>'iceberg'</code>.</td>
+ </tr>
+ <tr>
+ <td>name</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The name of the sink.</td>
+ </tr>
+ <tr>
+ <td>catalog.properties.type</td>
+ <td>required</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Metastore of iceberg catalog, supports hadoop and hive.</td>
+ </tr>
+ <tr>
+ <td>catalog.properties.warehouse</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The warehouse root path of catalog.</td>
+ </tr>
+ <tr>
+ <td>catalog.properties.uri</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Uri of metastore server.</td>
+ </tr>
+ <tr>
+ <td>partition.key</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Partition keys for each partitioned table, allow setting multiple primary keys for multiTables. Each table are separated by ';', and each partition key are separated by ','. For example, we can set partition.key of two tables by 'testdb.table1:id1,id2;testdb.table2:name'.</td>
+ </tr>
+ <tr>
+ <td>catalog.properties.*</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Pass options of Iceberg catalog to pipeline,See <a href="https://iceberg.apache.org/docs/nightly/flink-configuration/#catalog-configuration">Iceberg catalog options</a>. </td>
+ </tr>
+ <tr>
+ <td>table.properties.*</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Pass options of Iceberg table to pipeline,See <a href="https://iceberg.apache.org/docs/nightly/configuration/#write-properties">Iceberg table options</a>. </td>
+ </tr>
+ </tbody>
+</table>
+</div>
+
+Usage Notes
+--------
+
+* Only support Iceberg primary key table, so the source table must have primary keys.
+
+* Not support exactly-once. The connector uses at-least-once + primary key table for idempotent writing.
+
+Data Type Mapping
+----------------
+<div class="wy-table-responsive">
+<table class="colwidths-auto docutils">
+ <thead>
+ <tr>
+ <th class="text-left">CDC type</th>
+ <th class="text-left">Iceberg type</th>
+ <th class="text-left" style="width:60%;">NOTE</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>TINYINT</td>
+ <td>TINYINT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>SMALLINT</td>
+ <td>SMALLINT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>INT</td>
+ <td>INT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>BIGINT</td>
+ <td>BIGINT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>FLOAT</td>
+ <td>FLOAT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>DOUBLE</td>
+ <td>DOUBLE</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>DECIMAL(p, s)</td>
+ <td>DECIMAL(p, s)</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>BOOLEAN</td>
+ <td>BOOLEAN</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>DATE</td>
+ <td>DATE</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>TIMESTAMP</td>
+ <td>TIMESTAMP</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>TIMESTAMP_LTZ</td>
+ <td>TIMESTAMP_LTZ</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>CHAR(n)</td>
+ <td>CHAR(n)</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>VARCHAR(n)</td>
+ <td>VARCHAR(n)</td>
+ <td></td>
+ </tr>
+ </tbody>
+</table>
+</div>
+
+{{< top >}}
\ No newline at end of file
diff --git a/docs/content/docs/connectors/pipeline-connectors/iceberg.md b/docs/content/docs/connectors/pipeline-connectors/iceberg.md
new file mode 100644
index 000000000..30199c0f3
--- /dev/null
+++ b/docs/content/docs/connectors/pipeline-connectors/iceberg.md
@@ -0,0 +1,228 @@
+---
+title: "Iceberg"
+weight: 9
+type: docs
+aliases:
+- /connectors/pipeline-connectors/iceberg
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Iceberg Pipeline Connector
+
+The Iceberg Pipeline Connector functions as a *Data Sink* for data pipelines, enabling data writes to Apache Iceberg tables[Iceberg](https://iceberg.apache.org). This document explains how to configure the connector.
+
+## Key Capabilities
+* Automatic Table Creation
+Creates Iceberg tables dynamically when they do not exist
+* Schema Synchronization
+Propagates schema changes (e.g., column additions) from source systems to Iceberg
+* Data Replication
+Supports both batch and streaming data synchronization
+
+How to create Pipeline
+----------------
+
+The pipeline for reading data from MySQL and sink to Iceberg can be defined as follows:
+
+```yaml
+source:
+ type: mysql
+ name: MySQL Source
+ hostname: 127.0.0.1
+ port: 3306
+ username: admin
+ password: pass
+ tables: adb.\.*, bdb.user_table_[0-9]+, [app|web].order_\.*
+ server-id: 5401-5404
+
+sink:
+ type: iceberg
+ name: Iceberg Sink
+ catalog.properties.type: hadoop
+ catalog.properties.warehouse: /path/warehouse
+
+pipeline:
+ name: MySQL to Iceberg Pipeline
+ parallelism: 2
+```
+
+Pipeline Connector Options
+----------------
+<div class="highlight">
+<table class="colwidths-auto docutils">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 25%">Option</th>
+ <th class="text-left" style="width: 8%">Required</th>
+ <th class="text-left" style="width: 7%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 50%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>type</td>
+ <td>required</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Specify what connector to use, here should be <code>'iceberg'</code>.</td>
+ </tr>
+ <tr>
+ <td>name</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The name of the sink.</td>
+ </tr>
+ <tr>
+ <td>catalog.properties.type</td>
+ <td>required</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Metastore of iceberg catalog, supports hadoop and hive.</td>
+ </tr>
+ <tr>
+ <td>catalog.properties.warehouse</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The warehouse root path of catalog.</td>
+ </tr>
+ <tr>
+ <td>catalog.properties.uri</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Uri of metastore server.</td>
+ </tr>
+ <tr>
+ <td>partition.key</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Partition keys for each partitioned table, allow setting multiple primary keys for multiTables. Each table are separated by ';', and each partition key are separated by ','. For example, we can set partition.key of two tables by 'testdb.table1:id1,id2;testdb.table2:name'.</td>
+ </tr>
+ <tr>
+ <td>catalog.properties.*</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Pass options of Iceberg catalog to pipeline,See <a href="https://iceberg.apache.org/docs/nightly/flink-configuration/#catalog-configuration">Iceberg catalog options</a>. </td>
+ </tr>
+ <tr>
+ <td>table.properties.*</td>
+ <td>optional</td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Pass options of Iceberg table to pipeline,See <a href="https://iceberg.apache.org/docs/nightly/configuration/#write-properties">Iceberg table options</a>. </td>
+ </tr>
+ </tbody>
+</table>
+</div>
+
+Usage Notes
+--------
+
+* Only support Iceberg primary key table, so the source table must have primary keys.
+
+* Not support exactly-once. The connector uses at-least-once + primary key table for idempotent writing.
+
+Data Type Mapping
+----------------
+<div class="wy-table-responsive">
+<table class="colwidths-auto docutils">
+ <thead>
+ <tr>
+ <th class="text-left">CDC type</th>
+ <th class="text-left">Iceberg type</th>
+ <th class="text-left" style="width:60%;">NOTE</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>TINYINT</td>
+ <td>TINYINT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>SMALLINT</td>
+ <td>SMALLINT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>INT</td>
+ <td>INT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>BIGINT</td>
+ <td>BIGINT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>FLOAT</td>
+ <td>FLOAT</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>DOUBLE</td>
+ <td>DOUBLE</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>DECIMAL(p, s)</td>
+ <td>DECIMAL(p, s)</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>BOOLEAN</td>
+ <td>BOOLEAN</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>DATE</td>
+ <td>DATE</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>TIMESTAMP</td>
+ <td>TIMESTAMP</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>TIMESTAMP_LTZ</td>
+ <td>TIMESTAMP_LTZ</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>CHAR(n)</td>
+ <td>CHAR(n)</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>VARCHAR(n)</td>
+ <td>VARCHAR(n)</td>
+ <td></td>
+ </tr>
+ </tbody>
+</table>
+</div>
+
+{{< top >}}
\ No newline at end of file
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/pom.xml b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/pom.xml
new file mode 100644
index 000000000..a298f51c9
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/pom.xml
@@ -0,0 +1,294 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>flink-cdc-pipeline-connectors</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>flink-cdc-pipeline-connector-iceberg</artifactId>
+ <packaging>jar</packaging>
+
+ <name>flink-cdc-pipeline-connector-iceberg</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-flink-runtime-1.19</artifactId>
+ <version>${iceberg.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-composer</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils</artifactId>
+ <version>${flink.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>junit</artifactId>
+ <groupId>junit</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>junit-vintage-engine</artifactId>
+ <groupId>org.junit.vintage</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+
+ <!-- hadoop dependency -->
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <!-- For Java 9+ compatibility, refer to https://issues.apache.org/jira/browse/FLINK-17701 -->
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-compress</artifactId>
+ <groupId>org.apache.commons</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-io</artifactId>
+ <groupId>commons-io</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-io</artifactId>
+ <groupId>commons-io</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- hive dependency -->
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>${hive.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-compress</artifactId>
+ <groupId>org.apache.commons</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-io</artifactId>
+ <groupId>commons-io</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>junit</artifactId>
+ <groupId>junit</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.pentaho</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-druid</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-avatica</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-compress</artifactId>
+ <groupId>org.apache.commons</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-io</artifactId>
+ <groupId>commons-io</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven.shade.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>org.apache.iceberg:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.kafka</pattern>
+ <shadedPattern>shaded.iceberg.org.apache.kafka</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>test-jar</id>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSink.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSink.java
new file mode 100644
index 000000000..0581858da
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSink.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink;
+
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.sink.DataSink;
+import org.apache.flink.cdc.common.sink.EventSinkProvider;
+import org.apache.flink.cdc.common.sink.FlinkSinkProvider;
+import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.connectors.iceberg.sink.v2.IcebergSink;
+import org.apache.flink.cdc.connectors.iceberg.sink.v2.compaction.CompactionOptions;
+
+import java.io.Serializable;
+import java.time.ZoneId;
+import java.util.List;
+import java.util.Map;
+
+/** A {@link DataSink} for Apache Iceberg. */
+public class IcebergDataSink implements DataSink, Serializable {
+
+ // options for creating Iceberg catalog.
+ private final Map<String, String> catalogOptions;
+
+ // options for creating Iceberg table.
+ private final Map<String, String> tableOptions;
+
+ private final Map<TableId, List<String>> partitionMaps;
+
+ private final ZoneId zoneId;
+
+ public final String schemaOperatorUid;
+
+ public final CompactionOptions compactionOptions;
+
+ public IcebergDataSink(
+ Map<String, String> catalogOptions,
+ Map<String, String> tableOptions,
+ Map<TableId, List<String>> partitionMaps,
+ ZoneId zoneId,
+ String schemaOperatorUid,
+ CompactionOptions compactionOptions) {
+ this.catalogOptions = catalogOptions;
+ this.tableOptions = tableOptions;
+ this.partitionMaps = partitionMaps;
+ this.zoneId = zoneId;
+ this.schemaOperatorUid = schemaOperatorUid;
+ this.compactionOptions = compactionOptions;
+ }
+
+ @Override
+ public EventSinkProvider getEventSinkProvider() {
+ IcebergSink icebergEventSink =
+ new IcebergSink(catalogOptions, tableOptions, zoneId, compactionOptions);
+ return FlinkSinkProvider.of(icebergEventSink);
+ }
+
+ @Override
+ public MetadataApplier getMetadataApplier() {
+ return new IcebergMetadataApplier(catalogOptions, tableOptions, partitionMaps);
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSinkFactory.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSinkFactory.java
new file mode 100644
index 000000000..722f5f7bc
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSinkFactory.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink;
+
+import org.apache.flink.cdc.common.configuration.ConfigOption;
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.factories.DataSinkFactory;
+import org.apache.flink.cdc.common.factories.FactoryHelper;
+import org.apache.flink.cdc.common.pipeline.PipelineOptions;
+import org.apache.flink.cdc.common.sink.DataSink;
+import org.apache.flink.cdc.connectors.iceberg.sink.utils.OptionUtils;
+import org.apache.flink.cdc.connectors.iceberg.sink.v2.compaction.CompactionOptions;
+
+import org.apache.flink.shaded.guava31.com.google.common.collect.ImmutableMap;
+
+import org.apache.iceberg.CatalogProperties;
+
+import java.time.ZoneId;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.apache.flink.cdc.connectors.iceberg.sink.IcebergDataSinkOptions.PREFIX_CATALOG_PROPERTIES;
+import static org.apache.flink.cdc.connectors.iceberg.sink.IcebergDataSinkOptions.PREFIX_TABLE_PROPERTIES;
+
+/** A {@link DataSinkFactory} for Apache Iceberg. */
+public class IcebergDataSinkFactory implements DataSinkFactory {
+
+ public static final String IDENTIFIER = "iceberg";
+
+ // Hard code to disable catalog cache as we always need latest schema after schema change.
+ private static final Map<String, String> FIXED_CATALOG_OPTIONS =
+ ImmutableMap.of(CatalogProperties.CACHE_ENABLED, "false");
+
+ @Override
+ public DataSink createDataSink(Context context) {
+ FactoryHelper.createFactoryHelper(this, context)
+ .validateExcept(PREFIX_TABLE_PROPERTIES, PREFIX_CATALOG_PROPERTIES);
+
+ Map<String, String> allOptions = context.getFactoryConfiguration().toMap();
+ OptionUtils.printOptions(IDENTIFIER, allOptions);
+
+ Map<String, String> catalogOptions = new HashMap<>();
+ Map<String, String> tableOptions = new HashMap<>();
+ allOptions.forEach(
+ (key, value) -> {
+ if (key.startsWith(PREFIX_TABLE_PROPERTIES)) {
+ tableOptions.put(key.substring(PREFIX_TABLE_PROPERTIES.length()), value);
+ } else if (key.startsWith(IcebergDataSinkOptions.PREFIX_CATALOG_PROPERTIES)) {
+ catalogOptions.put(
+ key.substring(
+ IcebergDataSinkOptions.PREFIX_CATALOG_PROPERTIES.length()),
+ value);
+ }
+ });
+ catalogOptions.putAll(FIXED_CATALOG_OPTIONS);
+ ZoneId zoneId = ZoneId.systemDefault();
+ if (!Objects.equals(
+ context.getPipelineConfiguration().get(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE),
+ PipelineOptions.PIPELINE_LOCAL_TIME_ZONE.defaultValue())) {
+ zoneId =
+ ZoneId.of(
+ context.getPipelineConfiguration()
+ .get(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE));
+ }
+ String schemaOperatorUid =
+ context.getPipelineConfiguration()
+ .get(PipelineOptions.PIPELINE_SCHEMA_OPERATOR_UID);
+ CompactionOptions compactionOptions =
+ getCompactionStrategy(context.getFactoryConfiguration());
+
+ return new IcebergDataSink(
+ catalogOptions,
+ tableOptions,
+ new HashMap<>(),
+ zoneId,
+ schemaOperatorUid,
+ compactionOptions);
+ }
+
+ private CompactionOptions getCompactionStrategy(Configuration configuration) {
+ return CompactionOptions.builder()
+ .enabled(configuration.get(IcebergDataSinkOptions.SINK_COMPACTION_ENABLED))
+ .commitInterval(
+ configuration.get(IcebergDataSinkOptions.SINK_COMPACTION_COMMIT_INTERVAL))
+ .parallelism(configuration.get(IcebergDataSinkOptions.SINK_COMPACTION_PARALLELISM))
+ .build();
+ }
+
+ @Override
+ public String identifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return new HashSet<>();
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ Set<ConfigOption<?>> options = new HashSet<>();
+ options.add(IcebergDataSinkOptions.TYPE);
+ options.add(IcebergDataSinkOptions.WAREHOUSE);
+ options.add(IcebergDataSinkOptions.PARTITION_KEY);
+ options.add(IcebergDataSinkOptions.SINK_COMPACTION_ENABLED);
+ options.add(IcebergDataSinkOptions.SINK_COMPACTION_COMMIT_INTERVAL);
+ return options;
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSinkOptions.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSinkOptions.java
new file mode 100644
index 000000000..c20486345
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSinkOptions.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink;
+
+import org.apache.flink.cdc.common.annotation.Experimental;
+import org.apache.flink.cdc.common.configuration.ConfigOption;
+
+import static org.apache.flink.cdc.common.configuration.ConfigOptions.key;
+
+/** Config options for {@link IcebergDataSink}. */
+public class IcebergDataSinkOptions {
+
+ // prefix for passing properties for table creation.
+ public static final String PREFIX_TABLE_PROPERTIES = "table.properties.";
+
+ // prefix for passing properties for catalog creation.
+ public static final String PREFIX_CATALOG_PROPERTIES = "catalog.properties.";
+
+ public static final ConfigOption<String> TYPE =
+ key("catalog.properties.type")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Type of iceberg catalog, supports `hadoop` and `hive`.");
+
+ public static final ConfigOption<String> WAREHOUSE =
+ key("catalog.properties.warehouse")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The warehouse root path of catalog, only usable when catalog.properties.type is `hadoop`.");
+
+ public static final ConfigOption<String> PARTITION_KEY =
+ key("partition.key")
+ .stringType()
+ .defaultValue("")
+ .withDescription(
+ "Partition keys for each partitioned table, allow setting multiple primary keys for multiTables. "
+ + "Tables are separated by ';', and partition keys are separated by ','. "
+ + "For example, we can set partition.key of two tables by 'testdb.table1:id1,id2;testdb.table2:name'.");
+
+ @Experimental
+ public static final ConfigOption<Boolean> SINK_COMPACTION_ENABLED =
+ key("sink.compaction.enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Enable iceberg small file optimization. If there are too many tables after enabling it, data flow may be blocked."
+ + " Please enable it carefully or use table maintenance service to compaction small file");
+
+ @Experimental
+ public static final ConfigOption<Integer> SINK_COMPACTION_COMMIT_INTERVAL =
+ key("sink.compaction.commit.interval")
+ .intType()
+ .defaultValue(10)
+ .withDescription(
+ "The commit interval for file compaction of each Iceberg table.");
+
+ @Experimental
+ public static final ConfigOption<Integer> SINK_COMPACTION_PARALLELISM =
+ key("sink.compaction.commit.parallelism")
+ .intType()
+ .defaultValue(-1)
+ .withDescription(
+ "The parallelism for file compaction, default value is -1, which means that compaction parallelism is equal to sink writer parallelism.");
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergMetadataApplier.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergMetadataApplier.java
new file mode 100644
index 000000000..7427960f9
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergMetadataApplier.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink;
+
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DropColumnEvent;
+import org.apache.flink.cdc.common.event.RenameColumnEvent;
+import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.cdc.common.event.SchemaChangeEventType;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.event.visitor.SchemaChangeEventVisitor;
+import org.apache.flink.cdc.common.exceptions.SchemaEvolveException;
+import org.apache.flink.cdc.common.exceptions.UnsupportedSchemaChangeEventException;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.PhysicalColumn;
+import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.common.types.utils.DataTypeUtils;
+import org.apache.flink.cdc.connectors.iceberg.sink.utils.IcebergTypeUtils;
+
+import org.apache.flink.shaded.guava31.com.google.common.collect.Sets;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.UpdateSchema;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.flink.FlinkSchemaUtil;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.flink.cdc.common.utils.Preconditions.checkNotNull;
+
+/** A {@link MetadataApplier} for Apache Iceberg. */
+public class IcebergMetadataApplier implements MetadataApplier {
+
+ private static final Logger LOG = LoggerFactory.getLogger(IcebergMetadataApplier.class);
+
+ private transient Catalog catalog;
+
+ private final Map<String, String> catalogOptions;
+
+ // currently, we set table options for all tables using the same options.
+ private final Map<String, String> tableOptions;
+
+ private final Map<TableId, List<String>> partitionMaps;
+
+ private Set<SchemaChangeEventType> enabledSchemaEvolutionTypes;
+
+ public IcebergMetadataApplier(Map<String, String> catalogOptions) {
+ this(catalogOptions, new HashMap<>(), new HashMap<>());
+ }
+
+ public IcebergMetadataApplier(
+ Map<String, String> catalogOptions,
+ Map<String, String> tableOptions,
+ Map<TableId, List<String>> partitionMaps) {
+ this.catalogOptions = catalogOptions;
+ this.tableOptions = tableOptions;
+ this.partitionMaps = partitionMaps;
+ this.enabledSchemaEvolutionTypes = getSupportedSchemaEvolutionTypes();
+ }
+
+ @Override
+ public void applySchemaChange(SchemaChangeEvent schemaChangeEvent)
+ throws SchemaEvolveException {
+ if (catalog == null) {
+ catalog =
+ CatalogUtil.buildIcebergCatalog(
+ this.getClass().getSimpleName(), catalogOptions, new Configuration());
+ }
+ SchemaChangeEventVisitor.visit(
+ schemaChangeEvent,
+ addColumnEvent -> {
+ applyAddColumn(addColumnEvent);
+ return null;
+ },
+ alterColumnTypeEvent -> {
+ applyAlterColumnType(alterColumnTypeEvent);
+ return null;
+ },
+ createTableEvent -> {
+ applyCreateTable(createTableEvent);
+ return null;
+ },
+ dropColumnEvent -> {
+ applyDropColumn(dropColumnEvent);
+ return null;
+ },
+ dropTableEvent -> {
+ throw new UnsupportedSchemaChangeEventException(dropTableEvent);
+ },
+ renameColumnEvent -> {
+ applyRenameColumn(renameColumnEvent);
+ return null;
+ },
+ truncateTableEvent -> {
+ throw new UnsupportedSchemaChangeEventException(truncateTableEvent);
+ });
+ }
+
+ private void applyCreateTable(CreateTableEvent event) {
+ try {
+ TableIdentifier tableIdentifier = TableIdentifier.parse(event.tableId().identifier());
+
+ // Step1: Build Schema.
+ org.apache.flink.cdc.common.schema.Schema cdcSchema = event.getSchema();
+ List<Types.NestedField> columns = new ArrayList<>();
+ Set<Integer> identifierFieldIds = new HashSet<>();
+ for (int index = 0; index < event.getSchema().getColumnCount(); index++) {
+ columns.add(
+ IcebergTypeUtils.convertCdcColumnToIcebergField(
+ index, (PhysicalColumn) cdcSchema.getColumns().get(index)));
+ if (cdcSchema.primaryKeys().contains(cdcSchema.getColumns().get(index).getName())) {
+ identifierFieldIds.add(index);
+ }
+ }
+
+ // Step2: Build partition spec.
+ Schema icebergSchema = new Schema(columns, identifierFieldIds);
+ List<String> partitionColumns = cdcSchema.partitionKeys();
+ if (partitionMaps.containsKey(event.tableId())) {
+ partitionColumns = partitionMaps.get(event.tableId());
+ }
+ PartitionSpec.Builder builder = PartitionSpec.builderFor(icebergSchema);
+ for (String name : partitionColumns) {
+ // TODO Add more partition transforms, see
+ // https://iceberg.apache.org/spec/#partition-transforms.
+ builder.identity(name);
+ }
+ PartitionSpec partitionSpec = builder.build();
+ if (!catalog.tableExists(tableIdentifier)) {
+ catalog.createTable(tableIdentifier, icebergSchema, partitionSpec, tableOptions);
+ }
+ } catch (Exception e) {
+ throw new SchemaEvolveException(event, e.getMessage(), e);
+ }
+ }
+
+ private void applyAddColumn(AddColumnEvent event) {
+ TableIdentifier tableIdentifier = TableIdentifier.parse(event.tableId().identifier());
+ try {
+ Table table = catalog.loadTable(tableIdentifier);
+ applyAddColumnEventWithPosition(table, event);
+ } catch (Exception e) {
+ throw new SchemaEvolveException(event, e.getMessage(), e);
+ }
+ }
+
+ private void applyAddColumnEventWithPosition(Table table, AddColumnEvent event)
+ throws SchemaEvolveException {
+
+ try {
+ UpdateSchema updateSchema = table.updateSchema();
+ for (AddColumnEvent.ColumnWithPosition columnWithPosition : event.getAddedColumns()) {
+ Column addColumn = columnWithPosition.getAddColumn();
+ String columnName = addColumn.getName();
+ String columnComment = addColumn.getComment();
+ Type icebergType =
+ FlinkSchemaUtil.convert(
+ DataTypeUtils.toFlinkDataType(addColumn.getType())
+ .getLogicalType());
+ switch (columnWithPosition.getPosition()) {
+ case FIRST:
+ updateSchema.addColumn(columnName, icebergType, columnComment);
+ table.updateSchema().moveFirst(columnName);
+ break;
+ case LAST:
+ updateSchema.addColumn(columnName, icebergType, columnComment);
+ break;
+ case BEFORE:
+ updateSchema.addColumn(columnName, icebergType, columnComment);
+ updateSchema.moveBefore(
+ columnName, columnWithPosition.getExistedColumnName());
+ break;
+ case AFTER:
+ checkNotNull(
+ columnWithPosition.getExistedColumnName(),
+ "Existing column name must be provided for AFTER position");
+ updateSchema.addColumn(columnName, icebergType, columnComment);
+ updateSchema.moveAfter(
+ columnName, columnWithPosition.getExistedColumnName());
+ break;
+ default:
+ throw new SchemaEvolveException(
+ event,
+ "Unknown column position: " + columnWithPosition.getPosition());
+ }
+ }
+ updateSchema.commit();
+ } catch (Exception e) {
+ throw new SchemaEvolveException(event, e.getMessage(), e);
+ }
+ }
+
+ private void applyDropColumn(DropColumnEvent event) {
+ try {
+ UpdateSchema updateSchema =
+ catalog.loadTable(TableIdentifier.parse(event.tableId().identifier()))
+ .updateSchema();
+ event.getDroppedColumnNames().forEach(updateSchema::deleteColumn);
+ updateSchema.commit();
+ } catch (Exception e) {
+ throw new SchemaEvolveException(event, e.getMessage(), e);
+ }
+ }
+
+ private void applyRenameColumn(RenameColumnEvent event) {
+ try {
+ UpdateSchema updateSchema =
+ catalog.loadTable(TableIdentifier.parse(event.tableId().identifier()))
+ .updateSchema();
+ event.getNameMapping().forEach(updateSchema::renameColumn);
+ updateSchema.commit();
+ } catch (Exception e) {
+ throw new SchemaEvolveException(event, e.getMessage(), e);
+ }
+ }
+
+ private void applyAlterColumnType(AlterColumnTypeEvent event) {
+ try {
+ UpdateSchema updateSchema =
+ catalog.loadTable(TableIdentifier.parse(event.tableId().identifier()))
+ .updateSchema();
+ event.getTypeMapping()
+ .forEach(
+ (name, newType) -> {
+ Type.PrimitiveType type =
+ FlinkSchemaUtil.convert(
+ DataTypeUtils.toFlinkDataType(newType)
+ .getLogicalType())
+ .asPrimitiveType();
+ updateSchema.updateColumn(name, type);
+ });
+ updateSchema.commit();
+ } catch (Exception e) {
+ throw new SchemaEvolveException(event, e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public MetadataApplier setAcceptedSchemaEvolutionTypes(
+ Set<SchemaChangeEventType> schemaEvolutionTypes) {
+ this.enabledSchemaEvolutionTypes = schemaEvolutionTypes;
+ return this;
+ }
+
+ @Override
+ public boolean acceptsSchemaEvolutionType(SchemaChangeEventType schemaChangeEventType) {
+ return enabledSchemaEvolutionTypes.contains(schemaChangeEventType);
+ }
+
+ @Override
+ public Set<SchemaChangeEventType> getSupportedSchemaEvolutionTypes() {
+ return Sets.newHashSet(
+ SchemaChangeEventType.CREATE_TABLE,
+ SchemaChangeEventType.ADD_COLUMN,
+ SchemaChangeEventType.DROP_COLUMN,
+ SchemaChangeEventType.RENAME_COLUMN,
+ SchemaChangeEventType.ALTER_COLUMN_TYPE);
+ }
+
+ @Override
+ public void close() {
+ catalog = null;
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/utils/IcebergTypeUtils.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/utils/IcebergTypeUtils.java
new file mode 100644
index 000000000..470e7b15a
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/utils/IcebergTypeUtils.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.utils;
+
+import org.apache.flink.cdc.common.data.DecimalData;
+import org.apache.flink.cdc.common.data.RecordData;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.PhysicalColumn;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypeChecks;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.connectors.iceberg.sink.IcebergDataSink;
+import org.apache.flink.table.data.TimestampData;
+
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.flink.cdc.common.types.DataTypeChecks.getFieldCount;
+import static org.apache.flink.cdc.common.types.DataTypeChecks.getPrecision;
+
+/** Util class for types in {@link IcebergDataSink}. */
+public class IcebergTypeUtils {
+
+ /** Convert column from Flink CDC to Iceberg format. */
+ public static Types.NestedField convertCdcColumnToIcebergField(
+ int index, PhysicalColumn column) {
+ DataType dataType = column.getType();
+ return Types.NestedField.of(
+ index,
+ dataType.isNullable(),
+ column.getName(),
+ convertCDCTypeToIcebergType(dataType),
+ column.getComment());
+ }
+
+ /**
+ * Convert data type from Flink CDC to Iceberg format, refer to <a
+ * href="https://iceberg.apache.org/docs/nightly/flink/#flink-to-iceberg">...</a>.
+ */
+ public static Type convertCDCTypeToIcebergType(DataType type) {
+ // ordered by type root definition
+ List<DataType> children = type.getChildren();
+ int length = DataTypes.getLength(type).orElse(0);
+ int precision = DataTypes.getPrecision(type).orElse(0);
+ int scale = DataTypes.getScale(type).orElse(0);
+ switch (type.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ return new Types.StringType();
+ case BOOLEAN:
+ return new Types.BooleanType();
+ case BINARY:
+ case VARBINARY:
+ return new Types.BinaryType();
+ case DECIMAL:
+ return Types.DecimalType.of(precision, scale);
+ case TINYINT:
+ return new Types.BooleanType();
+ case SMALLINT:
+ case INTEGER:
+ return new Types.IntegerType();
+ case BIGINT:
+ return new Types.LongType();
+ case FLOAT:
+ return new Types.FloatType();
+ case DOUBLE:
+ return new Types.DoubleType();
+ case DATE:
+ return new Types.DateType();
+ case TIME_WITHOUT_TIME_ZONE:
+ return Types.TimeType.get();
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ return Types.TimestampType.withoutZone();
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ case TIMESTAMP_WITH_TIME_ZONE:
+ return Types.TimestampType.withZone();
+ default:
+ throw new IllegalArgumentException("Unsupported cdc type in iceberg: " + type);
+ }
+ }
+
+ public static List<RecordData.FieldGetter> createFieldGetters(Schema schema, ZoneId zoneId) {
+ List<Column> columns = schema.getColumns();
+ List<RecordData.FieldGetter> fieldGetters = new ArrayList<>(columns.size());
+ for (int i = 0; i < columns.size(); i++) {
+ fieldGetters.add(
+ IcebergTypeUtils.createFieldGetter(columns.get(i).getType(), i, zoneId));
+ }
+ return fieldGetters;
+ }
+
+ /** Create a {@link RecordData.FieldGetter} for the given {@link DataType}. */
+ public static RecordData.FieldGetter createFieldGetter(
+ DataType fieldType, int fieldPos, ZoneId zoneId) {
+ final RecordData.FieldGetter fieldGetter;
+ // ordered by type root definition
+ switch (fieldType.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ fieldGetter =
+ row ->
+ org.apache.flink.table.data.StringData.fromString(
+ row.getString(fieldPos).toString());
+ break;
+ case BOOLEAN:
+ fieldGetter = row -> row.getBoolean(fieldPos);
+ break;
+ case BINARY:
+ case VARBINARY:
+ fieldGetter = row -> row.getBinary(fieldPos);
+ break;
+ case DECIMAL:
+ final int decimalScale = DataTypeChecks.getScale(fieldType);
+ int precision = getPrecision(fieldType);
+ fieldGetter =
+ row -> {
+ DecimalData decimalData =
+ row.getDecimal(fieldPos, precision, decimalScale);
+ return org.apache.flink.table.data.DecimalData.fromBigDecimal(
+ decimalData.toBigDecimal(), precision, decimalScale);
+ };
+ break;
+ case TINYINT:
+ fieldGetter = row -> row.getBoolean(fieldPos);
+ break;
+ case SMALLINT:
+ fieldGetter = row -> row.getInt(fieldPos);
+ break;
+ case BIGINT:
+ fieldGetter = row -> row.getLong(fieldPos);
+ break;
+ case FLOAT:
+ fieldGetter = row -> row.getFloat(fieldPos);
+ break;
+ case DOUBLE:
+ fieldGetter = row -> row.getDouble(fieldPos);
+ break;
+ case INTEGER:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ fieldGetter = (row) -> row.getInt(fieldPos);
+ break;
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ fieldGetter =
+ (row) ->
+ TimestampData.fromTimestamp(
+ row.getTimestamp(fieldPos, getPrecision(fieldType))
+ .toTimestamp());
+ break;
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ fieldGetter =
+ row ->
+ TimestampData.fromInstant(
+ row.getLocalZonedTimestampData(
+ fieldPos,
+ DataTypeChecks.getPrecision(fieldType))
+ .toInstant());
+ break;
+ case TIMESTAMP_WITH_TIME_ZONE:
+ fieldGetter =
+ (row) ->
+ TimestampData.fromTimestamp(
+ row.getZonedTimestamp(fieldPos, getPrecision(fieldType))
+ .toTimestamp());
+ break;
+ case ROW:
+ final int rowFieldCount = getFieldCount(fieldType);
+ fieldGetter = row -> row.getRow(fieldPos, rowFieldCount);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "don't support type of " + fieldType.getTypeRoot());
+ }
+ if (!fieldType.isNullable()) {
+ return fieldGetter;
+ }
+ return row -> {
+ if (row.isNullAt(fieldPos)) {
+ return null;
+ }
+ return fieldGetter.getFieldOrNull(row);
+ };
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/utils/OptionUtils.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/utils/OptionUtils.java
new file mode 100644
index 000000000..14d2c4bf2
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/utils/OptionUtils.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.utils;
+
+import org.apache.flink.configuration.ConfigurationUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/** A utility class to print configuration of connectors. */
+public class OptionUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(OptionUtils.class);
+
+ /** Utility class can not be instantiated. */
+ private OptionUtils() {}
+
+ public static void printOptions(String identifier, Map<String, String> config) {
+ Map<String, String> hideMap = ConfigurationUtils.hideSensitiveValues(config);
+ LOG.info("Print {} connector configuration:", identifier);
+ for (String key : hideMap.keySet()) {
+ LOG.info("{} = {}", key, hideMap.get(key));
+ }
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/utils/RowDataUtils.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/utils/RowDataUtils.java
new file mode 100644
index 000000000..32ffa4a0a
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/utils/RowDataUtils.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.utils;
+
+import org.apache.flink.cdc.common.data.RecordData;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.types.RowKind;
+
+import java.util.List;
+
+/** Utils for convertion of {@link RowData} and {@link DataChangeEvent}. */
+public class RowDataUtils {
+
+ /** Convert {@link DataChangeEvent} to {@link RowData}. */
+ public static RowData convertDataChangeEventToRowData(
+ DataChangeEvent dataChangeEvent, List<RecordData.FieldGetter> fieldGetters) {
+ RecordData recordData;
+ RowKind kind;
+ switch (dataChangeEvent.op()) {
+ case INSERT:
+ case UPDATE:
+ case REPLACE:
+ {
+ recordData = dataChangeEvent.after();
+ kind = RowKind.INSERT;
+ break;
+ }
+ case DELETE:
+ {
+ recordData = dataChangeEvent.before();
+ kind = RowKind.DELETE;
+ break;
+ }
+ default:
+ throw new IllegalArgumentException("don't support type of " + dataChangeEvent.op());
+ }
+ GenericRowData genericRowData = new GenericRowData(recordData.getArity());
+ genericRowData.setRowKind(kind);
+ for (int i = 0; i < recordData.getArity(); i++) {
+ genericRowData.setField(i, fieldGetters.get(i).getFieldOrNull(recordData));
+ }
+ return genericRowData;
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergCommitter.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergCommitter.java
new file mode 100644
index 000000000..7d4eb5092
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergCommitter.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.api.connector.sink2.Committer;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.SinkCommitterMetricGroup;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.AppendFiles;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.RowDelta;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.io.WriteResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static java.util.stream.Collectors.toList;
+
+/** A {@link Committer} for Apache Iceberg. */
+public class IcebergCommitter implements Committer<WriteResultWrapper> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IcebergCommitter.class);
+
+ public static final String NAMESPACE_GROUP_KEY = "namespace";
+
+ public static final String SCHEMA_GROUP_KEY = "schema";
+
+ public static final String TABLE_GROUP_KEY = "table";
+
+ private final Catalog catalog;
+
+ private final SinkCommitterMetricGroup metricGroup;
+
+ private final Map<TableId, TableMetric> tableIdMetricMap;
+
+ public IcebergCommitter(Map<String, String> catalogOptions) {
+ this(catalogOptions, null);
+ }
+
+ public IcebergCommitter(
+ Map<String, String> catalogOptions, SinkCommitterMetricGroup metricGroup) {
+ this.catalog =
+ CatalogUtil.buildIcebergCatalog(
+ this.getClass().getSimpleName(), catalogOptions, new Configuration());
+ this.metricGroup = metricGroup;
+ this.tableIdMetricMap = new HashMap<>();
+ }
+
+ @Override
+ public void commit(Collection<CommitRequest<WriteResultWrapper>> collection) {
+ List<WriteResultWrapper> results =
+ collection.stream().map(CommitRequest::getCommittable).collect(toList());
+ commit(results);
+ }
+
+ private void commit(List<WriteResultWrapper> writeResultWrappers) {
+ Map<TableId, List<WriteResult>> tableMap = new HashMap<>();
+ for (WriteResultWrapper writeResultWrapper : writeResultWrappers) {
+ List<WriteResult> writeResult =
+ tableMap.getOrDefault(writeResultWrapper.getTableId(), new ArrayList<>());
+ writeResult.add(writeResultWrapper.getWriteResult());
+ tableMap.put(writeResultWrapper.getTableId(), writeResult);
+ LOGGER.info(writeResultWrapper.buildDescription());
+ }
+ for (Map.Entry<TableId, List<WriteResult>> entry : tableMap.entrySet()) {
+ TableId tableId = entry.getKey();
+ Optional<TableMetric> tableMetric = getTableMetric(tableId);
+ tableMetric.ifPresent(TableMetric::increaseCommitTimes);
+ Table table =
+ catalog.loadTable(
+ TableIdentifier.of(tableId.getSchemaName(), tableId.getTableName()));
+ List<WriteResult> results = entry.getValue();
+ List<DataFile> dataFiles =
+ results.stream()
+ .filter(payload -> payload.dataFiles() != null)
+ .flatMap(payload -> Arrays.stream(payload.dataFiles()))
+ .filter(dataFile -> dataFile.recordCount() > 0)
+ .collect(toList());
+ List<DeleteFile> deleteFiles =
+ results.stream()
+ .filter(payload -> payload.deleteFiles() != null)
+ .flatMap(payload -> Arrays.stream(payload.deleteFiles()))
+ .filter(deleteFile -> deleteFile.recordCount() > 0)
+ .collect(toList());
+ if (dataFiles.isEmpty() && deleteFiles.isEmpty()) {
+ LOGGER.info(String.format("Nothing to commit to table %s, skipping", table.name()));
+ } else {
+ if (deleteFiles.isEmpty()) {
+ AppendFiles append = table.newAppend();
+ dataFiles.forEach(append::appendFile);
+ append.commit();
+ } else {
+ RowDelta delta = table.newRowDelta();
+ dataFiles.forEach(delta::addRows);
+ deleteFiles.forEach(delta::addDeletes);
+ delta.commit();
+ }
+ }
+ }
+ }
+
+ private Optional<TableMetric> getTableMetric(TableId tableId) {
+ if (tableIdMetricMap.containsKey(tableId)) {
+ return Optional.of(tableIdMetricMap.get(tableId));
+ } else {
+ if (metricGroup == null) {
+ return Optional.empty();
+ }
+ MetricGroup tableIdMetricGroup =
+ metricGroup
+ .addGroup(NAMESPACE_GROUP_KEY, tableId.getNamespace())
+ .addGroup(SCHEMA_GROUP_KEY, tableId.getSchemaName())
+ .addGroup(TABLE_GROUP_KEY, tableId.getTableName());
+ TableMetric tableMetric = new TableMetric(tableIdMetricGroup);
+ tableIdMetricMap.put(tableId, tableMetric);
+ return Optional.of(tableMetric);
+ }
+ }
+
+ @Override
+ public void close() {}
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergSink.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergSink.java
new file mode 100644
index 000000000..02d693aba
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergSink.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Committer;
+import org.apache.flink.api.connector.sink2.CommitterInitContext;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
+import org.apache.flink.api.connector.sink2.WriterInitContext;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.connectors.iceberg.sink.v2.compaction.CompactionOperator;
+import org.apache.flink.cdc.connectors.iceberg.sink.v2.compaction.CompactionOptions;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.metrics.groups.SinkCommitterMetricGroup;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessageTypeInfo;
+import org.apache.flink.streaming.api.connector.sink2.CommittableWithLineage;
+import org.apache.flink.streaming.api.connector.sink2.WithPostCommitTopology;
+import org.apache.flink.streaming.api.connector.sink2.WithPreCommitTopology;
+import org.apache.flink.streaming.api.connector.sink2.WithPreWriteTopology;
+import org.apache.flink.streaming.api.datastream.DataStream;
+
+import java.time.ZoneId;
+import java.util.Map;
+import java.util.Objects;
+
+/** A {@link Sink} implementation for Apache Iceberg. */
+public class IcebergSink
+ implements Sink<Event>,
+ WithPreWriteTopology<Event>,
+ WithPreCommitTopology<Event, WriteResultWrapper>,
+ TwoPhaseCommittingSink<Event, WriteResultWrapper>,
+ WithPostCommitTopology<Event, WriteResultWrapper> {
+
+ protected final Map<String, String> catalogOptions;
+ protected final Map<String, String> tableOptions;
+
+ private final ZoneId zoneId;
+
+ private final CompactionOptions compactionOptions;
+
+ public IcebergSink(
+ Map<String, String> catalogOptions,
+ Map<String, String> tableOptions,
+ ZoneId zoneId,
+ CompactionOptions compactionOptions) {
+ this.catalogOptions = catalogOptions;
+ this.tableOptions = tableOptions;
+ this.zoneId = zoneId;
+ this.compactionOptions = compactionOptions;
+ }
+
+ @Override
+ public DataStream<Event> addPreWriteTopology(DataStream<Event> dataStream) {
+ return dataStream;
+ }
+
+ @Override
+ public Committer<WriteResultWrapper> createCommitter() {
+ return new IcebergCommitter(catalogOptions);
+ }
+
+ @Override
+ public Committer<WriteResultWrapper> createCommitter(
+ CommitterInitContext committerInitContext) {
+ SinkCommitterMetricGroup metricGroup = committerInitContext.metricGroup();
+ return new IcebergCommitter(catalogOptions, metricGroup);
+ }
+
+ @Override
+ public SimpleVersionedSerializer<WriteResultWrapper> getCommittableSerializer() {
+ return new WriteResultWrapperSerializer();
+ }
+
+ @Override
+ public SinkWriter<Event> createWriter(InitContext context) {
+ return new IcebergWriter(
+ catalogOptions,
+ context.getTaskInfo().getIndexOfThisSubtask(),
+ context.getTaskInfo().getAttemptNumber(),
+ zoneId);
+ }
+
+ @Override
+ public SinkWriter<Event> createWriter(WriterInitContext context) {
+ return new IcebergWriter(
+ catalogOptions,
+ context.getTaskInfo().getIndexOfThisSubtask(),
+ context.getTaskInfo().getAttemptNumber(),
+ zoneId);
+ }
+
+ @Override
+ public DataStream<CommittableMessage<WriteResultWrapper>> addPreCommitTopology(
+ DataStream<CommittableMessage<WriteResultWrapper>> committables) {
+ // Refer to
+ // https://github.com/apache/iceberg/blob/1d9fefeb9680d782dc128f242604903e71c32f97/flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/sink/IcebergSink.java#L106-L119.
+ return committables.global();
+ }
+
+ @Override
+ public SimpleVersionedSerializer<WriteResultWrapper> getWriteResultSerializer() {
+ return new WriteResultWrapperSerializer();
+ }
+
+ @Override
+ public void addPostCommitTopology(
+ DataStream<CommittableMessage<WriteResultWrapper>> committableMessageDataStream) {
+ if (compactionOptions.isEnabled()) {
+ TypeInformation<CommittableMessage<WriteResultWrapper>> typeInformation =
+ CommittableMessageTypeInfo.of(this::getCommittableSerializer);
+
+ int parallelism =
+ compactionOptions.getParallelism() == -1
+ ? committableMessageDataStream.getParallelism()
+ : compactionOptions.getParallelism();
+
+ // Shuffle by different table id.
+ DataStream<CommittableMessage<WriteResultWrapper>> keyedStream =
+ committableMessageDataStream.partitionCustom(
+ (bucket, numPartitions) -> bucket % numPartitions,
+ (committableMessage) -> {
+ if (committableMessage instanceof CommittableWithLineage) {
+ WriteResultWrapper multiTableCommittable =
+ ((CommittableWithLineage<WriteResultWrapper>)
+ committableMessage)
+ .getCommittable();
+ TableId tableId = multiTableCommittable.getTableId();
+ return tableId.hashCode();
+ } else {
+ return Objects.hash(committableMessage);
+ }
+ });
+
+ // Small file compaction.
+ keyedStream
+ .transform(
+ "Compaction",
+ typeInformation,
+ new CompactionOperator(catalogOptions, compactionOptions))
+ .setParallelism(parallelism);
+ }
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergWriter.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergWriter.java
new file mode 100644
index 000000000..2914f3bca
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergWriter.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.api.common.eventtime.Watermark;
+import org.apache.flink.api.connector.sink2.CommittingSinkWriter;
+import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.connectors.iceberg.sink.utils.RowDataUtils;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.RowType;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.flink.FlinkSchemaUtil;
+import org.apache.iceberg.flink.sink.RowDataTaskWriterFactory;
+import org.apache.iceberg.io.TaskWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** A {@link SinkWriter} for Apache Iceberg. */
+public class IcebergWriter implements CommittingSinkWriter<Event, WriteResultWrapper> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(IcebergWriter.class);
+
+ public static final String DEFAULT_FILE_FORMAT = "parquet";
+
+ public static final long DEFAULT_MAX_FILE_SIZE = 256 * 1024 * 1024;
+
+ private Map<TableId, RowDataTaskWriterFactory> writerFactoryMap;
+
+ private Map<TableId, TaskWriter<RowData>> writerMap;
+
+ private Map<TableId, TableSchemaWrapper> schemaMap;
+
+ private final List<WriteResultWrapper> temporaryWriteResult;
+
+ private Catalog catalog;
+
+ private final int taskId;
+
+ private final int attemptId;
+
+ private final ZoneId zoneId;
+
+ public IcebergWriter(
+ Map<String, String> catalogOptions, int taskId, int attemptId, ZoneId zoneId) {
+ catalog =
+ CatalogUtil.buildIcebergCatalog(
+ this.getClass().getSimpleName(), catalogOptions, new Configuration());
+ writerFactoryMap = new HashMap<>();
+ writerMap = new HashMap<>();
+ schemaMap = new HashMap<>();
+ temporaryWriteResult = new ArrayList<>();
+ this.taskId = taskId;
+ this.attemptId = attemptId;
+ this.zoneId = zoneId;
+ }
+
+ @Override
+ public Collection<WriteResultWrapper> prepareCommit() throws IOException, InterruptedException {
+ List<WriteResultWrapper> list = new ArrayList<>();
+ list.addAll(temporaryWriteResult);
+ list.addAll(getWriteResult());
+ temporaryWriteResult.clear();
+ return list;
+ }
+
+ private RowDataTaskWriterFactory getRowDataTaskWriterFactory(TableId tableId) {
+ Table table = catalog.loadTable(TableIdentifier.parse(tableId.identifier()));
+ RowType rowType = FlinkSchemaUtil.convert(table.schema());
+ RowDataTaskWriterFactory rowDataTaskWriterFactory =
+ new RowDataTaskWriterFactory(
+ table,
+ rowType,
+ DEFAULT_MAX_FILE_SIZE,
+ FileFormat.fromString(DEFAULT_FILE_FORMAT),
+ new HashMap<>(),
+ new ArrayList<>(table.schema().identifierFieldIds()),
+ true);
+ rowDataTaskWriterFactory.initialize(taskId, attemptId);
+ return rowDataTaskWriterFactory;
+ }
+
+ @Override
+ public void write(Event event, Context context) throws IOException {
+ if (event instanceof DataChangeEvent) {
+ DataChangeEvent dataChangeEvent = (DataChangeEvent) event;
+ TableId tableId = dataChangeEvent.tableId();
+ writerFactoryMap.computeIfAbsent(tableId, this::getRowDataTaskWriterFactory);
+ TaskWriter<RowData> writer =
+ writerMap.computeIfAbsent(
+ tableId, tableId1 -> writerFactoryMap.get(tableId1).create());
+ TableSchemaWrapper tableSchemaWrapper = schemaMap.get(tableId);
+ RowData rowData =
+ RowDataUtils.convertDataChangeEventToRowData(
+ dataChangeEvent, tableSchemaWrapper.getFieldGetters());
+ writer.write(rowData);
+ } else {
+ SchemaChangeEvent schemaChangeEvent = (SchemaChangeEvent) event;
+ TableId tableId = schemaChangeEvent.tableId();
+ TableSchemaWrapper tableSchemaWrapper = schemaMap.get(tableId);
+
+ Schema newSchema =
+ tableSchemaWrapper != null
+ ? SchemaUtils.applySchemaChangeEvent(
+ tableSchemaWrapper.getSchema(), schemaChangeEvent)
+ : SchemaUtils.applySchemaChangeEvent(null, schemaChangeEvent);
+ schemaMap.put(tableId, new TableSchemaWrapper(newSchema, zoneId));
+ }
+ }
+
+ @Override
+ public void flush(boolean flush) throws IOException {
+ // Notice: flush method may be called many times during one checkpoint.
+ temporaryWriteResult.addAll(getWriteResult());
+ }
+
+ private List<WriteResultWrapper> getWriteResult() throws IOException {
+ List<WriteResultWrapper> writeResults = new ArrayList<>();
+ for (Map.Entry<TableId, TaskWriter<RowData>> entry : writerMap.entrySet()) {
+ WriteResultWrapper writeResultWrapper =
+ new WriteResultWrapper(entry.getValue().complete(), entry.getKey());
+ writeResults.add(writeResultWrapper);
+ LOGGER.info(writeResultWrapper.buildDescription());
+ }
+ writerMap.clear();
+ writerFactoryMap.clear();
+ return writeResults;
+ }
+
+ @Override
+ public void writeWatermark(Watermark watermark) {}
+
+ @Override
+ public void close() throws Exception {
+ if (schemaMap != null) {
+ schemaMap.clear();
+ schemaMap = null;
+ }
+
+ if (writerMap != null) {
+ for (TaskWriter<RowData> writer : writerMap.values()) {
+ writer.close();
+ }
+ writerMap.clear();
+ writerMap = null;
+ }
+
+ if (writerFactoryMap != null) {
+ writerFactoryMap.clear();
+ writerFactoryMap = null;
+ }
+
+ catalog = null;
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/TableMetric.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/TableMetric.java
new file mode 100644
index 000000000..49b917738
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/TableMetric.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.metrics.MetricGroup;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/** Metrics of commit message for each table. */
+public class TableMetric {
+
+ /** Number of commit times. */
+ public static final String NUM_COMMIT_TIMES = "numCommitTimes";
+
+ private final AtomicInteger numCommitTimes = new AtomicInteger(0);
+
+ public TableMetric(MetricGroup metricGroup) {
+ metricGroup.gauge(NUM_COMMIT_TIMES, numCommitTimes::intValue);
+ }
+
+ public void increaseCommitTimes() {
+ numCommitTimes.incrementAndGet();
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/TableSchemaWrapper.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/TableSchemaWrapper.java
new file mode 100644
index 000000000..efa60a0da
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/TableSchemaWrapper.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.cdc.common.data.RecordData;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.connectors.iceberg.sink.utils.IcebergTypeUtils;
+
+import java.time.ZoneId;
+import java.util.List;
+
+/** A Wrapper for {@link Schema}. */
+public class TableSchemaWrapper {
+
+ private final Schema schema;
+
+ private final List<RecordData.FieldGetter> fieldGetters;
+
+ public TableSchemaWrapper(Schema schema, ZoneId zoneId) {
+ this.schema = schema;
+ this.fieldGetters = IcebergTypeUtils.createFieldGetters(schema, zoneId);
+ }
+
+ public Schema getSchema() {
+ return schema;
+ }
+
+ public List<RecordData.FieldGetter> getFieldGetters() {
+ return fieldGetters;
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/WriteResultWrapper.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/WriteResultWrapper.java
new file mode 100644
index 000000000..5aae210c9
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/WriteResultWrapper.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.cdc.common.event.TableId;
+
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.io.WriteResult;
+
+import java.io.Serializable;
+
+/** A wrapper class for {@link WriteResult} and {@link TableId}. */
+public class WriteResultWrapper implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final WriteResult writeResult;
+
+ private final TableId tableId;
+
+ public WriteResultWrapper(WriteResult writeResult, TableId tableId) {
+ this.writeResult = writeResult;
+ this.tableId = tableId;
+ }
+
+ public WriteResult getWriteResult() {
+ return writeResult;
+ }
+
+ public TableId getTableId() {
+ return tableId;
+ }
+
+ /** Build a simple description for the write result. */
+ public String buildDescription() {
+ long addCount = 0;
+ if (writeResult.dataFiles() != null) {
+ for (DataFile dataFile : writeResult.dataFiles()) {
+ addCount += dataFile.recordCount();
+ }
+ }
+ long deleteCount = 0;
+ if (writeResult.deleteFiles() != null) {
+ for (DeleteFile dataFile : writeResult.deleteFiles()) {
+ deleteCount += dataFile.recordCount();
+ }
+ }
+ return "WriteResult of "
+ + tableId
+ + ", AddCount: "
+ + addCount
+ + ", DeleteCount: "
+ + deleteCount;
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/WriteResultWrapperSerializer.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/WriteResultWrapperSerializer.java
new file mode 100644
index 000000000..ca8775d3e
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/WriteResultWrapperSerializer.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+/** Serializer for {@link WriteResultWrapper}. */
+class WriteResultWrapperSerializer implements SimpleVersionedSerializer<WriteResultWrapper> {
+ private static final int VERSION = 1;
+
+ @Override
+ public int getVersion() {
+ return VERSION;
+ }
+
+ @Override
+ public byte[] serialize(WriteResultWrapper writeResultWrapper) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(out);
+ byte[] result = InstantiationUtil.serializeObject(writeResultWrapper);
+ view.write(result);
+ return out.toByteArray();
+ }
+
+ @Override
+ public WriteResultWrapper deserialize(int version, byte[] serialized) throws IOException {
+ if (version == 1) {
+ DataInputDeserializer view = new DataInputDeserializer(serialized);
+ byte[] resultBuf = new byte[serialized.length];
+ view.read(resultBuf);
+ try {
+ return InstantiationUtil.deserializeObject(
+ resultBuf, WriteResultWrapperSerializer.class.getClassLoader());
+ } catch (ClassNotFoundException cnc) {
+ throw new IOException("Could not deserialize the WriteResult object", cnc);
+ }
+ }
+ throw new IOException("Unrecognized version or corrupt state: " + version);
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/compaction/CompactionOperator.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/compaction/CompactionOperator.java
new file mode 100644
index 000000000..abdbe4ea0
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/compaction/CompactionOperator.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2.compaction;
+
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.connectors.iceberg.sink.v2.WriteResultWrapper;
+import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.connector.sink2.CommittableWithLineage;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.actions.RewriteDataFilesActionResult;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.flink.actions.Actions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/** An Operator to trigger file compaction conditionally. */
+public class CompactionOperator
+ extends AbstractStreamOperator<CommittableMessage<WriteResultWrapper>>
+ implements OneInputStreamOperator<
+ CommittableMessage<WriteResultWrapper>, CommittableMessage<WriteResultWrapper>> {
+ protected static final Logger LOGGER = LoggerFactory.getLogger(CompactionOperator.class);
+
+ private final Map<String, String> catalogOptions;
+
+ private Catalog catalog;
+
+ /** store a list of MultiTableCommittable in one checkpoint. */
+ private final Map<TableId, Integer> tableCommitTimes;
+
+ private final Set<TableId> compactedTables;
+
+ private final CompactionOptions compactionOptions;
+
+ private volatile Throwable throwable;
+
+ private ExecutorService compactExecutor;
+
+ public CompactionOperator(
+ Map<String, String> catalogOptions, CompactionOptions compactionOptions) {
+ this.tableCommitTimes = new HashMap<>();
+ this.compactedTables = new HashSet<>();
+ this.catalogOptions = catalogOptions;
+ this.compactionOptions = compactionOptions;
+ }
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+ if (compactExecutor == null) {
+ this.compactExecutor =
+ Executors.newSingleThreadScheduledExecutor(
+ new ExecutorThreadFactory(
+ Thread.currentThread().getName() + "-Cdc-Compaction"));
+ }
+ }
+
+ @Override
+ public void processElement(StreamRecord<CommittableMessage<WriteResultWrapper>> element) {
+ if (element.getValue() instanceof CommittableWithLineage) {
+ TableId tableId =
+ ((CommittableWithLineage<WriteResultWrapper>) element.getValue())
+ .getCommittable()
+ .getTableId();
+ tableCommitTimes.put(tableId, tableCommitTimes.getOrDefault(tableId, 0) + 1);
+ int commitTimes = tableCommitTimes.get(tableId);
+ if (commitTimes >= compactionOptions.getCommitInterval()
+ && !compactedTables.contains(tableId)) {
+ if (throwable != null) {
+ throw new RuntimeException(throwable);
+ }
+ compactedTables.add(tableId);
+ if (compactExecutor == null) {
+ compact(tableId);
+ } else {
+ compactExecutor.submit(() -> compact(tableId));
+ }
+ }
+ }
+ }
+
+ private void compact(TableId tableId) {
+ if (catalog == null) {
+ catalog =
+ CatalogUtil.buildIcebergCatalog(
+ this.getClass().getSimpleName(), catalogOptions, new Configuration());
+ }
+ try {
+ RewriteDataFilesActionResult rewriteResult =
+ Actions.forTable(
+ StreamExecutionEnvironment.createLocalEnvironment(),
+ catalog.loadTable(TableIdentifier.parse(tableId.identifier())))
+ .rewriteDataFiles()
+ .execute();
+ LOGGER.info(
+ "Iceberg small file compact result for {}: added {} data files and deleted {} files.",
+ tableId.identifier(),
+ rewriteResult.addedDataFiles().size(),
+ rewriteResult.deletedDataFiles().size());
+ } catch (Throwable t) {
+ throwable = t;
+ }
+ }
+
+ @Override
+ public void snapshotState(StateSnapshotContext context) {
+ // Reset the commit times.
+ for (TableId tableId : compactedTables) {
+ tableCommitTimes.put(tableId, 0);
+ }
+ compactedTables.clear();
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ if (compactExecutor != null) {
+ compactExecutor.shutdown();
+ }
+ catalog = null;
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/compaction/CompactionOptions.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/compaction/CompactionOptions.java
new file mode 100644
index 000000000..bcea299fd
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/compaction/CompactionOptions.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2.compaction;
+
+import java.io.Serializable;
+
+/** Strategy for small file compaction. */
+public class CompactionOptions implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private int commitInterval = 1;
+ private boolean enabled = false;
+ private int parallelism = -1;
+
+ // Private constructor to enforce the use of the Builder
+ private CompactionOptions() {}
+
+ public void setCommitInterval(int commitInterval) {
+ this.commitInterval = commitInterval;
+ }
+
+ public int getCommitInterval() {
+ return commitInterval;
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ public int getParallelism() {
+ return parallelism;
+ }
+
+ public void setParallelism(int parallelism) {
+ this.parallelism = parallelism;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ /** Builder for {@link CompactionOptions}. */
+ public static class Builder {
+
+ private final CompactionOptions compactionOptions = new CompactionOptions();
+
+ public Builder commitInterval(int commitInterval) {
+ compactionOptions.setCommitInterval(commitInterval);
+ return this;
+ }
+
+ public Builder enabled(boolean enabled) {
+ compactionOptions.setEnabled(enabled);
+ return this;
+ }
+
+ public Builder parallelism(int parallelism) {
+ compactionOptions.setParallelism(parallelism);
+ return this;
+ }
+
+ public CompactionOptions build() {
+ return compactionOptions;
+ }
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory
new file mode 100644
index 000000000..2ebf93639
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/main/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.cdc.connectors.iceberg.sink.IcebergDataSinkFactory
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSinkFactoryTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSinkFactoryTest.java
new file mode 100644
index 000000000..8a5e5e4d0
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergDataSinkFactoryTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink;
+
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.factories.DataSinkFactory;
+import org.apache.flink.cdc.common.factories.FactoryHelper;
+import org.apache.flink.cdc.common.sink.DataSink;
+import org.apache.flink.cdc.composer.utils.FactoryDiscoveryUtils;
+import org.apache.flink.table.api.ValidationException;
+
+import org.apache.flink.shaded.guava31.com.google.common.collect.ImmutableMap;
+
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/** Tests for {@link IcebergDataSinkFactory}. */
+public class IcebergDataSinkFactoryTest {
+
+ @Test
+ void testCreateDataSink() {
+ DataSinkFactory sinkFactory =
+ FactoryDiscoveryUtils.getFactoryByIdentifier("iceberg", DataSinkFactory.class);
+ Assertions.assertThat(sinkFactory).isInstanceOf(IcebergDataSinkFactory.class);
+
+ Configuration conf = Configuration.fromMap(ImmutableMap.<String, String>builder().build());
+ conf.set(IcebergDataSinkOptions.WAREHOUSE, "/tmp/warehouse");
+ DataSink dataSink =
+ sinkFactory.createDataSink(
+ new FactoryHelper.DefaultContext(
+ conf, conf, Thread.currentThread().getContextClassLoader()));
+ Assertions.assertThat(dataSink).isInstanceOf(IcebergDataSink.class);
+ }
+
+ @Test
+ void testUnsupportedOption() {
+
+ DataSinkFactory sinkFactory =
+ FactoryDiscoveryUtils.getFactoryByIdentifier("iceberg", DataSinkFactory.class);
+ Assertions.assertThat(sinkFactory).isInstanceOf(IcebergDataSinkFactory.class);
+
+ Configuration conf =
+ Configuration.fromMap(
+ ImmutableMap.<String, String>builder()
+ .put("unsupported_key", "unsupported_value")
+ .build());
+
+ Assertions.assertThatThrownBy(
+ () ->
+ sinkFactory.createDataSink(
+ new FactoryHelper.DefaultContext(
+ conf,
+ conf,
+ Thread.currentThread().getContextClassLoader())))
+ .isInstanceOf(ValidationException.class)
+ .hasMessageContaining(
+ "Unsupported options found for 'iceberg'.\n\n"
+ + "Unsupported options:\n\n"
+ + "unsupported_key");
+ }
+
+ @Test
+ void testPrefixRequireOption() {
+ DataSinkFactory sinkFactory =
+ FactoryDiscoveryUtils.getFactoryByIdentifier("iceberg", DataSinkFactory.class);
+ Assertions.assertThat(sinkFactory).isInstanceOf(IcebergDataSinkFactory.class);
+
+ Configuration conf =
+ Configuration.fromMap(
+ ImmutableMap.<String, String>builder()
+ .put("catalog.properties.type", "hadoop")
+ .build());
+ DataSink dataSink =
+ sinkFactory.createDataSink(
+ new FactoryHelper.DefaultContext(
+ conf, conf, Thread.currentThread().getContextClassLoader()));
+ Assertions.assertThat(dataSink).isInstanceOf(IcebergDataSink.class);
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergMetadataApplierTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergMetadataApplierTest.java
new file mode 100644
index 000000000..05232c6f2
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/IcebergMetadataApplierTest.java
@@ -0,0 +1,413 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink;
+
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DropColumnEvent;
+import org.apache.flink.cdc.common.event.RenameColumnEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.schema.PhysicalColumn;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataTypes;
+
+import org.apache.flink.shaded.guava31.com.google.common.collect.ImmutableMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** A test for {@link IcebergMetadataApplier}. */
+public class IcebergMetadataApplierTest {
+
+ @TempDir public static java.nio.file.Path temporaryFolder;
+
+ @Test
+ public void testApplySchemaChange() {
+ Map<String, String> catalogOptions = new HashMap<>();
+ String warehouse =
+ new File(temporaryFolder.toFile(), UUID.randomUUID().toString()).toString();
+ catalogOptions.put("type", "hadoop");
+ catalogOptions.put("warehouse", warehouse);
+ catalogOptions.put("cache-enabled", "false");
+
+ Catalog catalog =
+ CatalogUtil.buildIcebergCatalog(
+ "cdc-iceberg-catalog", catalogOptions, new Configuration());
+
+ IcebergMetadataApplier icebergMetadataApplier = new IcebergMetadataApplier(catalogOptions);
+ String defaultTableId = "test.iceberg_table";
+ TableId tableId = TableId.parse(defaultTableId);
+
+ // Create Table.
+ CreateTableEvent createTableEvent =
+ new CreateTableEvent(
+ tableId,
+ Schema.newBuilder()
+ .physicalColumn(
+ "id",
+ DataTypes.BIGINT().notNull(),
+ "column for id",
+ "AUTO_DECREMENT()")
+ .physicalColumn(
+ "name",
+ DataTypes.VARCHAR(255).notNull(),
+ "column for name",
+ "John Smith")
+ .physicalColumn(
+ "tinyIntCol",
+ DataTypes.TINYINT(),
+ "column for tinyIntCol",
+ "1")
+ .physicalColumn(
+ "description",
+ DataTypes.STRING(),
+ "column for descriptions",
+ "not important")
+ .physicalColumn(
+ "bool_column",
+ DataTypes.BOOLEAN(),
+ "column for bool",
+ "false")
+ .physicalColumn(
+ "float_column",
+ DataTypes.FLOAT(),
+ "column for float",
+ "1.0")
+ .physicalColumn(
+ "double_column",
+ DataTypes.DOUBLE(),
+ "column for double",
+ "1.0")
+ .physicalColumn(
+ "decimal_column",
+ DataTypes.DECIMAL(10, 2),
+ "column for decimal",
+ "1.0")
+ .primaryKey("id")
+ .partitionKey("id", "name")
+ .build());
+ icebergMetadataApplier.applySchemaChange(createTableEvent);
+ Table table = catalog.loadTable(TableIdentifier.parse(defaultTableId));
+ org.apache.iceberg.Schema schema =
+ new org.apache.iceberg.Schema(
+ 0,
+ Arrays.asList(
+ Types.NestedField.of(
+ 1, false, "id", Types.LongType.get(), "column for id"),
+ Types.NestedField.of(
+ 2,
+ false,
+ "name",
+ Types.StringType.get(),
+ "column for name"),
+ Types.NestedField.of(
+ 3,
+ true,
+ "tinyIntCol",
+ Types.BooleanType.get(),
+ "column for tinyIntCol"),
+ Types.NestedField.of(
+ 4,
+ true,
+ "description",
+ Types.StringType.get(),
+ "column for descriptions"),
+ Types.NestedField.of(
+ 5,
+ true,
+ "bool_column",
+ Types.BooleanType.get(),
+ "column for bool"),
+ Types.NestedField.of(
+ 6,
+ true,
+ "float_column",
+ Types.FloatType.get(),
+ "column for float"),
+ Types.NestedField.of(
+ 7,
+ true,
+ "double_column",
+ Types.DoubleType.get(),
+ "column for double"),
+ Types.NestedField.of(
+ 8,
+ true,
+ "decimal_column",
+ Types.DecimalType.of(10, 2),
+ "column for decimal")),
+ new HashSet<>(Collections.singletonList(1)));
+ assertThat(table.schema().sameSchema(schema)).isTrue();
+
+ // Add column.
+ AddColumnEvent addColumnEvent =
+ new AddColumnEvent(
+ tableId,
+ Collections.singletonList(
+ AddColumnEvent.last(
+ new PhysicalColumn(
+ "newIntColumn",
+ DataTypes.INT(),
+ "comment for newIntColumn",
+ "not important"))));
+ icebergMetadataApplier.applySchemaChange(addColumnEvent);
+ table = catalog.loadTable(TableIdentifier.parse(defaultTableId));
+ schema =
+ new org.apache.iceberg.Schema(
+ 0,
+ Arrays.asList(
+ Types.NestedField.of(
+ 1, false, "id", Types.LongType.get(), "column for id"),
+ Types.NestedField.of(
+ 2,
+ false,
+ "name",
+ Types.StringType.get(),
+ "column for name"),
+ Types.NestedField.of(
+ 3,
+ true,
+ "tinyIntCol",
+ Types.BooleanType.get(),
+ "column for tinyIntCol"),
+ Types.NestedField.of(
+ 4,
+ true,
+ "description",
+ Types.StringType.get(),
+ "column for descriptions"),
+ Types.NestedField.of(
+ 5,
+ true,
+ "bool_column",
+ Types.BooleanType.get(),
+ "column for bool"),
+ Types.NestedField.of(
+ 6,
+ true,
+ "float_column",
+ Types.FloatType.get(),
+ "column for float"),
+ Types.NestedField.of(
+ 7,
+ true,
+ "double_column",
+ Types.DoubleType.get(),
+ "column for double"),
+ Types.NestedField.of(
+ 8,
+ true,
+ "decimal_column",
+ Types.DecimalType.of(10, 2),
+ "column for decimal"),
+ Types.NestedField.of(
+ 9,
+ true,
+ "newIntColumn",
+ Types.IntegerType.get(),
+ "comment for newIntColumn")),
+ new HashSet<>(Collections.singletonList(1)));
+ assertThat(table.schema().sameSchema(schema)).isTrue();
+
+ // Drop Column.
+ DropColumnEvent dropColumnEvent =
+ new DropColumnEvent(tableId, Collections.singletonList("description"));
+ icebergMetadataApplier.applySchemaChange(dropColumnEvent);
+ table = catalog.loadTable(TableIdentifier.parse(defaultTableId));
+ schema =
+ new org.apache.iceberg.Schema(
+ 0,
+ Arrays.asList(
+ Types.NestedField.of(
+ 1, false, "id", Types.LongType.get(), "column for id"),
+ Types.NestedField.of(
+ 2,
+ false,
+ "name",
+ Types.StringType.get(),
+ "column for name"),
+ Types.NestedField.of(
+ 3,
+ true,
+ "tinyIntCol",
+ Types.BooleanType.get(),
+ "column for tinyIntCol"),
+ Types.NestedField.of(
+ 5,
+ true,
+ "bool_column",
+ Types.BooleanType.get(),
+ "column for bool"),
+ Types.NestedField.of(
+ 6,
+ true,
+ "float_column",
+ Types.FloatType.get(),
+ "column for float"),
+ Types.NestedField.of(
+ 7,
+ true,
+ "double_column",
+ Types.DoubleType.get(),
+ "column for double"),
+ Types.NestedField.of(
+ 8,
+ true,
+ "decimal_column",
+ Types.DecimalType.of(10, 2),
+ "column for decimal"),
+ Types.NestedField.of(
+ 9,
+ true,
+ "newIntColumn",
+ Types.IntegerType.get(),
+ "comment for newIntColumn")),
+ new HashSet<>(Collections.singletonList(1)));
+ assertThat(table.schema().sameSchema(schema)).isTrue();
+
+ // Rename Column.
+ RenameColumnEvent renameColumnEvent =
+ new RenameColumnEvent(tableId, ImmutableMap.of("newIntColumn", "renamedIntColumn"));
+ icebergMetadataApplier.applySchemaChange(renameColumnEvent);
+ table = catalog.loadTable(TableIdentifier.parse(defaultTableId));
+ schema =
+ new org.apache.iceberg.Schema(
+ 0,
+ Arrays.asList(
+ Types.NestedField.of(
+ 1, false, "id", Types.LongType.get(), "column for id"),
+ Types.NestedField.of(
+ 2,
+ false,
+ "name",
+ Types.StringType.get(),
+ "column for name"),
+ Types.NestedField.of(
+ 3,
+ true,
+ "tinyIntCol",
+ Types.BooleanType.get(),
+ "column for tinyIntCol"),
+ Types.NestedField.of(
+ 5,
+ true,
+ "bool_column",
+ Types.BooleanType.get(),
+ "column for bool"),
+ Types.NestedField.of(
+ 6,
+ true,
+ "float_column",
+ Types.FloatType.get(),
+ "column for float"),
+ Types.NestedField.of(
+ 7,
+ true,
+ "double_column",
+ Types.DoubleType.get(),
+ "column for double"),
+ Types.NestedField.of(
+ 8,
+ true,
+ "decimal_column",
+ Types.DecimalType.of(10, 2),
+ "column for decimal"),
+ Types.NestedField.of(
+ 9,
+ true,
+ "renamedIntColumn",
+ Types.IntegerType.get(),
+ "comment for newIntColumn")),
+ new HashSet<>(Collections.singletonList(1)));
+ assertThat(table.schema().sameSchema(schema)).isTrue();
+
+ // Alter Column Type.
+ AlterColumnTypeEvent alterColumnTypeEvent =
+ new AlterColumnTypeEvent(
+ tableId, ImmutableMap.of("renamedIntColumn", DataTypes.BIGINT()));
+ icebergMetadataApplier.applySchemaChange(alterColumnTypeEvent);
+ table = catalog.loadTable(TableIdentifier.parse(defaultTableId));
+ schema =
+ new org.apache.iceberg.Schema(
+ 0,
+ Arrays.asList(
+ Types.NestedField.of(
+ 1, false, "id", Types.LongType.get(), "column for id"),
+ Types.NestedField.of(
+ 2,
+ false,
+ "name",
+ Types.StringType.get(),
+ "column for name"),
+ Types.NestedField.of(
+ 3,
+ true,
+ "tinyIntCol",
+ Types.BooleanType.get(),
+ "column for tinyIntCol"),
+ Types.NestedField.of(
+ 5,
+ true,
+ "bool_column",
+ Types.BooleanType.get(),
+ "column for bool"),
+ Types.NestedField.of(
+ 6,
+ true,
+ "float_column",
+ Types.FloatType.get(),
+ "column for float"),
+ Types.NestedField.of(
+ 7,
+ true,
+ "double_column",
+ Types.DoubleType.get(),
+ "column for double"),
+ Types.NestedField.of(
+ 8,
+ true,
+ "decimal_column",
+ Types.DecimalType.of(10, 2),
+ "column for decimal"),
+ Types.NestedField.of(
+ 9,
+ true,
+ "renamedIntColumn",
+ Types.LongType.get(),
+ "comment for newIntColumn")),
+ new HashSet<>(Collections.singletonList(1)));
+ assertThat(table.schema().sameSchema(schema)).isTrue();
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/CompactionOperatorTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/CompactionOperatorTest.java
new file mode 100644
index 000000000..1935335b9
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/CompactionOperatorTest.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.api.connector.sink2.Committer;
+import org.apache.flink.cdc.common.data.DecimalData;
+import org.apache.flink.cdc.common.data.RecordData;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.connectors.iceberg.sink.IcebergMetadataApplier;
+import org.apache.flink.cdc.connectors.iceberg.sink.v2.compaction.CompactionOperator;
+import org.apache.flink.cdc.connectors.iceberg.sink.v2.compaction.CompactionOptions;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+import org.apache.flink.streaming.api.connector.sink2.CommittableWithLineage;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.ZoneId;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/** Tests for {@link CompactionOperator}. */
+public class CompactionOperatorTest {
+
+ @TempDir public static java.nio.file.Path temporaryFolder;
+
+ @Test
+ public void testCompationOperator() throws IOException, InterruptedException {
+ Map<String, String> catalogOptions = new HashMap<>();
+ String warehouse =
+ new File(temporaryFolder.toFile(), UUID.randomUUID().toString()).toString();
+ catalogOptions.put("type", "hadoop");
+ catalogOptions.put("warehouse", warehouse);
+ catalogOptions.put("cache-enabled", "false");
+ Catalog catalog =
+ CatalogUtil.buildIcebergCatalog(
+ "cdc-iceberg-catalog", catalogOptions, new Configuration());
+ IcebergWriter icebergWriter =
+ new IcebergWriter(catalogOptions, 1, 1, ZoneId.systemDefault());
+ IcebergMetadataApplier icebergMetadataApplier = new IcebergMetadataApplier(catalogOptions);
+ TableId tableId = TableId.parse("test.iceberg_table");
+
+ // Create Table.
+ CreateTableEvent createTableEvent =
+ new CreateTableEvent(
+ tableId,
+ Schema.newBuilder()
+ .physicalColumn(
+ "id",
+ DataTypes.BIGINT().notNull(),
+ "column for id",
+ "AUTO_DECREMENT()")
+ .physicalColumn(
+ "name",
+ DataTypes.VARCHAR(255).notNull(),
+ "column for name",
+ "John Smith")
+ .physicalColumn("age", DataTypes.INT(), "column for age", "91")
+ .physicalColumn(
+ "description",
+ DataTypes.STRING(),
+ "column for descriptions",
+ "not important")
+ .physicalColumn(
+ "bool_column",
+ DataTypes.BOOLEAN(),
+ "column for bool",
+ "false")
+ .physicalColumn(
+ "float_column",
+ DataTypes.FLOAT(),
+ "column for float",
+ "1.0")
+ .physicalColumn(
+ "double_column",
+ DataTypes.DOUBLE(),
+ "column for double",
+ "1.0")
+ .physicalColumn(
+ "decimal_column",
+ DataTypes.DECIMAL(10, 2),
+ "column for decimal",
+ "1.0")
+ .physicalColumn(
+ "date_column", DataTypes.DATE(), "column for date", "12345")
+ .primaryKey("id")
+ .build());
+ icebergMetadataApplier.applySchemaChange(createTableEvent);
+ icebergWriter.write(createTableEvent, null);
+ BinaryRecordDataGenerator binaryRecordDataGenerator =
+ new BinaryRecordDataGenerator(
+ createTableEvent.getSchema().getColumnDataTypes().toArray(new DataType[0]));
+ IcebergCommitter icebergCommitter = new IcebergCommitter(catalogOptions);
+ // Commit many times.
+ int smallFileCount = 100;
+ for (long i = 0; i < smallFileCount; i++) {
+ RecordData recordData =
+ binaryRecordDataGenerator.generate(
+ new Object[] {
+ i,
+ BinaryStringData.fromString("Mark"),
+ 10,
+ BinaryStringData.fromString("test"),
+ true,
+ 1.0f,
+ 1.0d,
+ DecimalData.fromBigDecimal(new BigDecimal(1.0), 10, 2),
+ 9
+ });
+ icebergWriter.write(DataChangeEvent.insertEvent(tableId, recordData), null);
+ Collection<Committer.CommitRequest<WriteResultWrapper>> collection =
+ icebergWriter.prepareCommit().stream()
+ .map(IcebergWriterTest.MockCommitRequestImpl::new)
+ .collect(Collectors.toList());
+ icebergCommitter.commit(collection);
+ }
+ CompactionOperator compactionOperator =
+ new CompactionOperator(
+ catalogOptions, CompactionOptions.builder().commitInterval(1).build());
+ compactionOperator.processElement(
+ new StreamRecord<>(
+ new CommittableWithLineage<>(
+ new WriteResultWrapper(null, tableId), 0L, 0)));
+ Map<String, String> summary =
+ catalog.loadTable(TableIdentifier.parse(tableId.identifier()))
+ .currentSnapshot()
+ .summary();
+ Assertions.assertThat(summary.get("deleted-data-files"))
+ .isEqualTo(String.valueOf(smallFileCount));
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergSinkITCase.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergSinkITCase.java
new file mode 100644
index 000000000..cb8daccba
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergSinkITCase.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.schema.PhysicalColumn;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.common.types.RowType;
+import org.apache.flink.cdc.connectors.iceberg.sink.IcebergMetadataApplier;
+import org.apache.flink.cdc.connectors.iceberg.sink.v2.compaction.CompactionOptions;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.data.IcebergGenerics;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.UUID;
+
+/** A test for {@link IcebergSink}. */
+public class IcebergSinkITCase {
+
+ @TempDir public static java.nio.file.Path temporaryFolder;
+
+ private static StreamExecutionEnvironment env;
+
+ private static Catalog catalog;
+
+ private static final int DEFAULT_PARALLELISM = 1;
+
+ private static final long DEFAULT_TIMEOUT_MS = 30000;
+
+ private static final Map<String, String> catalogOptions = new HashMap<>();
+
+ private static final TableId tableId = TableId.tableId("test_db", "test_table");
+
+ @BeforeAll
+ public static void before() {
+ env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(DEFAULT_PARALLELISM);
+ env.enableCheckpointing(1000);
+ env.setRestartStrategy(RestartStrategies.noRestart());
+ String warehouse =
+ new File(temporaryFolder.toFile(), UUID.randomUUID().toString()).toString();
+ catalogOptions.put("type", "hadoop");
+ catalogOptions.put("warehouse", warehouse);
+ catalogOptions.put("cache-enabled", "false");
+ catalog =
+ CatalogUtil.buildIcebergCatalog(
+ "cdc-iceberg-catalog", catalogOptions, new Configuration());
+ }
+
+ @Test
+ public void testIcebergSink() throws Exception {
+ List<Event> events = new ArrayList<>();
+ events.addAll(generateEvents(tableId));
+ events.addAll(generateEvents(TableId.tableId("test_db", "test_table_2")));
+ events.addAll(generateEvents(tableId));
+ DataStream<Event> stream = env.fromData(events, TypeInformation.of(Event.class));
+
+ Sink<Event> icebergSink =
+ new IcebergSink(catalogOptions, null, null, CompactionOptions.builder().build());
+ String[] expected = new String[] {"21, 1.732, Disenchanted", "17, 6.28, Doris Day"};
+ stream.sinkTo(icebergSink);
+ env.execute("Values to Iceberg Sink");
+
+ long startTime = System.currentTimeMillis();
+ List<String> actual = new ArrayList<>();
+ while (actual.size() != expected.length) {
+ if (System.currentTimeMillis() - startTime > DEFAULT_TIMEOUT_MS) {
+ throw new RuntimeException("Timeout waiting for table to be populated");
+ }
+ Thread.sleep(1000);
+ actual = fetchTableContent(catalog, tableId);
+ }
+ Assertions.assertThat(actual).containsExactlyInAnyOrder(expected);
+
+ expected = new String[] {"21, 1.732, Disenchanted", "17, 6.28, Doris Day"};
+ actual = new ArrayList<>();
+ while (actual.size() != expected.length) {
+ if (System.currentTimeMillis() - startTime > DEFAULT_TIMEOUT_MS) {
+ throw new RuntimeException("Timeout waiting for table to be populated");
+ }
+ Thread.sleep(1000);
+ actual = fetchTableContent(catalog, TableId.tableId("test_db", "test_table_2"));
+ }
+ Assertions.assertThat(actual).containsExactlyInAnyOrder(expected);
+ }
+
+ private List<Event> generateEvents(TableId tableId) {
+ Schema schema =
+ Schema.newBuilder()
+ .column(new PhysicalColumn("id", DataTypes.INT().notNull(), null))
+ .column(new PhysicalColumn("number", DataTypes.DOUBLE(), null))
+ .column(new PhysicalColumn("name", DataTypes.VARCHAR(17), null))
+ .primaryKey("id")
+ .build();
+ IcebergMetadataApplier icebergMetadataApplier = new IcebergMetadataApplier(catalogOptions);
+ icebergMetadataApplier.applySchemaChange(new CreateTableEvent(tableId, schema));
+ BinaryRecordDataGenerator generator =
+ new BinaryRecordDataGenerator(
+ RowType.of(DataTypes.INT(), DataTypes.DOUBLE(), DataTypes.VARCHAR(17)));
+
+ return Arrays.asList(
+ new CreateTableEvent(tableId, schema),
+ DataChangeEvent.insertEvent(
+ tableId,
+ generator.generate(
+ new Object[] {17, 3.14, BinaryStringData.fromString("Doris Day")})),
+ DataChangeEvent.insertEvent(
+ tableId,
+ generator.generate(
+ new Object[] {
+ 19, 2.718, BinaryStringData.fromString("Que Sera Sera")
+ })),
+ DataChangeEvent.insertEvent(
+ tableId,
+ generator.generate(
+ new Object[] {
+ 21, 1.732, BinaryStringData.fromString("Disenchanted")
+ })),
+ DataChangeEvent.updateEvent(
+ tableId,
+ generator.generate(
+ new Object[] {17, 3.14, BinaryStringData.fromString("Doris Day")}),
+ generator.generate(
+ new Object[] {17, 6.28, BinaryStringData.fromString("Doris Day")})),
+ DataChangeEvent.deleteEvent(
+ tableId,
+ generator.generate(
+ new Object[] {
+ 19, 2.718, BinaryStringData.fromString("Que Sera Sera")
+ })));
+ }
+
+ private List<String> fetchTableContent(Catalog catalog, TableId tableId) {
+ List<String> results = new ArrayList<>();
+ Table table =
+ catalog.loadTable(
+ TableIdentifier.of(tableId.getSchemaName(), tableId.getTableName()));
+ org.apache.iceberg.Schema schema = table.schema();
+ CloseableIterable<Record> records = IcebergGenerics.read(table).project(schema).build();
+ for (Record record : records) {
+ List<String> fieldValues = new ArrayList<>();
+ for (Types.NestedField field : schema.columns()) {
+ String fieldValue = Objects.toString(record.getField(field.name()), "null");
+ fieldValues.add(fieldValue);
+ }
+ String joinedString = String.join(", ", fieldValues);
+ results.add(joinedString);
+ }
+ return results;
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergWriterTest.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergWriterTest.java
new file mode 100644
index 000000000..6b3305ff1
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/java/org/apache/flink/cdc/connectors/iceberg/sink/v2/IcebergWriterTest.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.iceberg.sink.v2;
+
+import org.apache.flink.api.connector.sink2.Committer;
+import org.apache.flink.cdc.common.data.DecimalData;
+import org.apache.flink.cdc.common.data.LocalZonedTimestampData;
+import org.apache.flink.cdc.common.data.RecordData;
+import org.apache.flink.cdc.common.data.TimestampData;
+import org.apache.flink.cdc.common.data.ZonedTimestampData;
+import org.apache.flink.cdc.common.data.binary.BinaryRecordData;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.schema.PhysicalColumn;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.connectors.iceberg.sink.IcebergMetadataApplier;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.metrics.groups.InternalSinkCommitterMetricGroup;
+import org.apache.flink.streaming.runtime.operators.sink.committables.CommitRequestImpl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.data.IcebergGenerics;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/** A test for {@link IcebergWriter}. */
+public class IcebergWriterTest {
+
+ @TempDir public static java.nio.file.Path temporaryFolder;
+
+ @Test
+ public void testWriteWithSchemaEvolution() throws Exception {
+ Map<String, String> catalogOptions = new HashMap<>();
+ String warehouse =
+ new File(temporaryFolder.toFile(), UUID.randomUUID().toString()).toString();
+ catalogOptions.put("type", "hadoop");
+ catalogOptions.put("warehouse", warehouse);
+ catalogOptions.put("cache-enabled", "false");
+ Catalog catalog =
+ CatalogUtil.buildIcebergCatalog(
+ "cdc-iceberg-catalog", catalogOptions, new Configuration());
+ IcebergWriter icebergWriter =
+ new IcebergWriter(catalogOptions, 1, 1, ZoneId.systemDefault());
+ IcebergMetadataApplier icebergMetadataApplier = new IcebergMetadataApplier(catalogOptions);
+ TableId tableId = TableId.parse("test.iceberg_table");
+
+ // Create Table.
+ CreateTableEvent createTableEvent =
+ new CreateTableEvent(
+ tableId,
+ Schema.newBuilder()
+ .physicalColumn(
+ "id",
+ DataTypes.BIGINT().notNull(),
+ "column for id",
+ "AUTO_DECREMENT()")
+ .physicalColumn(
+ "name",
+ DataTypes.VARCHAR(255).notNull(),
+ "column for name",
+ "John Smith")
+ .physicalColumn("age", DataTypes.INT(), "column for age", "91")
+ .physicalColumn(
+ "description",
+ DataTypes.STRING(),
+ "column for descriptions",
+ "not important")
+ .physicalColumn(
+ "bool_column",
+ DataTypes.BOOLEAN(),
+ "column for bool",
+ "false")
+ .physicalColumn(
+ "float_column",
+ DataTypes.FLOAT(),
+ "column for float",
+ "1.0")
+ .physicalColumn(
+ "double_column",
+ DataTypes.DOUBLE(),
+ "column for double",
+ "1.0")
+ .physicalColumn(
+ "decimal_column",
+ DataTypes.DECIMAL(10, 2),
+ "column for decimal",
+ "1.0")
+ .physicalColumn(
+ "date_column", DataTypes.DATE(), "column for date", "12345")
+ .primaryKey("id")
+ .partitionKey("id", "name")
+ .build());
+ icebergMetadataApplier.applySchemaChange(createTableEvent);
+ icebergWriter.write(createTableEvent, null);
+ BinaryRecordDataGenerator binaryRecordDataGenerator =
+ new BinaryRecordDataGenerator(
+ createTableEvent.getSchema().getColumnDataTypes().toArray(new DataType[0]));
+ RecordData recordData =
+ binaryRecordDataGenerator.generate(
+ new Object[] {
+ 1L,
+ BinaryStringData.fromString("Mark"),
+ 10,
+ BinaryStringData.fromString("test"),
+ true,
+ 1.0f,
+ 1.0d,
+ DecimalData.fromBigDecimal(new BigDecimal("1.0"), 10, 2),
+ 9
+ });
+ DataChangeEvent dataChangeEvent = DataChangeEvent.insertEvent(tableId, recordData);
+ icebergWriter.write(dataChangeEvent, null);
+ RecordData recordData2 =
+ binaryRecordDataGenerator.generate(
+ new Object[] {
+ 2L,
+ BinaryStringData.fromString("Bob"),
+ 10,
+ BinaryStringData.fromString("test"),
+ true,
+ 1.0f,
+ 1.0d,
+ DecimalData.fromBigDecimal(new BigDecimal("1.0"), 10, 2),
+ 9
+ });
+ DataChangeEvent dataChangeEvent2 = DataChangeEvent.insertEvent(tableId, recordData2);
+ icebergWriter.write(dataChangeEvent2, null);
+ Collection<WriteResultWrapper> writeResults = icebergWriter.prepareCommit();
+ IcebergCommitter icebergCommitter = new IcebergCommitter(catalogOptions);
+ Collection<Committer.CommitRequest<WriteResultWrapper>> collection =
+ writeResults.stream().map(MockCommitRequestImpl::new).collect(Collectors.toList());
+ icebergCommitter.commit(collection);
+ List<String> result = fetchTableContent(catalog, tableId);
+ Assertions.assertThat(result)
+ .containsExactlyInAnyOrder(
+ "1, Mark, 10, test, true, 1.0, 1.0, 1.00, 1970-01-10",
+ "2, Bob, 10, test, true, 1.0, 1.0, 1.00, 1970-01-10");
+
+ // Add column.
+ RecordData recordData3 =
+ binaryRecordDataGenerator.generate(
+ new Object[] {
+ 3L,
+ BinaryStringData.fromString("Bob"),
+ 10,
+ BinaryStringData.fromString("test"),
+ true,
+ 1.0f,
+ 1.0d,
+ DecimalData.fromBigDecimal(new BigDecimal("1.0"), 10, 2),
+ 9
+ });
+ DataChangeEvent dataChangeEvent3 = DataChangeEvent.insertEvent(tableId, recordData3);
+ icebergWriter.write(dataChangeEvent3, null);
+ RecordData recordData4 =
+ binaryRecordDataGenerator.generate(
+ new Object[] {
+ 4L,
+ BinaryStringData.fromString("Bob"),
+ 10,
+ BinaryStringData.fromString("test"),
+ true,
+ 1.0f,
+ 1.0d,
+ DecimalData.fromBigDecimal(new BigDecimal("1.0"), 10, 2),
+ 9
+ });
+ DataChangeEvent dataChangeEvent4 = DataChangeEvent.insertEvent(tableId, recordData4);
+ icebergWriter.write(dataChangeEvent4, null);
+ icebergWriter.flush(false);
+ AddColumnEvent addColumnEvent =
+ new AddColumnEvent(
+ tableId,
+ Arrays.asList(
+ AddColumnEvent.last(
+ new PhysicalColumn(
+ "newStringColumn",
+ DataTypes.STRING(),
+ "comment for newStringColumn",
+ "not important"))));
+ icebergMetadataApplier.applySchemaChange(addColumnEvent);
+ icebergWriter.write(addColumnEvent, null);
+ binaryRecordDataGenerator =
+ new BinaryRecordDataGenerator(
+ SchemaUtils.applySchemaChangeEvent(
+ createTableEvent.getSchema(), addColumnEvent)
+ .getColumnDataTypes()
+ .toArray(new DataType[0]));
+ RecordData recordData5 =
+ binaryRecordDataGenerator.generate(
+ new Object[] {
+ 5L,
+ BinaryStringData.fromString("Mark"),
+ 10,
+ BinaryStringData.fromString("test"),
+ true,
+ 1.0f,
+ 1.0d,
+ DecimalData.fromBigDecimal(new BigDecimal("1.0"), 10, 2),
+ 9,
+ BinaryStringData.fromString("newStringColumn"),
+ });
+ DataChangeEvent dataChangeEvent5 = DataChangeEvent.insertEvent(tableId, recordData5);
+ icebergWriter.write(dataChangeEvent5, null);
+ writeResults = icebergWriter.prepareCommit();
+ collection =
+ writeResults.stream().map(MockCommitRequestImpl::new).collect(Collectors.toList());
+ icebergCommitter.commit(collection);
+ result = fetchTableContent(catalog, tableId);
+ Assertions.assertThat(result)
+ .containsExactlyInAnyOrder(
+ "1, Mark, 10, test, true, 1.0, 1.0, 1.00, 1970-01-10, null",
+ "2, Bob, 10, test, true, 1.0, 1.0, 1.00, 1970-01-10, null",
+ "3, Bob, 10, test, true, 1.0, 1.0, 1.00, 1970-01-10, null",
+ "4, Bob, 10, test, true, 1.0, 1.0, 1.00, 1970-01-10, null",
+ "5, Mark, 10, test, true, 1.0, 1.0, 1.00, 1970-01-10, newStringColumn");
+ }
+
+ @Test
+ public void testWriteWithAllSupportedTypes() throws Exception {
+ Map<String, String> catalogOptions = new HashMap<>();
+ String warehouse =
+ new File(temporaryFolder.toFile(), UUID.randomUUID().toString()).toString();
+ catalogOptions.put("type", "hadoop");
+ catalogOptions.put("warehouse", warehouse);
+ catalogOptions.put("cache-enabled", "false");
+ Catalog catalog =
+ CatalogUtil.buildIcebergCatalog(
+ "cdc-iceberg-catalog", catalogOptions, new Configuration());
+ IcebergWriter icebergWriter =
+ new IcebergWriter(catalogOptions, 1, 1, ZoneId.systemDefault());
+ IcebergMetadataApplier icebergMetadataApplier = new IcebergMetadataApplier(catalogOptions);
+ TableId tableId = TableId.parse("test.iceberg_table");
+
+ // Create Table.
+ CreateTableEvent createTableEvent =
+ new CreateTableEvent(
+ tableId,
+ Schema.newBuilder()
+ .physicalColumn("char(5)", DataTypes.CHAR(5))
+ .physicalColumn("varchar(10)", DataTypes.VARCHAR(10))
+ .physicalColumn("string", DataTypes.STRING())
+ .physicalColumn("boolean", DataTypes.BOOLEAN())
+ .physicalColumn("binary(5)", DataTypes.BINARY(5))
+ .physicalColumn("varbinary(10)", DataTypes.BINARY(10))
+ .physicalColumn("decimal(10, 2)", DataTypes.DECIMAL(10, 2))
+ .physicalColumn("tinyint", DataTypes.TINYINT())
+ .physicalColumn("smallint", DataTypes.SMALLINT())
+ .physicalColumn("int", DataTypes.INT())
+ .physicalColumn("bigint", DataTypes.BIGINT())
+ .physicalColumn("float", DataTypes.FLOAT())
+ .physicalColumn("double", DataTypes.DOUBLE())
+ .physicalColumn("time", DataTypes.TIME())
+ .physicalColumn("date", DataTypes.DATE())
+ .physicalColumn("timestamp", DataTypes.TIMESTAMP())
+ .physicalColumn("timestamp_ltz", DataTypes.TIMESTAMP_LTZ())
+ .physicalColumn("timestamp_tz", DataTypes.TIMESTAMP_TZ())
+ .primaryKey("id")
+ .build());
+ icebergMetadataApplier.applySchemaChange(createTableEvent);
+ icebergWriter.write(createTableEvent, null);
+ BinaryRecordDataGenerator dataGenerator =
+ new BinaryRecordDataGenerator(
+ createTableEvent.getSchema().getColumnDataTypes().toArray(new DataType[0]));
+ BinaryRecordData record1 =
+ dataGenerator.generate(
+ new Object[] {
+ BinaryStringData.fromString("char"),
+ BinaryStringData.fromString("varchar"),
+ BinaryStringData.fromString("string"),
+ false,
+ new byte[] {1, 2, 3, 4, 5},
+ new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+ DecimalData.zero(10, 2),
+ (byte) 1,
+ (short) 2,
+ 12345,
+ 12345L,
+ 123.456f,
+ 123456.789d,
+ 12345,
+ 12345,
+ TimestampData.fromTimestamp(Timestamp.valueOf("1970-01-01 00:00:00")),
+ LocalZonedTimestampData.fromInstant(Instant.ofEpochSecond(0)),
+ ZonedTimestampData.fromZonedDateTime(
+ ZonedDateTime.ofInstant(
+ Instant.ofEpochSecond(0), ZoneId.of("Asia/Shanghai")))
+ });
+ DataChangeEvent dataChangeEvent = DataChangeEvent.insertEvent(tableId, record1);
+ icebergWriter.write(dataChangeEvent, null);
+ Collection<WriteResultWrapper> writeResults = icebergWriter.prepareCommit();
+ IcebergCommitter icebergCommitter = new IcebergCommitter(catalogOptions);
+ Collection<Committer.CommitRequest<WriteResultWrapper>> collection =
+ writeResults.stream().map(MockCommitRequestImpl::new).collect(Collectors.toList());
+ icebergCommitter.commit(collection);
+ List<String> result = fetchTableContent(catalog, tableId);
+ Assertions.assertThat(result)
+ .containsExactlyInAnyOrder(
+ "char, varchar, string, false, [1,2,3,4,5,], [1,2,3,4,5,6,7,8,9,10,], 0.00, true, 2, 12345, 12345, 123.456, 123456.789, 00:00:12.345, 2003-10-20, 1970-01-01T00:00, 1970-01-01T00:00Z, 1970-01-01T08:00Z");
+ }
+
+ /** Mock CommitRequestImpl. */
+ public static class MockCommitRequestImpl<CommT> extends CommitRequestImpl<CommT> {
+
+ protected MockCommitRequestImpl(CommT committable) {
+ super(
+ committable,
+ InternalSinkCommitterMetricGroup.wrap(
+ UnregisteredMetricsGroup.createOperatorMetricGroup()));
+ }
+ }
+
+ private List<String> fetchTableContent(Catalog catalog, TableId tableId) {
+ List<String> results = new ArrayList<>();
+ Table table =
+ catalog.loadTable(
+ TableIdentifier.of(tableId.getSchemaName(), tableId.getTableName()));
+ org.apache.iceberg.Schema schema = table.schema();
+ CloseableIterable<Record> records = IcebergGenerics.read(table).project(schema).build();
+ for (Record record : records) {
+ List<String> fieldValues = new ArrayList<>();
+ for (Types.NestedField field : schema.columns()) {
+ if (field.type() instanceof Types.BinaryType) {
+ java.nio.ByteBuffer byteBuffer =
+ (java.nio.ByteBuffer) record.getField(field.name());
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("[");
+ while (byteBuffer.hasRemaining()) {
+ stringBuilder.append(byteBuffer.get()).append(",");
+ }
+ stringBuilder.append("]");
+ fieldValues.add(stringBuilder.toString());
+ } else {
+ String fieldValue = Objects.toString(record.getField(field.name()), "null");
+ fieldValues.add(fieldValue);
+ }
+ }
+ String joinedString = String.join(", ", fieldValues);
+ results.add(joinedString);
+ }
+ return results;
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/resources/log4j2-test.properties b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/resources/log4j2-test.properties
new file mode 100644
index 000000000..5fe3b384b
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/src/test/resources/log4j2-test.properties
@@ -0,0 +1,26 @@
+################################################################################
+# Copyright 2023 Ververica Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Set root logger level to ERROR to not flood build logs
+# set manually to INFO for debugging purposes
+rootLogger.level=ERROR
+rootLogger.appenderRef.test.ref = TestLogger
+
+appender.testlogger.name = TestLogger
+appender.testlogger.type = CONSOLE
+appender.testlogger.target = SYSTEM_ERR
+appender.testlogger.layout.type = PatternLayout
+appender.testlogger.layout.pattern = %-4r [%t] %-5p %c - %m%n
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/pom.xml b/flink-cdc-connect/flink-cdc-pipeline-connectors/pom.xml
index 6557e2a9b..c384c240d 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/pom.xml
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/pom.xml
@@ -37,6 +37,7 @@ limitations under the License.
<module>flink-cdc-pipeline-connector-elasticsearch</module>
<module>flink-cdc-pipeline-connector-oceanbase</module>
<module>flink-cdc-pipeline-connector-maxcompute</module>
+ <module>flink-cdc-pipeline-connector-iceberg</module>
</modules>
<dependencies>
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/pom.xml b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/pom.xml
index 47bea3d27..3f577e83b 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/pom.xml
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/pom.xml
@@ -36,11 +36,12 @@ limitations under the License.
<!-- TODO: Update this, when StarRocks releases a 1.20 compatible connector. -->
<starrocks.connector.version>1.2.10_flink-${flink-major-1.19}</starrocks.connector.version>
<paimon.version>1.0.1</paimon.version>
- <flink.hadoop.version>3.3.4</flink.hadoop.version>
<flink.release.download.skip>false</flink.release.download.skip>
<flink.release.name>flink-${flink.version}-bin-scala_${scala.binary.version}.tgz</flink.release.name>
<flink.release.mirror>https://dlcdn.apache.org/flink/flink-${flink.version}</flink.release.mirror>
<maven.plugin.download.version>1.6.8</maven.plugin.download.version>
+ <iceberg.version>1.6.1</iceberg.version>
+ <hive.version>2.3.9</hive.version>
</properties>
<dependencies>
@@ -120,6 +121,13 @@ limitations under the License.
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-pipeline-connector-iceberg</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-cdc-pipeline-connector-kafka</artifactId>
@@ -158,6 +166,12 @@ limitations under the License.
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-flink-runtime-${flink-major-1.19}</artifactId>
+ <version>${iceberg.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-test-util</artifactId>
@@ -210,7 +224,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
- <version>${flink.hadoop.version}</version>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
@@ -232,7 +246,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${flink.hadoop.version}</version>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
@@ -253,7 +267,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
- <version>${flink.hadoop.version}</version>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
@@ -274,7 +288,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
- <version>${flink.hadoop.version}</version>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<type>test-jar</type>
<exclusions>
@@ -296,7 +310,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${flink.hadoop.version}</version>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<type>test-jar</type>
<exclusions>
@@ -524,6 +538,15 @@ limitations under the License.
<outputDirectory>${project.build.directory}/dependencies
</outputDirectory>
</artifactItem>
+ <artifactItem>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-pipeline-connector-iceberg</artifactId>
+ <version>${project.version}</version>
+ <destFileName>iceberg-cdc-pipeline-connector.jar</destFileName>
+ <type>jar</type>
+ <outputDirectory>${project.build.directory}/dependencies
+ </outputDirectory>
+ </artifactItem>
<artifactItem>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded-hadoop-2-uber</artifactId>
@@ -542,6 +565,15 @@ limitations under the License.
<outputDirectory>${project.build.directory}/dependencies
</outputDirectory>
</artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <destFileName>hive-exec.jar</destFileName>
+ <type>jar</type>
+ <outputDirectory>${project.build.directory}/dependencies
+ </outputDirectory>
+ </artifactItem>
<artifactItem>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-flink-${flink-major-1.20}</artifactId>
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToIcebergE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToIcebergE2eITCase.java
new file mode 100644
index 000000000..0f465da3e
--- /dev/null
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToIcebergE2eITCase.java
@@ -0,0 +1,383 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.pipeline.tests;
+
+import org.apache.flink.cdc.common.test.utils.TestUtils;
+import org.apache.flink.cdc.connectors.mysql.testutils.MySqlContainer;
+import org.apache.flink.cdc.connectors.mysql.testutils.MySqlVersion;
+import org.apache.flink.cdc.connectors.mysql.testutils.UniqueDatabase;
+import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.data.IcebergGenerics;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.containers.output.ToStringConsumer;
+import org.testcontainers.lifecycle.Startables;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/** End-to-end tests for mysql cdc to Iceberg pipeline job. */
+public class MySqlToIcebergE2eITCase extends PipelineTestEnvironment {
+ private static final Logger LOG = LoggerFactory.getLogger(MySqlToIcebergE2eITCase.class);
+
+ public static final Duration TESTCASE_TIMEOUT = Duration.ofMinutes(3);
+
+ // ------------------------------------------------------------------------------------------
+ // MySQL Variables (we always use MySQL as the data source for easier verifying)
+ // ------------------------------------------------------------------------------------------
+ protected static final String MYSQL_TEST_USER = "mysqluser";
+ protected static final String MYSQL_TEST_PASSWORD = "mysqlpw";
+
+ @org.testcontainers.junit.jupiter.Container
+ public static final MySqlContainer MYSQL =
+ (MySqlContainer)
+ new MySqlContainer(
+ MySqlVersion.V8_0) // v8 support both ARM and AMD architectures
+ .withConfigurationOverride("docker/mysql/my.cnf")
+ .withSetupSQL("docker/mysql/setup.sql")
+ .withDatabaseName("flink-test")
+ .withUsername("flinkuser")
+ .withPassword("flinkpw")
+ .withNetwork(NETWORK)
+ .withNetworkAliases("mysql")
+ .withLogConsumer(new Slf4jLogConsumer(LOG));
+
+ protected final UniqueDatabase inventoryDatabase =
+ new UniqueDatabase(MYSQL, "iceberg_inventory", MYSQL_TEST_USER, MYSQL_TEST_PASSWORD);
+
+ private String warehouse;
+
+ @BeforeAll
+ public static void initializeContainers() {
+ LOG.info("Starting containers...");
+ Startables.deepStart(Stream.of(MYSQL)).join();
+ LOG.info("Containers are started.");
+ }
+
+ @BeforeEach
+ public void before() throws Exception {
+ LOG.info("Starting containers...");
+ Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxrwxrwx");
+ FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
+ warehouse =
+ Files.createDirectory(temporaryFolder.resolve(UUID.randomUUID().toString()), attr)
+ .toString();
+ jobManagerConsumer = new ToStringConsumer();
+ jobManager =
+ new GenericContainer<>(getFlinkDockerImageTag())
+ .withCommand("jobmanager")
+ .withNetwork(NETWORK)
+ .withNetworkAliases(INTER_CONTAINER_JM_ALIAS)
+ .withExposedPorts(JOB_MANAGER_REST_PORT)
+ .withEnv("FLINK_PROPERTIES", FLINK_PROPERTIES)
+ .withCreateContainerCmdModifier(cmd -> cmd.withVolumes(sharedVolume))
+ .withFileSystemBind(warehouse, warehouse, BindMode.READ_WRITE)
+ .withLogConsumer(jobManagerConsumer);
+ Startables.deepStart(Stream.of(jobManager)).join();
+ LOG.info("JobManager is started.");
+ runInContainerAsRoot(jobManager, "chmod", "0777", "-R", sharedVolume.toString());
+ runInContainerAsRoot(jobManager, "chmod", "0777", "-R", warehouse);
+
+ taskManagerConsumer = new ToStringConsumer();
+ taskManager =
+ new GenericContainer<>(getFlinkDockerImageTag())
+ .withCommand("taskmanager")
+ .withNetwork(NETWORK)
+ .withNetworkAliases(INTER_CONTAINER_TM_ALIAS)
+ .withEnv("FLINK_PROPERTIES", FLINK_PROPERTIES)
+ .dependsOn(jobManager)
+ .withVolumesFrom(jobManager, BindMode.READ_WRITE)
+ .withFileSystemBind(warehouse, warehouse, BindMode.READ_WRITE)
+ .withLogConsumer(taskManagerConsumer);
+ Startables.deepStart(Stream.of(taskManager)).join();
+ LOG.info("TaskManager is started.");
+ runInContainerAsRoot(taskManager, "chmod", "0777", "-R", sharedVolume.toString());
+ runInContainerAsRoot(taskManager, "chmod", "0777", "-R", warehouse);
+ inventoryDatabase.createAndInitialize();
+ }
+
+ @AfterEach
+ public void after() {
+ try {
+ super.after();
+ inventoryDatabase.dropDatabase();
+ } catch (Exception e) {
+ LOG.error("Failed to clean up resources", e);
+ }
+ }
+
+ @Test
+ public void testSyncWholeDatabase() throws Exception {
+ String database = inventoryDatabase.getDatabaseName();
+ String pipelineJob =
+ String.format(
+ "source:\n"
+ + " type: mysql\n"
+ + " hostname: mysql\n"
+ + " port: 3306\n"
+ + " username: %s\n"
+ + " password: %s\n"
+ + " tables: %s.\\.*\n"
+ + " server-id: 5400-5404\n"
+ + " server-time-zone: UTC\n"
+ + "\n"
+ + "sink:\n"
+ + " type: iceberg\n"
+ + " catalog.properties.warehouse: %s\n"
+ + " catalog.properties.type: hadoop\n"
+ + "\n"
+ + "pipeline:\n"
+ + " schema.change.behavior: evolve\n"
+ + " parallelism: %s",
+ MYSQL_TEST_USER, MYSQL_TEST_PASSWORD, database, warehouse, parallelism);
+ Path mysqlCdcJar = TestUtils.getResource("mysql-cdc-pipeline-connector.jar");
+ Path icebergCdcConnector = TestUtils.getResource("iceberg-cdc-pipeline-connector.jar");
+ Path hadoopJar = TestUtils.getResource("flink-shade-hadoop.jar");
+ Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar");
+ submitPipelineJob(pipelineJob, mysqlCdcJar, icebergCdcConnector, mysqlDriverJar, hadoopJar);
+ waitUntilJobRunning(Duration.ofSeconds(60));
+ LOG.info("Pipeline job is running");
+ validateSinkResult(
+ warehouse,
+ database,
+ "products",
+ Arrays.asList(
+ "101, One, Alice, 3.202, red, {\"key1\": \"value1\"}, null",
+ "102, Two, Bob, 1.703, white, {\"key2\": \"value2\"}, null",
+ "103, Three, Cecily, 4.105, red, {\"key3\": \"value3\"}, null",
+ "104, Four, Derrida, 1.857, white, {\"key4\": \"value4\"}, null",
+ "105, Five, Evelyn, 5.211, red, {\"K\": \"V\", \"k\": \"v\"}, null",
+ "106, Six, Ferris, 9.813, null, null, null",
+ "107, Seven, Grace, 2.117, null, null, null",
+ "108, Eight, Hesse, 6.819, null, null, null",
+ "109, Nine, IINA, 5.223, null, null, null"));
+
+ validateSinkResult(
+ warehouse,
+ database,
+ "customers",
+ Arrays.asList(
+ "101, user_1, Shanghai, 123567891234",
+ "102, user_2, Shanghai, 123567891234",
+ "103, user_3, Shanghai, 123567891234",
+ "104, user_4, Shanghai, 123567891234"));
+
+ LOG.info("Begin incremental reading stage.");
+ // generate binlogs
+ String mysqlJdbcUrl =
+ String.format(
+ "jdbc:mysql://%s:%s/%s",
+ MYSQL.getHost(), MYSQL.getDatabasePort(), database);
+ List<String> recordsInIncrementalPhase;
+ try (Connection conn =
+ DriverManager.getConnection(
+ mysqlJdbcUrl, MYSQL_TEST_USER, MYSQL_TEST_PASSWORD);
+ Statement stat = conn.createStatement()) {
+
+ stat.execute(
+ "INSERT INTO products VALUES (default,'Ten','Jukebox',0.2, null, null, null);"); // 110
+ stat.execute("UPDATE products SET description='Fay' WHERE id=106;");
+ stat.execute("UPDATE products SET weight='5.125' WHERE id=107;");
+
+ // modify table schema
+ stat.execute("ALTER TABLE products DROP COLUMN point_c;");
+ stat.execute("DELETE FROM products WHERE id=101;");
+
+ stat.execute(
+ "INSERT INTO products VALUES (default,'Eleven','Kryo',5.18, null, null);"); // 111
+ stat.execute(
+ "INSERT INTO products VALUES (default,'Twelve', 'Lily', 2.14, null, null);"); // 112
+ recordsInIncrementalPhase = createChangesAndValidate(stat);
+ } catch (SQLException e) {
+ LOG.error("Update table for CDC failed.", e);
+ throw e;
+ }
+ List<String> recordsInSnapshotPhase =
+ new ArrayList<>(
+ Arrays.asList(
+ "102, Two, Bob, 1.703, white, {\"key2\": \"value2\"}, null, null, null, null, null, null, null, null, null, null",
+ "103, Three, Cecily, 4.105, red, {\"key3\": \"value3\"}, null, null, null, null, null, null, null, null, null, null",
+ "104, Four, Derrida, 1.857, white, {\"key4\": \"value4\"}, null, null, null, null, null, null, null, null, null, null",
+ "105, Five, Evelyn, 5.211, red, {\"K\": \"V\", \"k\": \"v\"}, null, null, null, null, null, null, null, null, null, null",
+ "106, Six, Fay, 9.813, null, null, null, null, null, null, null, null, null, null, null, null",
+ "107, Seven, Grace, 5.125, null, null, null, null, null, null, null, null, null, null, null, null",
+ "108, Eight, Hesse, 6.819, null, null, null, null, null, null, null, null, null, null, null, null",
+ "109, Nine, IINA, 5.223, null, null, null, null, null, null, null, null, null, null, null, null",
+ "110, Ten, Jukebox, 0.2, null, null, null, null, null, null, null, null, null, null, null, null",
+ "111, Eleven, Kryo, 5.18, null, null, null, null, null, null, null, null, null, null, null, null",
+ "112, Twelve, Lily, 2.14, null, null, null, null, null, null, null, null, null, null, null, null"));
+ recordsInSnapshotPhase.addAll(recordsInIncrementalPhase);
+ recordsInSnapshotPhase =
+ recordsInSnapshotPhase.stream().sorted().collect(Collectors.toList());
+ validateSinkResult(warehouse, database, "products", recordsInSnapshotPhase);
+ }
+
+ /**
+ * Basic Schema: id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, description VARCHAR(512),
+ * weight FLOAT, enum_c enum('red', 'white'), json_c JSON.
+ */
+ private List<String> createChangesAndValidate(Statement stat) throws SQLException {
+ List<String> result = new ArrayList<>();
+ StringBuilder sqlFields = new StringBuilder();
+
+ // Add Column.
+ for (int addColumnRepeat = 0; addColumnRepeat < 10; addColumnRepeat++) {
+ stat.execute(
+ String.format(
+ "ALTER TABLE products ADD COLUMN point_c_%s VARCHAR(10);",
+ addColumnRepeat));
+ sqlFields.append(", '1'");
+ StringBuilder resultFields = new StringBuilder();
+ for (int addedFieldCount = 0; addedFieldCount < 10; addedFieldCount++) {
+ if (addedFieldCount <= addColumnRepeat) {
+ resultFields.append(", 1");
+ } else {
+ resultFields.append(", null");
+ }
+ }
+ for (int statementCount = 0; statementCount < 1000; statementCount++) {
+ stat.addBatch(
+ String.format(
+ "INSERT INTO products VALUES (default,'finally', null, 2.14, null, null %s);",
+ sqlFields));
+ int id = addColumnRepeat * 1000 + statementCount + 113;
+ result.add(
+ String.format("%s, finally, null, 2.14, null, null%s", id, resultFields));
+ }
+ stat.executeBatch();
+ }
+
+ // Modify Column type.
+ for (int modifyColumnRepeat = 0; modifyColumnRepeat < 10; modifyColumnRepeat++) {
+ for (int statementCount = 0; statementCount < 1000; statementCount++) {
+ stat.addBatch(
+ String.format(
+ "INSERT INTO products VALUES (default,'finally', null, 2.14, null, null %s);",
+ sqlFields));
+ int id = modifyColumnRepeat * 1000 + statementCount + 10113;
+ result.add(
+ String.format(
+ "%s, finally, null, 2.14, null, null%s",
+ id, ", 1, 1, 1, 1, 1, 1, 1, 1, 1, 1"));
+ }
+ stat.executeBatch();
+ stat.execute(
+ String.format(
+ "ALTER TABLE products MODIFY point_c_0 VARCHAR(%s);",
+ 10 + modifyColumnRepeat));
+ }
+
+ return result;
+ }
+
+ private List<String> fetchIcebergTableRows(
+ String warehouse, String databaseName, String tableName) throws Exception {
+ Map<String, String> catalogOptions = new HashMap<>();
+ catalogOptions.put("type", "hadoop");
+ catalogOptions.put("warehouse", warehouse);
+ catalogOptions.put("cache-enabled", "false");
+ Catalog catalog =
+ CatalogUtil.buildIcebergCatalog(
+ "cdc-iceberg-catalog", catalogOptions, new Configuration());
+ List<String> results = new ArrayList<>();
+ Table table = catalog.loadTable(TableIdentifier.of(databaseName, tableName));
+ org.apache.iceberg.Schema schema = table.schema();
+ CloseableIterable<Record> records = IcebergGenerics.read(table).project(schema).build();
+ for (Record record : records) {
+ List<String> fieldValues = new ArrayList<>();
+ for (Types.NestedField field : schema.columns()) {
+ String fieldValue = Objects.toString(record.getField(field.name()), "null");
+ fieldValues.add(fieldValue);
+ }
+ String joinedString = String.join(", ", fieldValues);
+ results.add(joinedString);
+ }
+ return results;
+ }
+
+ private void validateSinkResult(
+ String warehouse, String database, String table, List<String> expected)
+ throws InterruptedException {
+ runInContainerAsRoot(jobManager, "chmod", "0777", "-R", warehouse);
+ LOG.info("Verifying Iceberg {}::{}::{} results...", warehouse, database, table);
+ long deadline = System.currentTimeMillis() + TESTCASE_TIMEOUT.toMillis();
+ List<String> results = Collections.emptyList();
+ while (System.currentTimeMillis() < deadline) {
+ try {
+ results = fetchIcebergTableRows(warehouse, database, table);
+ results = results.stream().sorted().collect(Collectors.toList());
+ for (int recordIndex = 0; recordIndex < results.size(); recordIndex++) {
+ Assertions.assertThat(results.get(recordIndex))
+ .isEqualTo(expected.get(recordIndex));
+ }
+ LOG.info(
+ "Successfully verified {} records in {} seconds.",
+ expected.size(),
+ (System.currentTimeMillis() - deadline + TESTCASE_TIMEOUT.toMillis())
+ / 1000);
+ return;
+ } catch (Exception e) {
+ LOG.warn("Validate failed, waiting for the next loop...", e);
+ } catch (AssertionError ignored) {
+ // AssertionError contains way too much records and might flood the log output.
+ LOG.warn(
+ "Results mismatch, expected {} records, but got {} actually. Waiting for the next loop...",
+ expected.size(),
+ results.size());
+ }
+ Thread.sleep(10000L);
+ }
+ Assertions.assertThat(results).containsExactlyInAnyOrderElementsOf(expected);
+ }
+}
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
index a8d8a73e8..962fb704f 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
@@ -306,7 +306,7 @@ public abstract class PipelineTestEnvironment extends TestLogger {
"env.java.opts.all: -Doracle.jdbc.timezoneAsRegion=false"));
}
- private void runInContainerAsRoot(GenericContainer<?> container, String... command)
+ public void runInContainerAsRoot(GenericContainer<?> container, String... command)
throws InterruptedException {
ToStringConsumer stdoutConsumer = new ToStringConsumer();
ToStringConsumer stderrConsumer = new ToStringConsumer();
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/resources/ddl/iceberg_inventory.sql b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/resources/ddl/iceberg_inventory.sql
new file mode 100644
index 000000000..82dc6fd09
--- /dev/null
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/resources/ddl/iceberg_inventory.sql
@@ -0,0 +1,55 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- ----------------------------------------------------------------------------------------------------------------
+-- DATABASE: mysql_inventory
+-- ----------------------------------------------------------------------------------------------------------------
+
+-- Create and populate our products using a single insert with many rows
+CREATE TABLE products (
+ id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ name VARCHAR(255) NOT NULL DEFAULT 'flink',
+ description VARCHAR(512),
+ weight FLOAT,
+ enum_c enum('red', 'white') default 'red', -- test some complex types as well,
+ json_c JSON, -- because we use additional dependencies to deserialize complex types.
+ point_c POINT
+);
+ALTER TABLE products AUTO_INCREMENT = 101;
+
+INSERT INTO products
+VALUES (default,"One", "Alice", 3.202, 'red', '{"key1": "value1"}', null),
+ (default,"Two", "Bob", 1.703, 'white', '{"key2": "value2"}', null),
+ (default,"Three", "Cecily", 4.105, 'red', '{"key3": "value3"}', null),
+ (default,"Four", "Derrida", 1.857, 'white', '{"key4": "value4"}', null),
+ (default,"Five", "Evelyn", 5.211, 'red', '{"K": "V", "k": "v"}', null),
+ (default,"Six", "Ferris", 9.813, null, null, null),
+ (default,"Seven", "Grace", 2.117, null, null, null),
+ (default,"Eight", "Hesse", 6.819, null, null, null),
+ (default,"Nine", "IINA", 5.223, null, null, null);
+
+-- Create and populate our customers using a single insert with many rows
+CREATE TABLE customers (
+ id INTEGER NOT NULL PRIMARY KEY,
+ name VARCHAR(255) NOT NULL DEFAULT 'flink',
+ address VARCHAR(1024),
+ phone_number VARCHAR(512)
+);
+
+INSERT INTO customers
+VALUES (101,"user_1","Shanghai","123567891234"),
+ (102,"user_2","Shanghai","123567891234"),
+ (103,"user_3","Shanghai","123567891234"),
+ (104,"user_4","Shanghai","123567891234");
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/resources/docker/peek-iceberg.sql b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/resources/docker/peek-iceberg.sql
new file mode 100644
index 000000000..788424216
--- /dev/null
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/resources/docker/peek-iceberg.sql
@@ -0,0 +1,29 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- Format this file with the following arguments:
+-- Warehouse Path, Database Name, and Table Name.
+
+SET 'sql-client.execution.result-mode' = 'tableau';
+SET 'table.display.max-column-width' = '100000';
+SET 'execution.runtime-mode' = 'batch';
+
+CREATE CATALOG iceberg_catalog WITH (
+ 'type' = 'iceberg',
+ 'warehouse' = '%s',
+ 'cache-enabled' = 'false'
+);
+
+SELECT * FROM iceberg_catalog.%s.%s;
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 240a3a8e5..0d700efa8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -95,6 +95,9 @@ limitations under the License.
<scala.version>2.12.16</scala.version>
<!-- Maven shade plugin prior to 3.2.0 could not correctly shade nested classes with JDK 8+. -->
<maven.shade.plugin.version>3.6.0</maven.shade.plugin.version>
+ <iceberg.version>1.6.1</iceberg.version>
+ <hive.version>2.3.9</hive.version>
+ <hadoop.version>3.3.4</hadoop.version>
</properties>
<dependencyManagement>