You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by sh...@apache.org on 2026/08/23 13:06:53 UTC
(seatunnel) branch dev updated: [Feature][Connector-V2] Add DB2 CDC source connector (#10780)
This is an automated email from the ASF dual-hosted git repository.
zhangshenghang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new a9f69848a0 [Feature][Connector-V2] Add DB2 CDC source connector (#10780)
a9f69848a0 is described below
commit a9f69848a02232449a66087bb7df8b0c748a9f42
Author: David Zollo <da...@gmail.com>
AuthorDate: Sun Aug 23 21:06:46 2026 +0800
[Feature][Connector-V2] Add DB2 CDC source connector (#10780)
Co-authored-by: github-actions[bot] <gi...@users.noreply.github.com>
Co-authored-by: DanielLeens <48...@users.noreply.github.com>
Co-authored-by: zhiwei.niu <zn...@gmail.com>
---
config/plugin_config | 1 +
docs/en/connectors/changelog/connector-cdc-db2.md | 6 +
docs/en/connectors/source/DB2-CDC.md | 199 ++++++++
docs/zh/connectors/changelog/connector-cdc-db2.md | 6 +
docs/zh/connectors/source/DB2-CDC.md | 197 ++++++++
plugin-mapping.properties | 1 +
.../connector-cdc/connector-cdc-db2/pom.xml | 96 ++++
.../seatunnel/cdc/db2/config/Db2SourceConfig.java | 99 ++++
.../cdc/db2/config/Db2SourceConfigFactory.java | 106 ++++
.../seatunnel/cdc/db2/source/Db2Dialect.java | 179 +++++++
.../cdc/db2/source/Db2IncrementalSource.java | 120 +++++
.../db2/source/Db2IncrementalSourceFactory.java | 131 +++++
.../db2/source/Db2IncrementalSourceOptions.java | 51 ++
.../db2/source/enumerator/Db2ChunkSplitter.java | 90 ++++
.../seatunnel/cdc/db2/source/offset/LsnOffset.java | 108 +++++
.../cdc/db2/source/offset/LsnOffsetFactory.java | 77 +++
.../reader/fetch/Db2SourceFetchTaskContext.java | 320 +++++++++++++
.../reader/fetch/scan/Db2SnapshotFetchTask.java | 203 ++++++++
.../fetch/scan/Db2SnapshotSplitReadTask.java | 272 +++++++++++
.../SnapshotSplitChangeEventSourceContext.java | 55 +++
.../transactionlog/Db2TransactionLogFetchTask.java | 532 +++++++++++++++++++++
.../cdc/db2/utils/Db2ConnectionUtils.java | 32 ++
.../seatunnel/cdc/db2/utils/Db2Schema.java | 93 ++++
.../seatunnel/cdc/db2/utils/Db2TypeUtils.java | 43 ++
.../seatunnel/cdc/db2/utils/Db2Utils.java | 499 +++++++++++++++++++
.../cdc/db2/utils/TableDiscoveryUtils.java | 112 +++++
.../source/Db2IncrementalSourceFactoryTest.java | 28 ++
.../cdc/db2/source/offset/LsnOffsetTest.java | 51 ++
.../seatunnel/cdc/db2/utils/Db2UtilsTest.java | 73 +++
.../cdc/db2/utils/TableDiscoveryUtilsTest.java | 69 +++
seatunnel-connectors-v2/connector-cdc/pom.xml | 1 +
.../seatunnel/jdbc/catalog/db2/DB2Catalog.java | 275 +++++++++++
.../jdbc/catalog/db2/DB2CatalogFactory.java | 56 +++
.../jdbc/internal/dialect/db2/DB2Dialect.java | 11 +-
.../seatunnel/jdbc/catalog/db2/DB2CatalogTest.java | 79 +++
.../jdbc/internal/dialect/db2/DB2DialectTest.java | 9 +-
seatunnel-dist/pom.xml | 6 +
.../connector-cdc-db2-e2e/pom.xml | 83 ++++
.../seatunnel/e2e/connector/cdc/db2/Db2CDCIT.java | 405 ++++++++++++++++
.../src/test/resources/db2cdc_to_db2.conf | 51 ++
seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml | 1 +
.../container/seatunnel/SeaTunnelContainer.java | 27 +-
42 files changed, 4844 insertions(+), 9 deletions(-)
diff --git a/config/plugin_config b/config/plugin_config
index dc87c451d8..62d1157592 100644
--- a/config/plugin_config
+++ b/config/plugin_config
@@ -24,6 +24,7 @@ connector-amazondynamodb
connector-assert
connector-cassandra
connector-salesforce
+connector-cdc-db2
connector-cdc-mysql
connector-cdc-mongodb
connector-cdc-sqlserver
diff --git a/docs/en/connectors/changelog/connector-cdc-db2.md b/docs/en/connectors/changelog/connector-cdc-db2.md
new file mode 100644
index 0000000000..1cf41e674b
--- /dev/null
+++ b/docs/en/connectors/changelog/connector-cdc-db2.md
@@ -0,0 +1,6 @@
+<details><summary> Change Log </summary>
+
+| Change | Commit | Version |
+| --- | --- | --- |
+
+</details>
diff --git a/docs/en/connectors/source/DB2-CDC.md b/docs/en/connectors/source/DB2-CDC.md
new file mode 100644
index 0000000000..0210bbc64d
--- /dev/null
+++ b/docs/en/connectors/source/DB2-CDC.md
@@ -0,0 +1,199 @@
+import ChangeLog from '../changelog/connector-cdc-db2.md';
+
+# DB2 CDC
+
+> DB2 CDC source connector
+
+## Support DB2 Version
+
+- DB2 LUW 11.5 or later versions supported by Debezium DB2 connector
+
+## Support Those Engines
+
+> SeaTunnel Zeta<br/>
+> Flink <br/>
+
+## Key Features
+
+- [ ] [batch](../../introduction/concepts/connector-v2-features.md)
+- [x] [stream](../../introduction/concepts/connector-v2-features.md)
+- [x] [exactly-once](../../introduction/concepts/connector-v2-features.md)
+- [ ] [column projection](../../introduction/concepts/connector-v2-features.md)
+- [x] [parallelism](../../introduction/concepts/connector-v2-features.md)
+- [x] [support user-defined split](../../introduction/concepts/connector-v2-features.md)
+
+## Description
+
+The DB2 CDC connector reads snapshot data and incremental data from DB2 tables that have been put
+into capture mode. It uses Debezium DB2 internally and can continue streaming committed INSERT,
+UPDATE and DELETE changes after the initial snapshot is finished.
+
+## Supported DataSource Info
+
+| Datasource | Supported versions | Driver | Url | Maven |
+|------------|--------------------|--------|-----|-------|
+| DB2 | DB2 LUW 11.5 or later versions supported by Debezium DB2 connector | com.ibm.db2.jcc.DB2Driver | jdbc:db2://127.0.0.1:50000/testdb | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc |
+
+## Using Dependency
+
+### Install Jdbc Driver
+
+#### For Spark/Flink Engine
+
+> 1. You need to ensure that the [DB2 JDBC driver jar package](https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc) has been placed in directory `${SEATUNNEL_HOME}/plugins/`.
+
+#### For SeaTunnel Zeta Engine
+
+> 1. You need to ensure that the [DB2 JDBC driver jar package](https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc) has been placed in directory `${SEATUNNEL_HOME}/lib/`.
+
+## Data Type Mapping
+
+| DB2 Data Type | SeaTunnel Data Type |
+|---------------|---------------------|
+| BOOLEAN | BOOLEAN |
+| SMALLINT | SHORT |
+| INT<br/>INTEGER | INT |
+| BIGINT | BIGINT |
+| DECIMAL<br/>DEC<br/>NUMERIC<br/>NUM | DECIMAL |
+| REAL | FLOAT |
+| DOUBLE<br/>DECFLOAT | DOUBLE |
+| CHAR<br/>CHARACTER<br/>VARCHAR<br/>LONG VARCHAR<br/>CLOB<br/>GRAPHIC<br/>VARGRAPHIC<br/>DBCLOB<br/>XML | STRING |
+| BINARY<br/>VARBINARY<br/>BLOB | BYTES |
+| DATE | DATE |
+| TIME | TIME |
+| TIMESTAMP | TIMESTAMP |
+
+## Source Options
+
+| Name | Type | Required | Default | Description |
+|------|------|----------|---------|-------------|
+| username | String | Yes | - | User name used to connect to DB2. |
+| password | String | Yes | - | Password used to connect to DB2. |
+| url | String | Yes | - | DB2 JDBC URL. The URL must include a database name, for example `jdbc:db2://127.0.0.1:50000/testdb`. |
+| database-names | List | No | The database parsed from `url` | Database name to monitor. DB2 CDC captures one database in one source. |
+| table-names | List | Yes when `table-pattern` is not set | - | Table names to monitor, using `databaseName.schemaName.tableName`, for example `testdb.DB2INST1.CUSTOMERS`. |
+| table-pattern | String | Yes when `table-names` is not set | - | Regular expression used to discover captured tables. |
+| table-names-config | List | No | - | Table config list. For example: `[{"table": "testdb.DB2INST1.CUSTOMERS","primaryKeys": ["ID"],"snapshotSplitColumn": "ID"}]`. |
+| startup.mode | Enum | No | INITIAL | Optional startup mode for DB2 CDC source. Valid values are `initial`, `earliest` and `latest`. |
+| stop.mode | Enum | No | NEVER | Optional stop mode for DB2 CDC source. Valid value is `never`. |
+| incremental.parallelism | Integer | No | 1 | The number of parallel readers in the incremental phase. |
+| snapshot.split.size | Integer | No | 8096 | The split size of table snapshot. |
+| snapshot.fetch.size | Integer | No | 1024 | The maximum fetch size for each poll when reading table snapshot. |
+| server-time-zone | String | No | UTC | The session time zone in database server. |
+| connect.timeout.ms | Duration | No | 30s | The maximum time that the connector should wait after trying to connect to the database server before timing out. |
+| connect.max-retries | Integer | No | 3 | The maximum retry times to build database server connection. |
+| connection.pool.size | Integer | No | 20 | The connection pool size. |
+| chunk-key.even-distribution.factor.upper-bound | Double | No | 100 | The upper bound used to decide whether a split key is evenly distributed. |
+| chunk-key.even-distribution.factor.lower-bound | Double | No | 0.05 | The lower bound used to decide whether a split key is evenly distributed. |
+| sample-sharding.threshold | int | No | 1000 | The estimated shard count threshold that triggers sample-based sharding for unevenly distributed split keys. |
+| inverse-sampling.rate | int | No | 1000 | The inverse sampling rate used by sample-based sharding. |
+| exactly_once | Boolean | No | false | Enable exactly-once semantics for initial snapshot handoff. |
+| debezium.* | config | No | - | Pass-through Debezium DB2 connector properties. |
+| format | Enum | No | DEFAULT | Optional output format. Valid values are `DEFAULT` and `COMPATIBLE_DEBEZIUM_JSON`. |
+| common-options | | No | - | Source plugin common parameters, please refer to [Source Common Options](../common-options/source-common-options.md) for details. |
+
+### Enable DB2 CDC
+
+DB2 CDC depends on DB2 SQL replication and ASN capture tables. Verify the required IBM replication
+license for your environment before enabling capture. The database administrator must put every
+source table into capture mode before running SeaTunnel. You can use DB2 control commands or
+Debezium's management UDFs. The following commands show the common UDF workflow:
+
+```sql
+VALUES ASNCDC.ASNCDCSERVICES('status','asncdc');
+VALUES ASNCDC.ASNCDCSERVICES('start','asncdc');
+CALL ASNCDC.ADDTABLE('DB2INST1', 'CUSTOMERS');
+VALUES ASNCDC.ASNCDCSERVICES('reinit','asncdc');
+```
+
+For complete DB2 server setup, permissions and ASN capture agent configuration, refer to the
+[Debezium DB2 connector setup guide](https://debezium.io/documentation/reference/1.9/connectors/db2.html).
+
+## Task Example
+
+### Initial Read Simple
+
+> This example reads an initial snapshot and then continues to read incremental changes.
+
+```hocon
+env {
+ parallelism = 1
+ job.mode = "STREAMING"
+ checkpoint.interval = 5000
+}
+
+source {
+ DB2-CDC {
+ plugin_output = "customers"
+ username = "db2inst1"
+ password = "db2inst1"
+ startup.mode = "initial"
+ database-names = ["testdb"]
+ table-names = ["testdb.DB2INST1.CUSTOMERS"]
+ url = "jdbc:db2://127.0.0.1:50000/testdb"
+ }
+}
+
+sink {
+ console {
+ plugin_input = "customers"
+ }
+}
+```
+
+### Incremental Read Simple
+
+> This example starts from the latest DB2 LSN and prints newly changed data.
+
+```hocon
+env {
+ parallelism = 1
+ job.mode = "STREAMING"
+ checkpoint.interval = 5000
+}
+
+source {
+ DB2-CDC {
+ plugin_output = "customers"
+ username = "db2inst1"
+ password = "db2inst1"
+ startup.mode = "latest"
+ database-names = ["testdb"]
+ table-names = ["testdb.DB2INST1.CUSTOMERS"]
+ url = "jdbc:db2://127.0.0.1:50000/testdb"
+ }
+}
+
+sink {
+ console {
+ plugin_input = "customers"
+ }
+}
+```
+
+### Support Custom Primary Key For Table
+
+```hocon
+source {
+ DB2-CDC {
+ plugin_output = "customers"
+ username = "db2inst1"
+ password = "db2inst1"
+ startup.mode = "initial"
+ database-names = ["testdb"]
+ table-names = ["testdb.DB2INST1.CUSTOMERS"]
+ table-names-config = [
+ {
+ table = "testdb.DB2INST1.CUSTOMERS"
+ primaryKeys = ["ID"]
+ snapshotSplitColumn = "ID"
+ }
+ ]
+ url = "jdbc:db2://127.0.0.1:50000/testdb"
+ }
+}
+```
+
+## Changelog
+
+<ChangeLog />
diff --git a/docs/zh/connectors/changelog/connector-cdc-db2.md b/docs/zh/connectors/changelog/connector-cdc-db2.md
new file mode 100644
index 0000000000..1cf41e674b
--- /dev/null
+++ b/docs/zh/connectors/changelog/connector-cdc-db2.md
@@ -0,0 +1,6 @@
+<details><summary> Change Log </summary>
+
+| Change | Commit | Version |
+| --- | --- | --- |
+
+</details>
diff --git a/docs/zh/connectors/source/DB2-CDC.md b/docs/zh/connectors/source/DB2-CDC.md
new file mode 100644
index 0000000000..ed1dae1c9b
--- /dev/null
+++ b/docs/zh/connectors/source/DB2-CDC.md
@@ -0,0 +1,197 @@
+import ChangeLog from '../changelog/connector-cdc-db2.md';
+
+# DB2 CDC
+
+> DB2 CDC 源连接器
+
+## 支持 DB2 版本
+
+- DB2 LUW 11.5 或 Debezium DB2 连接器支持的更高版本
+
+## 支持的引擎
+
+> SeaTunnel Zeta<br/>
+> Flink <br/>
+
+## 主要功能
+
+- [ ] [批处理](../../introduction/concepts/connector-v2-features.md)
+- [x] [流处理](../../introduction/concepts/connector-v2-features.md)
+- [x] [精确一次](../../introduction/concepts/connector-v2-features.md)
+- [ ] [列投影](../../introduction/concepts/connector-v2-features.md)
+- [x] [并行度](../../introduction/concepts/connector-v2-features.md)
+- [x] [支持用户定义分割](../../introduction/concepts/connector-v2-features.md)
+
+## 描述
+
+DB2 CDC 连接器可以读取已启用 capture mode 的 DB2 表的快照数据和增量数据。连接器内部使用
+Debezium DB2,在初始快照完成后会继续读取已提交的 INSERT、UPDATE 和 DELETE 变更。
+
+## 支持的数据源信息
+
+| 数据源 | 支持版本 | 驱动 | Url | Maven |
+|--------|----------|------|-----|-------|
+| DB2 | DB2 LUW 11.5 或 Debezium DB2 连接器支持的更高版本 | com.ibm.db2.jcc.DB2Driver | jdbc:db2://127.0.0.1:50000/testdb | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc |
+
+## 使用依赖
+
+### 安装 Jdbc 驱动
+
+#### 对于 Spark/Flink 引擎
+
+> 1. 你需要确保 [DB2 JDBC 驱动 jar 包](https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc) 已经放置在 `${SEATUNNEL_HOME}/plugins/` 目录中。
+
+#### 对于 SeaTunnel Zeta 引擎
+
+> 1. 你需要确保 [DB2 JDBC 驱动 jar 包](https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc) 已经放置在 `${SEATUNNEL_HOME}/lib/` 目录中。
+
+## 数据类型映射
+
+| DB2 数据类型 | SeaTunnel 数据类型 |
+|--------------|---------------------|
+| BOOLEAN | BOOLEAN |
+| SMALLINT | SHORT |
+| INT<br/>INTEGER | INT |
+| BIGINT | BIGINT |
+| DECIMAL<br/>DEC<br/>NUMERIC<br/>NUM | DECIMAL |
+| REAL | FLOAT |
+| DOUBLE<br/>DECFLOAT | DOUBLE |
+| CHAR<br/>CHARACTER<br/>VARCHAR<br/>LONG VARCHAR<br/>CLOB<br/>GRAPHIC<br/>VARGRAPHIC<br/>DBCLOB<br/>XML | STRING |
+| BINARY<br/>VARBINARY<br/>BLOB | BYTES |
+| DATE | DATE |
+| TIME | TIME |
+| TIMESTAMP | TIMESTAMP |
+
+## 数据源参数
+
+| 名称 | 类型 | 是否必填 | 默认值 | 描述 |
+|------|------|----------|--------|------|
+| username | String | 是 | - | 连接 DB2 时使用的用户名。 |
+| password | String | 是 | - | 连接 DB2 时使用的密码。 |
+| url | String | 是 | - | DB2 JDBC URL。URL 必须包含数据库名,例如 `jdbc:db2://127.0.0.1:50000/testdb`。 |
+| database-names | List | 否 | 从 `url` 中解析出的数据库 | 要监控的数据库名。DB2 CDC 一个 source 监控一个数据库。 |
+| table-names | List | 未设置 `table-pattern` 时必填 | - | 要监控的表名,格式为 `databaseName.schemaName.tableName`,例如 `testdb.DB2INST1.CUSTOMERS`。 |
+| table-pattern | String | 未设置 `table-names` 时必填 | - | 用于发现已开启 capture mode 的表的正则表达式。 |
+| table-names-config | List | 否 | - | 表配置列表。例如:`[{"table": "testdb.DB2INST1.CUSTOMERS","primaryKeys": ["ID"],"snapshotSplitColumn": "ID"}]`。 |
+| startup.mode | Enum | 否 | INITIAL | DB2 CDC 的可选启动模式,有效值为 `initial`、`earliest` 和 `latest`。 |
+| stop.mode | Enum | 否 | NEVER | DB2 CDC 的可选停止模式,有效值为 `never`。 |
+| incremental.parallelism | Integer | 否 | 1 | 增量阶段中并行读取器的数量。 |
+| snapshot.split.size | Integer | 否 | 8096 | 表快照的分割大小。 |
+| snapshot.fetch.size | Integer | 否 | 1024 | 读取表快照时每次轮询的最大获取大小。 |
+| server-time-zone | String | 否 | UTC | 数据库服务器中的会话时区。 |
+| connect.timeout.ms | Duration | 否 | 30s | 连接器尝试连接到数据库服务器后,在超时之前等待的最长时间。 |
+| connect.max-retries | Integer | 否 | 3 | 连接器重试建立数据库连接的最大次数。 |
+| connection.pool.size | Integer | 否 | 20 | 连接池大小。 |
+| chunk-key.even-distribution.factor.upper-bound | Double | 否 | 100 | 用于判断分块键是否均匀分布的上界。 |
+| chunk-key.even-distribution.factor.lower-bound | Double | 否 | 0.05 | 用于判断分块键是否均匀分布的下界。 |
+| sample-sharding.threshold | int | 否 | 1000 | 分块键分布不均时触发采样分片策略的估计分片数阈值。 |
+| inverse-sampling.rate | int | 否 | 1000 | 采样分片策略使用的采样率倒数。 |
+| exactly_once | Boolean | 否 | false | 启用初始快照切换增量阶段时的精确一次语义。 |
+| debezium.* | config | 否 | - | 透传给 Debezium DB2 连接器的配置项。 |
+| format | Enum | 否 | DEFAULT | 可选输出格式,有效值为 `DEFAULT` 和 `COMPATIBLE_DEBEZIUM_JSON`。 |
+| common-options | | 否 | - | 源插件通用参数,请参考 [Source Common Options](../common-options/source-common-options.md) 获取详细信息。 |
+
+### 启用 DB2 CDC
+
+DB2 CDC 依赖 DB2 SQL replication 和 ASN capture tables。启用 capture 前,请先确认当前环境具备所需的
+IBM replication 授权。运行 SeaTunnel 前,数据库管理员必须先把要读取的表加入 capture mode。可以使用
+DB2 控制命令,也可以使用 Debezium 提供的管理 UDF。下面是常见的 UDF 流程:
+
+```sql
+VALUES ASNCDC.ASNCDCSERVICES('status','asncdc');
+VALUES ASNCDC.ASNCDCSERVICES('start','asncdc');
+CALL ASNCDC.ADDTABLE('DB2INST1', 'CUSTOMERS');
+VALUES ASNCDC.ASNCDCSERVICES('reinit','asncdc');
+```
+
+完整的 DB2 服务端配置、权限和 ASN capture agent 配置请参考
+[Debezium DB2 连接器设置文档](https://debezium.io/documentation/reference/1.9/connectors/db2.html)。
+
+## 任务示例
+
+### 初始读取简单示例
+
+> 该示例先读取初始快照,随后继续读取增量变更。
+
+```hocon
+env {
+ parallelism = 1
+ job.mode = "STREAMING"
+ checkpoint.interval = 5000
+}
+
+source {
+ DB2-CDC {
+ plugin_output = "customers"
+ username = "db2inst1"
+ password = "db2inst1"
+ startup.mode = "initial"
+ database-names = ["testdb"]
+ table-names = ["testdb.DB2INST1.CUSTOMERS"]
+ url = "jdbc:db2://127.0.0.1:50000/testdb"
+ }
+}
+
+sink {
+ console {
+ plugin_input = "customers"
+ }
+}
+```
+
+### 增量读取简单示例
+
+> 该示例从最新 DB2 LSN 开始读取,并打印新产生的变更数据。
+
+```hocon
+env {
+ parallelism = 1
+ job.mode = "STREAMING"
+ checkpoint.interval = 5000
+}
+
+source {
+ DB2-CDC {
+ plugin_output = "customers"
+ username = "db2inst1"
+ password = "db2inst1"
+ startup.mode = "latest"
+ database-names = ["testdb"]
+ table-names = ["testdb.DB2INST1.CUSTOMERS"]
+ url = "jdbc:db2://127.0.0.1:50000/testdb"
+ }
+}
+
+sink {
+ console {
+ plugin_input = "customers"
+ }
+}
+```
+
+### 支持表的自定义主键
+
+```hocon
+source {
+ DB2-CDC {
+ plugin_output = "customers"
+ username = "db2inst1"
+ password = "db2inst1"
+ startup.mode = "initial"
+ database-names = ["testdb"]
+ table-names = ["testdb.DB2INST1.CUSTOMERS"]
+ table-names-config = [
+ {
+ table = "testdb.DB2INST1.CUSTOMERS"
+ primaryKeys = ["ID"]
+ snapshotSplitColumn = "ID"
+ }
+ ]
+ url = "jdbc:db2://127.0.0.1:50000/testdb"
+ }
+}
+```
+
+## 变更日志
+
+<ChangeLog />
diff --git a/plugin-mapping.properties b/plugin-mapping.properties
index eb1aadf178..0a4f4a6311 100644
--- a/plugin-mapping.properties
+++ b/plugin-mapping.properties
@@ -110,6 +110,7 @@ seatunnel.sink.Airtable = connector-http-airtable
seatunnel.sink.RabbitMQ = connector-rabbitmq
seatunnel.source.RabbitMQ = connector-rabbitmq
seatunnel.source.OpenMldb = connector-openmldb
+seatunnel.source.DB2-CDC = connector-cdc-db2
seatunnel.source.SqlServer-CDC = connector-cdc-sqlserver
seatunnel.source.Doris = connector-doris
seatunnel.sink.Doris = connector-doris
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/pom.xml b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/pom.xml
new file mode 100644
index 0000000000..b54a909010
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/pom.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-cdc</artifactId>
+ <version>${revision}</version>
+ </parent>
+ <artifactId>connector-cdc-db2</artifactId>
+ <name>SeaTunnel : Connectors V2 : CDC : DB2</name>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-cdc-base</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.debezium</groupId>
+ <artifactId>debezium-connector-db2</artifactId>
+ <version>${debezium.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-cdc-base</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.debezium</groupId>
+ <artifactId>debezium-connector-db2</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.ibm.db2</groupId>
+ <artifactId>jcc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.debezium</groupId>
+ <artifactId>debezium-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.debezium</groupId>
+ <artifactId>debezium-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.ibm.db2.jcc</groupId>
+ <artifactId>db2jcc</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfig.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfig.java
new file mode 100644
index 0000000000..d334d0ddc3
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfig.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.config;
+
+import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceConfig;
+import org.apache.seatunnel.connectors.cdc.base.config.StartupConfig;
+import org.apache.seatunnel.connectors.cdc.base.config.StopConfig;
+
+import io.debezium.connector.db2.Db2ConnectorConfig;
+import io.debezium.relational.RelationalTableFilters;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Describes the connection information of the Db2 database and the configuration information for
+ * performing snapshotting and streaming reading, such as splitSize.
+ */
+public class Db2SourceConfig extends JdbcSourceConfig {
+
+ private static final long serialVersionUID = 1L;
+
+ public Db2SourceConfig(
+ StartupConfig startupConfig,
+ StopConfig stopConfig,
+ List<String> databaseList,
+ List<String> tableList,
+ int splitSize,
+ Map<String, String> splitColumn,
+ double distributionFactorUpper,
+ double distributionFactorLower,
+ int sampleShardingThreshold,
+ int inverseSamplingRate,
+ boolean sampleShardingAllow,
+ Properties dbzProperties,
+ String driverClassName,
+ String hostname,
+ int port,
+ String username,
+ String password,
+ String originUrl,
+ int fetchSize,
+ String serverTimeZone,
+ long connectTimeoutMillis,
+ int connectMaxRetries,
+ int connectionPoolSize,
+ boolean exactlyOnce) {
+ super(
+ startupConfig,
+ stopConfig,
+ databaseList,
+ tableList,
+ splitSize,
+ splitColumn,
+ distributionFactorUpper,
+ distributionFactorLower,
+ sampleShardingThreshold,
+ inverseSamplingRate,
+ sampleShardingAllow,
+ dbzProperties,
+ driverClassName,
+ hostname,
+ port,
+ username,
+ password,
+ originUrl,
+ fetchSize,
+ serverTimeZone,
+ connectTimeoutMillis,
+ connectMaxRetries,
+ connectionPoolSize,
+ exactlyOnce);
+ }
+
+ @Override
+ public Db2ConnectorConfig getDbzConnectorConfig() {
+ return new Db2ConnectorConfig(getDbzConfiguration());
+ }
+
+ public RelationalTableFilters getTableFilters() {
+ return getDbzConnectorConfig().getTableFilters();
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfigFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfigFactory.java
new file mode 100644
index 0000000000..0c5caba483
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/config/Db2SourceConfigFactory.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.config;
+
+import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceConfigFactory;
+import org.apache.seatunnel.connectors.cdc.debezium.EmbeddedDatabaseHistory;
+
+import io.debezium.connector.db2.Db2Connector;
+
+import java.util.Properties;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import static org.apache.seatunnel.shade.com.google.common.base.Preconditions.checkNotNull;
+
+/** Factory for creating {@link Db2SourceConfig}. */
+public class Db2SourceConfigFactory extends JdbcSourceConfigFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final String DATABASE_SERVER_NAME = "db2_transaction_log_source";
+ private static final String DRIVER_CLASS_NAME = "com.ibm.db2.jcc.DB2Driver";
+
+ @Override
+ public Db2SourceConfig create(int subtask) {
+ Properties props = new Properties();
+ props.setProperty("connector.class", Db2Connector.class.getCanonicalName());
+
+ // Debezium uses the logical server name as a namespace for offsets and schema history.
+ // SeaTunnel isolates each source task with a generated history instance name below, so a
+ // stable connector-level logical name is enough here.
+ props.setProperty("database.server.name", DATABASE_SERVER_NAME);
+ props.setProperty("database.hostname", checkNotNull(hostname));
+ props.setProperty("database.user", checkNotNull(username));
+ props.setProperty("database.password", checkNotNull(password));
+ props.setProperty("database.port", String.valueOf(port));
+ props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true));
+ props.setProperty("database.dbname", checkNotNull(databaseList.get(0)));
+
+ props.setProperty("database.history", EmbeddedDatabaseHistory.class.getCanonicalName());
+ props.setProperty("database.history.instance.name", UUID.randomUUID() + "_" + subtask);
+ props.setProperty("database.history.skip.unparseable.ddl", String.valueOf(true));
+ props.setProperty("database.history.refer.ddl", String.valueOf(true));
+
+ // TODO Not yet supported
+ props.setProperty("include.schema.changes", String.valueOf(false));
+
+ // Db2 captures a single configured database via database.dbname. Debezium reports captured
+ // table ids without a catalog, so setting database.include.list would filter out valid
+ // Db2 change tables before snapshot split discovery.
+ if (tableList != null) {
+ // SeaTunnel table names are database.schema.table, while Debezium Db2 expects
+ // schema.table in table.include.list for the single configured database.
+ String tableIncludeList =
+ tableList.stream()
+ .map(table -> table.substring(table.indexOf(".") + 1))
+ .collect(Collectors.joining(","));
+ props.setProperty("table.include.list", tableIncludeList);
+ }
+
+ if (dbzProperties != null) {
+ dbzProperties.forEach(props::put);
+ }
+
+ return new Db2SourceConfig(
+ startupConfig,
+ stopConfig,
+ databaseList,
+ tableList,
+ splitSize,
+ splitColumn,
+ distributionFactorUpper,
+ distributionFactorLower,
+ sampleShardingThreshold,
+ inverseSamplingRate,
+ sampleShardingAllow,
+ props,
+ DRIVER_CLASS_NAME,
+ hostname,
+ port,
+ username,
+ password,
+ originUrl,
+ fetchSize,
+ serverTimeZone,
+ connectTimeoutMillis,
+ connectMaxRetries,
+ connectionPoolSize,
+ exactlyOnce);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2Dialect.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2Dialect.java
new file mode 100644
index 0000000000..ffaefb3f3b
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2Dialect.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source;
+
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.ConstraintKey;
+import org.apache.seatunnel.api.table.catalog.PrimaryKey;
+import org.apache.seatunnel.common.utils.SeaTunnelException;
+import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceConfig;
+import org.apache.seatunnel.connectors.cdc.base.dialect.JdbcDataSourceDialect;
+import org.apache.seatunnel.connectors.cdc.base.source.enumerator.splitter.ChunkSplitter;
+import org.apache.seatunnel.connectors.cdc.base.source.reader.external.FetchTask;
+import org.apache.seatunnel.connectors.cdc.base.source.split.SourceSplitBase;
+import org.apache.seatunnel.connectors.cdc.base.utils.CatalogTableUtils;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfigFactory;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.enumerator.Db2ChunkSplitter;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.Db2SourceFetchTaskContext;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.scan.Db2SnapshotFetchTask;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.transactionlog.Db2TransactionLogFetchTask;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2ConnectionUtils;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2Schema;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.TableDiscoveryUtils;
+import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier;
+
+import io.debezium.connector.db2.Db2ChangeTable;
+import io.debezium.connector.db2.Db2Connection;
+import io.debezium.jdbc.JdbcConnection;
+import io.debezium.relational.TableId;
+import io.debezium.relational.history.TableChanges;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/** The {@link JdbcDataSourceDialect} implementation for Db2 datasource. */
+public class Db2Dialect implements JdbcDataSourceDialect {
+
+ private static final long serialVersionUID = 1L;
+ private final Db2SourceConfig sourceConfig;
+
+ private transient Db2Schema db2Schema;
+ private final Map<TableId, CatalogTable> tableMap;
+
+ public Db2Dialect(Db2SourceConfigFactory configFactory, List<CatalogTable> catalogTables) {
+ this.sourceConfig = configFactory.create(0);
+ this.tableMap = createDb2TableMap(catalogTables);
+ }
+
+ @Override
+ public String getName() {
+ return DatabaseIdentifier.DB_2;
+ }
+
+ @Override
+ public boolean isDataCollectionIdCaseSensitive(JdbcSourceConfig sourceConfig) {
+ // todo: need to check the case sensitive of the database
+ return true;
+ }
+
+ @Override
+ public JdbcConnection openJdbcConnection(JdbcSourceConfig sourceConfig) {
+ return Db2ConnectionUtils.createDb2Connection(sourceConfig.getDbzConfiguration());
+ }
+
+ @Override
+ public ChunkSplitter createChunkSplitter(JdbcSourceConfig sourceConfig) {
+ return new Db2ChunkSplitter(sourceConfig, this);
+ }
+
+ @Override
+ public List<TableId> discoverDataCollections(JdbcSourceConfig sourceConfig) {
+ Db2SourceConfig db2SourceConfig = (Db2SourceConfig) sourceConfig;
+ try (JdbcConnection jdbcConnection = openJdbcConnection(sourceConfig)) {
+ List<TableId> tables =
+ TableDiscoveryUtils.listTables(
+ jdbcConnection, db2SourceConfig.getTableFilters());
+ TableDiscoveryUtils.validateExplicitCaptureTables(
+ db2SourceConfig.getTableList(), tables);
+ return tables;
+ } catch (SQLException e) {
+ throw new SeaTunnelException("Error to discover tables: " + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void checkAllTablesEnabledCapture(JdbcConnection jdbcConnection, List<TableId> tableIds)
+ throws SQLException {
+ Set<TableId> tables =
+ ((Db2Connection) jdbcConnection)
+ .listOfChangeTables().stream()
+ .map(Db2ChangeTable::getSourceTableId)
+ .collect(java.util.stream.Collectors.toSet());
+ for (TableId tableId : tableIds) {
+ if (!tables.contains(toDb2TableId(tableId))) {
+ throw new SeaTunnelException("Table " + tableId + " is not enabled for capture");
+ }
+ }
+ }
+
+ @Override
+ public TableChanges.TableChange queryTableSchema(JdbcConnection jdbc, TableId tableId) {
+ if (db2Schema == null) {
+ db2Schema = new Db2Schema(sourceConfig.getDbzConnectorConfig(), tableMap);
+ }
+ return db2Schema.getTableSchema(jdbc, toDb2TableId(tableId));
+ }
+
+ @Override
+ public Db2SourceFetchTaskContext createFetchTaskContext(
+ SourceSplitBase sourceSplitBase, JdbcSourceConfig taskSourceConfig) {
+
+ return new Db2SourceFetchTaskContext((Db2SourceConfig) taskSourceConfig, this);
+ }
+
+ @Override
+ public FetchTask<SourceSplitBase> createFetchTask(SourceSplitBase sourceSplitBase) {
+ if (sourceSplitBase.isSnapshotSplit()) {
+ return new Db2SnapshotFetchTask(sourceSplitBase.asSnapshotSplit());
+ } else {
+ try (JdbcConnection jdbcConnection = openJdbcConnection(sourceConfig)) {
+ List<TableId> tables = sourceSplitBase.asIncrementalSplit().getTableIds();
+ this.checkAllTablesEnabledCapture(jdbcConnection, tables);
+ } catch (SQLException e) {
+ throw new SeaTunnelException("Error to check tables: " + e.getMessage(), e);
+ }
+ return new Db2TransactionLogFetchTask(sourceSplitBase.asIncrementalSplit());
+ }
+ }
+
+ @Override
+ public Optional<PrimaryKey> getPrimaryKey(JdbcConnection jdbcConnection, TableId tableId) {
+ return Optional.ofNullable(
+ tableMap.get(toDb2TableId(tableId)).getTableSchema().getPrimaryKey());
+ }
+
+ @Override
+ public List<ConstraintKey> getConstraintKeys(JdbcConnection jdbcConnection, TableId tableId) {
+ return tableMap.get(toDb2TableId(tableId)).getTableSchema().getConstraintKeys();
+ }
+
+ private static Map<TableId, CatalogTable> createDb2TableMap(List<CatalogTable> catalogTables) {
+ Map<TableId, CatalogTable> tables = new HashMap<>();
+ CatalogTableUtils.convertTables(catalogTables)
+ .forEach(
+ (tableId, catalogTable) -> {
+ tables.put(tableId, catalogTable);
+ tables.put(toDb2TableId(tableId), catalogTable);
+ });
+ return tables;
+ }
+
+ /**
+ * Debezium Db2 reports captured table ids with an empty catalog because the connector captures
+ * a single configured database. SeaTunnel catalog tables keep the database name, so all runtime
+ * lookups are normalized before comparing with Debezium metadata.
+ */
+ private static TableId toDb2TableId(TableId tableId) {
+ return new TableId("", tableId.schema(), tableId.table());
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSource.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSource.java
new file mode 100644
index 0000000000..85c5fbb702
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSource.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.source.SupportParallelism;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.common.utils.JdbcUrlUtil;
+import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceConfig;
+import org.apache.seatunnel.connectors.cdc.base.config.SourceConfig;
+import org.apache.seatunnel.connectors.cdc.base.dialect.DataSourceDialect;
+import org.apache.seatunnel.connectors.cdc.base.option.JdbcSourceOptions;
+import org.apache.seatunnel.connectors.cdc.base.option.StartupMode;
+import org.apache.seatunnel.connectors.cdc.base.option.StopMode;
+import org.apache.seatunnel.connectors.cdc.base.source.IncrementalSource;
+import org.apache.seatunnel.connectors.cdc.base.source.offset.OffsetFactory;
+import org.apache.seatunnel.connectors.cdc.debezium.DebeziumDeserializationSchema;
+import org.apache.seatunnel.connectors.cdc.debezium.DeserializeFormat;
+import org.apache.seatunnel.connectors.cdc.debezium.row.DebeziumJsonDeserializeSchema;
+import org.apache.seatunnel.connectors.cdc.debezium.row.SeaTunnelRowDebeziumDeserializeSchema;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfigFactory;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset.LsnOffsetFactory;
+import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions;
+
+import java.time.ZoneId;
+import java.util.List;
+import java.util.Optional;
+
+public class Db2IncrementalSource<T> extends IncrementalSource<T, JdbcSourceConfig>
+ implements SupportParallelism {
+
+ static final String IDENTIFIER = "DB2-CDC";
+
+ public Db2IncrementalSource(ReadonlyConfig options, List<CatalogTable> catalogTables) {
+ super(options, catalogTables);
+ }
+
+ @Override
+ public String getPluginName() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Option<StartupMode> getStartupModeOption() {
+ return Db2IncrementalSourceOptions.STARTUP_MODE;
+ }
+
+ @Override
+ public Option<StopMode> getStopModeOption() {
+ return Db2IncrementalSourceOptions.STOP_MODE;
+ }
+
+ @Override
+ public SourceConfig.Factory<JdbcSourceConfig> createSourceConfigFactory(ReadonlyConfig config) {
+ Db2SourceConfigFactory configFactory = new Db2SourceConfigFactory();
+ configFactory.fromReadonlyConfig(readonlyConfig);
+ configFactory.startupOptions(startupConfig);
+ configFactory.stopOptions(stopConfig);
+ JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(config.get(JdbcCommonOptions.URL));
+ configFactory.originUrl(urlInfo.getOrigin());
+ configFactory.hostname(urlInfo.getHost());
+ configFactory.port(urlInfo.getPort());
+ if (!config.getOptional(JdbcSourceOptions.DATABASE_NAMES).isPresent()
+ && urlInfo.getDefaultDatabase().isPresent()) {
+ configFactory.databaseList(urlInfo.getDefaultDatabase().get());
+ }
+ return configFactory;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public DebeziumDeserializationSchema<T> createDebeziumDeserializationSchema(
+ ReadonlyConfig config) {
+ if (DeserializeFormat.COMPATIBLE_DEBEZIUM_JSON.equals(
+ config.get(JdbcSourceOptions.FORMAT))) {
+ return (DebeziumDeserializationSchema<T>)
+ new DebeziumJsonDeserializeSchema(
+ config.get(JdbcSourceOptions.DEBEZIUM_PROPERTIES));
+ }
+
+ String zoneId = config.get(JdbcSourceOptions.SERVER_TIME_ZONE);
+ return (DebeziumDeserializationSchema<T>)
+ SeaTunnelRowDebeziumDeserializeSchema.builder()
+ .setTables(catalogTables)
+ .setServerTimeZone(ZoneId.of(zoneId))
+ .build();
+ }
+
+ @Override
+ public DataSourceDialect<JdbcSourceConfig> createDataSourceDialect(ReadonlyConfig config) {
+ return new Db2Dialect((Db2SourceConfigFactory) configFactory, catalogTables);
+ }
+
+ @Override
+ public OffsetFactory createOffsetFactory(ReadonlyConfig config) {
+ return new LsnOffsetFactory(
+ (Db2SourceConfigFactory) configFactory, (Db2Dialect) dataSourceDialect);
+ }
+
+ @Override
+ public Optional<String> driverName() {
+ return Optional.of("com.ibm.db2.jcc.DB2Driver");
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactory.java
new file mode 100644
index 0000000000..cc0ba5509e
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactory.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.options.ConnectorCommonOptions;
+import org.apache.seatunnel.api.source.SeaTunnelSource;
+import org.apache.seatunnel.api.source.SourceSplit;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.connector.TableSource;
+import org.apache.seatunnel.api.table.factory.Factory;
+import org.apache.seatunnel.api.table.factory.TableSourceFactory;
+import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext;
+import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceTableConfig;
+import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions;
+import org.apache.seatunnel.connectors.cdc.base.option.StartupMode;
+import org.apache.seatunnel.connectors.cdc.base.option.StopMode;
+import org.apache.seatunnel.connectors.cdc.base.utils.CatalogTableUtils;
+
+import com.google.auto.service.AutoService;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Optional;
+
+@AutoService(Factory.class)
+@Slf4j
+public class Db2IncrementalSourceFactory implements TableSourceFactory {
+
+ private static final String DRIVER_CLASS_NAME = "com.ibm.db2.jcc.DB2Driver";
+
+ @Override
+ public String factoryIdentifier() {
+ return Db2IncrementalSource.IDENTIFIER;
+ }
+
+ @Override
+ public OptionRule optionRule() {
+ return Db2IncrementalSourceOptions.getBaseRule()
+ .required(
+ Db2IncrementalSourceOptions.USERNAME,
+ Db2IncrementalSourceOptions.PASSWORD,
+ Db2IncrementalSourceOptions.URL)
+ .exclusive(ConnectorCommonOptions.TABLE_NAMES, ConnectorCommonOptions.TABLE_PATTERN)
+ .optional(
+ Db2IncrementalSourceOptions.DATABASE_NAMES,
+ Db2IncrementalSourceOptions.SERVER_TIME_ZONE,
+ Db2IncrementalSourceOptions.CONNECT_TIMEOUT_MS,
+ Db2IncrementalSourceOptions.CONNECT_MAX_RETRIES,
+ Db2IncrementalSourceOptions.CONNECTION_POOL_SIZE,
+ Db2IncrementalSourceOptions.CHUNK_KEY_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND,
+ Db2IncrementalSourceOptions.CHUNK_KEY_EVEN_DISTRIBUTION_FACTOR_UPPER_BOUND,
+ Db2IncrementalSourceOptions.SAMPLE_SHARDING_THRESHOLD,
+ Db2IncrementalSourceOptions.INVERSE_SAMPLING_RATE,
+ Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG)
+ .optional(
+ Db2IncrementalSourceOptions.STARTUP_MODE,
+ Db2IncrementalSourceOptions.STOP_MODE)
+ .conditional(
+ Db2IncrementalSourceOptions.STARTUP_MODE,
+ StartupMode.SPECIFIC,
+ SourceOptions.STARTUP_SPECIFIC_OFFSET_POS)
+ .conditional(
+ Db2IncrementalSourceOptions.STOP_MODE,
+ StopMode.SPECIFIC,
+ SourceOptions.STOP_SPECIFIC_OFFSET_POS)
+ .conditional(
+ Db2IncrementalSourceOptions.STARTUP_MODE,
+ StartupMode.TIMESTAMP,
+ SourceOptions.STARTUP_TIMESTAMP)
+ .conditional(
+ Db2IncrementalSourceOptions.STOP_MODE,
+ StopMode.TIMESTAMP,
+ SourceOptions.STOP_TIMESTAMP)
+ .conditional(
+ Db2IncrementalSourceOptions.STARTUP_MODE,
+ StartupMode.INITIAL,
+ SourceOptions.EXACTLY_ONCE)
+ .build();
+ }
+
+ @Override
+ public Class<? extends SeaTunnelSource> getSourceClass() {
+ return Db2IncrementalSource.class;
+ }
+
+ @Override
+ public <T, SplitT extends SourceSplit, StateT extends Serializable>
+ TableSource<T, SplitT, StateT> createSource(TableSourceFactoryContext context) {
+ return () -> {
+ // Load the JDBC driver in to DriverManager
+ try {
+ Class.forName(DRIVER_CLASS_NAME);
+ } catch (Exception e) {
+ log.warn("Failed to load JDBC driver {}", DRIVER_CLASS_NAME, e);
+ }
+ List<CatalogTable> catalogTables =
+ CatalogTableUtil.getCatalogTables(
+ context.getOptions(), context.getClassLoader());
+ Optional<List<JdbcSourceTableConfig>> tableConfigs =
+ context.getOptions()
+ .getOptional(Db2IncrementalSourceOptions.TABLE_NAMES_CONFIG);
+ if (tableConfigs.isPresent()) {
+ catalogTables =
+ CatalogTableUtils.mergeCatalogTableConfig(
+ catalogTables,
+ tableConfigs.get(),
+ text -> TablePath.of(text, true));
+ }
+ return new Db2IncrementalSource(context.getOptions(), catalogTables);
+ };
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceOptions.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceOptions.java
new file mode 100644
index 0000000000..1404a9e663
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceOptions.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source;
+
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.SingleChoiceOption;
+import org.apache.seatunnel.connectors.cdc.base.option.JdbcSourceOptions;
+import org.apache.seatunnel.connectors.cdc.base.option.StartupMode;
+import org.apache.seatunnel.connectors.cdc.base.option.StopMode;
+
+import java.util.Arrays;
+
+public class Db2IncrementalSourceOptions extends JdbcSourceOptions {
+ public static final SingleChoiceOption<StartupMode> STARTUP_MODE =
+ (SingleChoiceOption)
+ Options.key("startup.mode")
+ .singleChoice(
+ StartupMode.class,
+ Arrays.asList(
+ StartupMode.INITIAL,
+ StartupMode.EARLIEST,
+ StartupMode.LATEST))
+ .defaultValue(StartupMode.INITIAL)
+ .withDescription(
+ "Optional startup mode for CDC source, valid enumerations are "
+ + "\"initial\", \"earliest\" and \"latest\"");
+
+ public static final SingleChoiceOption<StopMode> STOP_MODE =
+ (SingleChoiceOption)
+ Options.key("stop.mode")
+ .singleChoice(StopMode.class, Arrays.asList(StopMode.NEVER))
+ .defaultValue(StopMode.NEVER)
+ .withDescription(
+ "Optional stop mode for CDC source, valid enumerations are "
+ + "\"never\"");
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/enumerator/Db2ChunkSplitter.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/enumerator/Db2ChunkSplitter.java
new file mode 100644
index 0000000000..6456ea8851
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/enumerator/Db2ChunkSplitter.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.enumerator;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceConfig;
+import org.apache.seatunnel.connectors.cdc.base.dialect.JdbcDataSourceDialect;
+import org.apache.seatunnel.connectors.cdc.base.source.enumerator.splitter.AbstractJdbcSourceChunkSplitter;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2TypeUtils;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2Utils;
+
+import io.debezium.jdbc.JdbcConnection;
+import io.debezium.relational.Column;
+import io.debezium.relational.Table;
+import io.debezium.relational.TableId;
+import lombok.extern.slf4j.Slf4j;
+
+import java.sql.SQLException;
+
+/** The {@code ChunkSplitter} used to split table into a set of chunks for JDBC data source. */
+@Slf4j
+public class Db2ChunkSplitter extends AbstractJdbcSourceChunkSplitter {
+
+ public Db2ChunkSplitter(JdbcSourceConfig sourceConfig, JdbcDataSourceDialect dialect) {
+ super(sourceConfig, dialect);
+ }
+
+ @Override
+ public Object[] queryMinMax(JdbcConnection jdbc, TableId tableId, String columnName)
+ throws SQLException {
+ return Db2Utils.queryMinMax(jdbc, tableId, columnName);
+ }
+
+ @Override
+ public Object queryMin(
+ JdbcConnection jdbc, TableId tableId, String columnName, Object excludedLowerBound)
+ throws SQLException {
+ return Db2Utils.queryMin(jdbc, tableId, columnName, excludedLowerBound);
+ }
+
+ @Override
+ public Object[] sampleDataFromColumn(
+ JdbcConnection jdbc, TableId tableId, String columnName, int inverseSamplingRate)
+ throws Exception {
+ return Db2Utils.skipReadAndSortSampleData(jdbc, tableId, columnName, inverseSamplingRate);
+ }
+
+ @Override
+ public Object queryNextChunkMax(
+ JdbcConnection jdbc,
+ TableId tableId,
+ String columnName,
+ int chunkSize,
+ Object includedLowerBound)
+ throws SQLException {
+ return Db2Utils.queryNextChunkMax(jdbc, tableId, columnName, chunkSize, includedLowerBound);
+ }
+
+ @Override
+ public Long queryApproximateRowCnt(JdbcConnection jdbc, TableId tableId) throws SQLException {
+ return Db2Utils.queryApproximateRowCnt(jdbc, tableId);
+ }
+
+ @Override
+ public String buildSplitScanQuery(
+ Table table, SeaTunnelRowType splitKeyType, boolean isFirstSplit, boolean isLastSplit) {
+ return Db2Utils.buildSplitScanQuery(table.id(), splitKeyType, isFirstSplit, isLastSplit);
+ }
+
+ @Override
+ public SeaTunnelDataType<?> fromDbzColumn(Column splitColumn) {
+ return Db2TypeUtils.convertFromColumn(splitColumn);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/offset/LsnOffset.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/offset/LsnOffset.java
new file mode 100644
index 0000000000..8aa5a617f1
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/offset/LsnOffset.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset;
+
+import org.apache.seatunnel.connectors.cdc.base.source.offset.Offset;
+
+import io.debezium.connector.db2.Lsn;
+import io.debezium.connector.db2.SourceInfo;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class LsnOffset extends Offset {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final LsnOffset INITIAL_OFFSET = new LsnOffset(null, null);
+ public static final LsnOffset NO_STOPPING_OFFSET =
+ valueOf(Lsn.valueOf(new byte[] {Byte.MAX_VALUE}).toString());
+
+ public static LsnOffset valueOf(String commitLsn) {
+ return new LsnOffset(Lsn.valueOf(commitLsn), null);
+ }
+
+ public static LsnOffset valueOf(String commitLsn, String changeLsn) {
+ return new LsnOffset(Lsn.valueOf(commitLsn), Lsn.valueOf(changeLsn));
+ }
+
+ public static LsnOffset valueOf(Map<String, ?> offset) {
+ Object commitLsn = offset.get(SourceInfo.COMMIT_LSN_KEY);
+ Object changeLsn = offset.get(SourceInfo.CHANGE_LSN_KEY);
+ return new LsnOffset(
+ Lsn.valueOf(commitLsn == null ? null : commitLsn.toString()),
+ Lsn.valueOf(changeLsn == null ? null : changeLsn.toString()));
+ }
+
+ private LsnOffset(Lsn commitLsn, Lsn changeLsn) {
+ Map<String, String> offsetMap = new HashMap<>();
+
+ if (commitLsn != null && commitLsn.isAvailable()) {
+ offsetMap.put(SourceInfo.COMMIT_LSN_KEY, commitLsn.toString());
+ }
+ if (changeLsn != null && changeLsn.isAvailable()) {
+ offsetMap.put(SourceInfo.CHANGE_LSN_KEY, changeLsn.toString());
+ }
+
+ this.offset = offsetMap;
+ }
+
+ public Lsn getChangeLsn() {
+ return Lsn.valueOf(offset.get(SourceInfo.CHANGE_LSN_KEY));
+ }
+
+ public Lsn getCommitLsn() {
+ return Lsn.valueOf(offset.get(SourceInfo.COMMIT_LSN_KEY));
+ }
+
+ public int compareTo(Offset o) {
+ LsnOffset that = (LsnOffset) o;
+ final int comparison = getCommitLsn().compareTo(that.getCommitLsn());
+ // Snapshot high-watermarks only carry a commit LSN. In that case commit LSN ordering is
+ // the boundary contract and row-level in-transaction LSN must not keep the bounded reader
+ // running forever on events from the same commit.
+ if (comparison == 0
+ && (!getChangeLsn().isAvailable() || !that.getChangeLsn().isAvailable())) {
+ return 0;
+ }
+ return comparison == 0 ? getChangeLsn().compareTo(that.getChangeLsn()) : comparison;
+ }
+
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ LsnOffset other = (LsnOffset) obj;
+ return offset.equals(other.offset);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((getCommitLsn() == null) ? 0 : getCommitLsn().hashCode());
+ result = prime * result + ((getChangeLsn() == null) ? 0 : getChangeLsn().hashCode());
+ return result;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/offset/LsnOffsetFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/offset/LsnOffsetFactory.java
new file mode 100644
index 0000000000..225f72ed94
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/offset/LsnOffsetFactory.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset;
+
+import org.apache.seatunnel.connectors.cdc.base.source.offset.Offset;
+import org.apache.seatunnel.connectors.cdc.base.source.offset.OffsetFactory;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfigFactory;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.Db2Dialect;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2Utils;
+
+import io.debezium.connector.db2.Db2Connection;
+import io.debezium.jdbc.JdbcConnection;
+
+import java.util.Map;
+
+public class LsnOffsetFactory extends OffsetFactory {
+
+ private final Db2SourceConfig sourceConfig;
+
+ private final Db2Dialect dialect;
+
+ public LsnOffsetFactory(Db2SourceConfigFactory configFactory, Db2Dialect dialect) {
+ this.sourceConfig = configFactory.create(0);
+ this.dialect = dialect;
+ }
+
+ @Override
+ public Offset earliest() {
+ return LsnOffset.INITIAL_OFFSET;
+ }
+
+ @Override
+ public Offset neverStop() {
+ return LsnOffset.NO_STOPPING_OFFSET;
+ }
+
+ @Override
+ public Offset latest() {
+ try (JdbcConnection jdbcConnection = dialect.openJdbcConnection(sourceConfig)) {
+ return Db2Utils.currentLsn((Db2Connection) jdbcConnection);
+ } catch (Exception e) {
+ throw new RuntimeException("Read the binlog offset error", e);
+ }
+ }
+
+ @Override
+ public Offset specific(Map<String, String> offset) {
+ return LsnOffset.valueOf(offset);
+ }
+
+ @Override
+ public Offset specific(String filename, Long position) {
+ throw new UnsupportedOperationException(
+ "not supported create new Offset by filename and position.");
+ }
+
+ @Override
+ public Offset timestamp(long timestamp) {
+ throw new UnsupportedOperationException("not supported create new Offset by timestamp.");
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/Db2SourceFetchTaskContext.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/Db2SourceFetchTaskContext.java
new file mode 100644
index 0000000000..7ad6c83545
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/Db2SourceFetchTaskContext.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.cdc.base.dialect.JdbcDataSourceDialect;
+import org.apache.seatunnel.connectors.cdc.base.relational.JdbcSourceEventDispatcher;
+import org.apache.seatunnel.connectors.cdc.base.source.offset.Offset;
+import org.apache.seatunnel.connectors.cdc.base.source.reader.external.JdbcSourceFetchTaskContext;
+import org.apache.seatunnel.connectors.cdc.base.source.split.SourceSplitBase;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.config.Db2SourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset.LsnOffset;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2ConnectionUtils;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2Utils;
+
+import org.apache.kafka.connect.data.Struct;
+import org.apache.kafka.connect.source.SourceRecord;
+
+import io.debezium.connector.base.ChangeEventQueue;
+import io.debezium.connector.db2.Db2Connection;
+import io.debezium.connector.db2.Db2Connector;
+import io.debezium.connector.db2.Db2ConnectorConfig;
+import io.debezium.connector.db2.Db2DatabaseSchema;
+import io.debezium.connector.db2.Db2OffsetContext;
+import io.debezium.connector.db2.Db2Partition;
+import io.debezium.connector.db2.Db2TaskContext;
+import io.debezium.connector.db2.Db2TopicSelector;
+import io.debezium.connector.db2.SourceInfo;
+import io.debezium.data.Envelope;
+import io.debezium.heartbeat.DefaultHeartbeatConnectionProvider;
+import io.debezium.heartbeat.HeartbeatFactory;
+import io.debezium.pipeline.DataChangeEvent;
+import io.debezium.pipeline.ErrorHandler;
+import io.debezium.pipeline.metrics.DefaultChangeEventSourceMetricsFactory;
+import io.debezium.pipeline.metrics.SnapshotChangeEventSourceMetrics;
+import io.debezium.pipeline.source.spi.EventMetadataProvider;
+import io.debezium.pipeline.spi.OffsetContext;
+import io.debezium.relational.Table;
+import io.debezium.relational.TableId;
+import io.debezium.relational.Tables;
+import io.debezium.schema.DataCollectionId;
+import io.debezium.schema.TopicSelector;
+import io.debezium.util.Collect;
+import lombok.extern.slf4j.Slf4j;
+
+import java.sql.SQLException;
+import java.time.Instant;
+import java.util.Map;
+
+/** The context for fetch task that fetching data of snapshot split from Db2 data source. */
+@Slf4j
+public class Db2SourceFetchTaskContext extends JdbcSourceFetchTaskContext {
+
+ private final Db2Connection dataConnection;
+
+ private Db2Connection metadataConnection;
+
+ private final Db2EventMetadataProvider metadataProvider;
+ private Db2DatabaseSchema databaseSchema;
+ private Db2OffsetContext offsetContext;
+ private Db2Partition partition;
+ private TopicSelector<TableId> topicSelector;
+ private JdbcSourceEventDispatcher<Db2Partition> dispatcher;
+ private ChangeEventQueue<DataChangeEvent> queue;
+ private ErrorHandler errorHandler;
+ private Db2TaskContext taskContext;
+
+ private SnapshotChangeEventSourceMetrics<Db2Partition> snapshotChangeEventSourceMetrics;
+
+ public Db2SourceFetchTaskContext(
+ Db2SourceConfig sourceConfig, JdbcDataSourceDialect dataSourceDialect) {
+ super(sourceConfig, dataSourceDialect);
+
+ this.dataConnection =
+ Db2ConnectionUtils.createDb2Connection(sourceConfig.getDbzConfiguration());
+ try {
+ this.dataConnection.setAutoCommit(false);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to disable auto commit for Db2 CDC connection", e);
+ }
+ this.metadataProvider = new Db2EventMetadataProvider();
+ }
+
+ @Override
+ public void configure(SourceSplitBase sourceSplitBase) {
+ super.registerDatabaseHistory(sourceSplitBase, dataConnection);
+
+ // initial stateful objects
+ final Db2ConnectorConfig connectorConfig = getDbzConnectorConfig();
+
+ this.topicSelector = Db2TopicSelector.defaultSelector(connectorConfig);
+
+ this.databaseSchema = Db2Utils.createDb2DatabaseSchema(connectorConfig, dataConnection);
+
+ String serverName = connectorConfig.getLogicalName();
+ this.partition = new Db2Partition(serverName);
+
+ this.offsetContext =
+ loadStartingOffsetState(
+ new Db2OffsetContext.Loader(connectorConfig), sourceSplitBase);
+ validateAndLoadDatabaseHistory(offsetContext, databaseSchema);
+
+ this.taskContext = new Db2TaskContext(connectorConfig, databaseSchema);
+
+ // If in the snapshot read phase and enable exactly-once, the queue needs to be set to a
+ // maximum size of `Integer.MAX_VALUE` (buffered a current snapshot all data). otherwise,
+ // use the configuration queue size.
+ final int queueSize =
+ sourceSplitBase.isSnapshotSplit() && isExactlyOnce()
+ ? Integer.MAX_VALUE
+ : getSourceConfig().getDbzConnectorConfig().getMaxQueueSize();
+
+ this.queue =
+ new ChangeEventQueue.Builder<DataChangeEvent>()
+ .pollInterval(connectorConfig.getPollInterval())
+ .maxBatchSize(connectorConfig.getMaxBatchSize())
+ .maxQueueSize(queueSize)
+ .maxQueueSizeInBytes(connectorConfig.getMaxQueueSizeInBytes())
+ .loggingContextSupplier(
+ () -> taskContext.configureLoggingContext("db2-cdc-connector-task"))
+ // do not buffer any element, we use signal event
+ // .buffering()
+ .build();
+ this.dispatcher =
+ new JdbcSourceEventDispatcher<>(
+ connectorConfig,
+ topicSelector,
+ databaseSchema,
+ queue,
+ connectorConfig.getTableFilters().dataCollectionFilter(),
+ DataChangeEvent::new,
+ metadataProvider,
+ new HeartbeatFactory<>(
+ connectorConfig,
+ topicSelector,
+ schemaNameAdjuster,
+ new DefaultHeartbeatConnectionProvider(dataConnection),
+ null),
+ schemaNameAdjuster);
+
+ final DefaultChangeEventSourceMetricsFactory<Db2Partition> changeEventSourceMetricsFactory =
+ new DefaultChangeEventSourceMetricsFactory();
+
+ this.snapshotChangeEventSourceMetrics =
+ changeEventSourceMetricsFactory.getSnapshotMetrics(
+ taskContext, queue, metadataProvider);
+
+ this.errorHandler = new ErrorHandler(Db2Connector.class, connectorConfig, queue);
+ if (sourceSplitBase.isIncrementalSplit() || isExactlyOnce()) {
+ initMetadataConnection();
+ }
+ }
+
+ private void initMetadataConnection() {
+ if (this.metadataConnection == null) {
+ synchronized (this) {
+ if (this.metadataConnection == null) {
+ this.metadataConnection =
+ Db2ConnectionUtils.createDb2Connection(
+ sourceConfig.getDbzConfiguration());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ this.dataConnection.close();
+ if (this.metadataConnection != null) {
+ this.metadataConnection.close();
+ }
+ } catch (SQLException e) {
+ log.warn("Failed to close connection", e);
+ }
+ }
+
+ @Override
+ public Db2SourceConfig getSourceConfig() {
+ return (Db2SourceConfig) sourceConfig;
+ }
+
+ public Db2Connection getDataConnection() {
+ return dataConnection;
+ }
+
+ public Db2Connection getMetadataConnection() {
+ return metadataConnection;
+ }
+
+ public SnapshotChangeEventSourceMetrics<Db2Partition> getSnapshotChangeEventSourceMetrics() {
+ return snapshotChangeEventSourceMetrics;
+ }
+
+ @Override
+ public Db2ConnectorConfig getDbzConnectorConfig() {
+ return (Db2ConnectorConfig) super.getDbzConnectorConfig();
+ }
+
+ @Override
+ public Db2OffsetContext getOffsetContext() {
+ return offsetContext;
+ }
+
+ @Override
+ public Db2Partition getPartition() {
+ return partition;
+ }
+
+ @Override
+ public ErrorHandler getErrorHandler() {
+ return errorHandler;
+ }
+
+ @Override
+ public Db2DatabaseSchema getDatabaseSchema() {
+ return databaseSchema;
+ }
+
+ @Override
+ public SeaTunnelRowType getSplitType(Table table) {
+ return Db2Utils.getSplitType(table);
+ }
+
+ @Override
+ public JdbcSourceEventDispatcher<Db2Partition> getDispatcher() {
+ return dispatcher;
+ }
+
+ @Override
+ public ChangeEventQueue<DataChangeEvent> getQueue() {
+ return queue;
+ }
+
+ @Override
+ public Tables.TableFilter getTableFilter() {
+ return getDbzConnectorConfig().getTableFilters().dataCollectionFilter();
+ }
+
+ @Override
+ public Offset getStreamOffset(SourceRecord sourceRecord) {
+ return Db2Utils.getLsn(sourceRecord);
+ }
+
+ private void validateAndLoadDatabaseHistory(Db2OffsetContext offset, Db2DatabaseSchema schema) {
+ schema.initializeStorage();
+ schema.recover(partition, offset);
+ }
+
+ /** Loads the connector's persistent offset (if present) via the given loader. */
+ private Db2OffsetContext loadStartingOffsetState(
+ Db2OffsetContext.Loader loader, SourceSplitBase split) {
+ Offset offset =
+ split.isSnapshotSplit()
+ ? LsnOffset.INITIAL_OFFSET
+ : split.asIncrementalSplit().getStartupOffset();
+
+ return loader.load(offset.getOffset());
+ }
+
+ public static class Db2EventMetadataProvider implements EventMetadataProvider {
+
+ @Override
+ public Instant getEventTimestamp(
+ DataCollectionId source, OffsetContext offset, Object key, Struct value) {
+ if (value == null) {
+ return null;
+ }
+ final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
+ if (source == null) {
+ return null;
+ }
+ final Long timestamp = sourceInfo.getInt64(SourceInfo.TIMESTAMP_KEY);
+ return timestamp == null ? null : Instant.ofEpochMilli(timestamp);
+ }
+
+ @Override
+ public Map<String, String> getEventSourcePosition(
+ DataCollectionId source, OffsetContext offset, Object key, Struct value) {
+ if (value == null) {
+ return null;
+ }
+ final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
+ if (source == null) {
+ return null;
+ }
+ return Collect.hashMapOf(
+ SourceInfo.COMMIT_LSN_KEY, sourceInfo.getString(SourceInfo.COMMIT_LSN_KEY),
+ SourceInfo.CHANGE_LSN_KEY, sourceInfo.getString(SourceInfo.CHANGE_LSN_KEY));
+ }
+
+ @Override
+ public String getTransactionId(
+ DataCollectionId source, OffsetContext offset, Object key, Struct value) {
+ if (value == null) {
+ return null;
+ }
+ final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE);
+ if (source == null) {
+ return null;
+ }
+ return sourceInfo.getString(SourceInfo.COMMIT_LSN_KEY);
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/scan/Db2SnapshotFetchTask.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/scan/Db2SnapshotFetchTask.java
new file mode 100644
index 0000000000..0a6be22e98
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/scan/Db2SnapshotFetchTask.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.scan;
+
+import org.apache.seatunnel.connectors.cdc.base.relational.JdbcSourceEventDispatcher;
+import org.apache.seatunnel.connectors.cdc.base.source.reader.external.FetchTask;
+import org.apache.seatunnel.connectors.cdc.base.source.split.IncrementalSplit;
+import org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit;
+import org.apache.seatunnel.connectors.cdc.base.source.split.SourceSplitBase;
+import org.apache.seatunnel.connectors.cdc.base.source.split.wartermark.WatermarkKind;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.Db2SourceFetchTaskContext;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.transactionlog.Db2TransactionLogFetchTask;
+
+import io.debezium.config.Configuration;
+import io.debezium.connector.db2.Db2ConnectorConfig;
+import io.debezium.connector.db2.Db2OffsetContext;
+import io.debezium.connector.db2.Db2Partition;
+import io.debezium.heartbeat.Heartbeat;
+import io.debezium.pipeline.source.spi.ChangeEventSource;
+import io.debezium.pipeline.spi.SnapshotResult;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Map;
+
+@Slf4j
+public class Db2SnapshotFetchTask implements FetchTask<SourceSplitBase> {
+
+ private final SnapshotSplit split;
+
+ private volatile boolean taskRunning = false;
+
+ private Db2SnapshotSplitReadTask snapshotSplitReadTask;
+
+ public Db2SnapshotFetchTask(SnapshotSplit split) {
+ this.split = split;
+ }
+
+ @Override
+ public void execute(FetchTask.Context context) throws Exception {
+ Db2SourceFetchTaskContext sourceFetchContext = (Db2SourceFetchTaskContext) context;
+ taskRunning = true;
+ snapshotSplitReadTask =
+ new Db2SnapshotSplitReadTask(
+ sourceFetchContext.getDbzConnectorConfig(),
+ sourceFetchContext.getOffsetContext(),
+ sourceFetchContext.getSnapshotChangeEventSourceMetrics(),
+ sourceFetchContext.getDatabaseSchema(),
+ sourceFetchContext.getDataConnection(),
+ sourceFetchContext.getDispatcher(),
+ split);
+ SnapshotSplitChangeEventSourceContext changeEventSourceContext =
+ new SnapshotSplitChangeEventSourceContext();
+
+ SnapshotResult<Db2OffsetContext> snapshotResult =
+ snapshotSplitReadTask.execute(
+ changeEventSourceContext,
+ sourceFetchContext.getPartition(),
+ sourceFetchContext.getOffsetContext());
+ if (!snapshotResult.isCompletedOrSkipped()) {
+ taskRunning = false;
+ throw new IllegalStateException(
+ String.format("Read snapshot for split %s fail", split));
+ }
+
+ boolean changed =
+ changeEventSourceContext
+ .getHighWatermark()
+ .isAfter(changeEventSourceContext.getLowWatermark());
+ if (!context.isExactlyOnce()) {
+ taskRunning = false;
+ if (changed) {
+ log.debug("Skip merge changelog(exactly-once) for snapshot split {}", split);
+ }
+ return;
+ }
+
+ final IncrementalSplit backfillSplit = createBackFillLsnSplit(changeEventSourceContext);
+ // optimization that skip the binlog read when the low watermark equals high
+ // watermark
+ if (!changed) {
+ dispatchLsnEndEvent(
+ backfillSplit,
+ sourceFetchContext.getPartition().getSourcePartition(),
+ sourceFetchContext.getDispatcher());
+ taskRunning = false;
+ return;
+ }
+
+ // execute stream read task
+ final Db2TransactionLogFetchTask.TransactionLogSplitReadTask backfillReadTask =
+ createBackFillLsnSplitReadTask(backfillSplit, sourceFetchContext);
+ Db2OffsetContext db2OffsetContext =
+ new Db2OffsetContext.Loader(sourceFetchContext.getDbzConnectorConfig())
+ .load(backfillSplit.getStartupOffset().getOffset());
+ log.info(
+ "start execute backfillReadTask, start offset : {}, stop offset : {}",
+ backfillSplit.getStartupOffset(),
+ backfillSplit.getStopOffset());
+ backfillReadTask.execute(
+ new SnapshotBinlogSplitChangeEventSourceContext(),
+ sourceFetchContext.getPartition(),
+ db2OffsetContext);
+ log.info("backfillReadTask execute end");
+ }
+
+ private IncrementalSplit createBackFillLsnSplit(
+ SnapshotSplitChangeEventSourceContext sourceContext) {
+ return new IncrementalSplit(
+ split.splitId(),
+ Collections.singletonList(split.getTableId()),
+ sourceContext.getLowWatermark(),
+ sourceContext.getHighWatermark(),
+ new ArrayList<>());
+ }
+
+ private Db2TransactionLogFetchTask.TransactionLogSplitReadTask createBackFillLsnSplitReadTask(
+ IncrementalSplit backfillBinlogSplit, Db2SourceFetchTaskContext context) {
+ // we should only capture events for the current table,
+ // otherwise, we may can't find corresponding schema
+ Configuration dezConf =
+ context.getSourceConfig()
+ .getDbzConfiguration()
+ .edit()
+ .with(
+ "table.include.list",
+ split.getTableId()
+ .toString()
+ .substring(split.getTableId().toString().indexOf(".") + 1))
+ // Disable heartbeat event in snapshot split fetcher
+ .with(Heartbeat.HEARTBEAT_INTERVAL, 0)
+ .build();
+ // task to read binlog and backfill for current split
+ return new Db2TransactionLogFetchTask.TransactionLogSplitReadTask(
+ new Db2ConnectorConfig(dezConf),
+ context.getDataConnection(),
+ context.getMetadataConnection(),
+ context.getDispatcher(),
+ context.getErrorHandler(),
+ context.getDatabaseSchema(),
+ backfillBinlogSplit);
+ }
+
+ private void dispatchLsnEndEvent(
+ IncrementalSplit backFillBinlogSplit,
+ Map<String, ?> sourcePartition,
+ JdbcSourceEventDispatcher<Db2Partition> eventDispatcher)
+ throws InterruptedException {
+ eventDispatcher.dispatchWatermarkEvent(
+ sourcePartition,
+ backFillBinlogSplit,
+ backFillBinlogSplit.getStopOffset(),
+ WatermarkKind.END);
+ }
+
+ @Override
+ public boolean isRunning() {
+ return taskRunning;
+ }
+
+ @Override
+ public void shutdown() {
+ taskRunning = false;
+ }
+
+ @Override
+ public SourceSplitBase getSplit() {
+ return split;
+ }
+
+ /**
+ * The {@link ChangeEventSource.ChangeEventSourceContext} implementation for bounded stream task
+ * of a snapshot split task.
+ */
+ public class SnapshotBinlogSplitChangeEventSourceContext
+ implements ChangeEventSource.ChangeEventSourceContext {
+
+ public void finished() {
+ taskRunning = false;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return taskRunning;
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/scan/Db2SnapshotSplitReadTask.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/scan/Db2SnapshotSplitReadTask.java
new file mode 100644
index 0000000000..ae54d6b4f2
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/scan/Db2SnapshotSplitReadTask.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.scan;
+
+import org.apache.seatunnel.connectors.cdc.base.relational.JdbcSourceEventDispatcher;
+import org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit;
+import org.apache.seatunnel.connectors.cdc.base.source.split.wartermark.WatermarkKind;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset.LsnOffset;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils.Db2Utils;
+
+import org.apache.kafka.connect.errors.ConnectException;
+
+import io.debezium.DebeziumException;
+import io.debezium.connector.db2.Db2Connection;
+import io.debezium.connector.db2.Db2ConnectorConfig;
+import io.debezium.connector.db2.Db2DatabaseSchema;
+import io.debezium.connector.db2.Db2OffsetContext;
+import io.debezium.connector.db2.Db2Partition;
+import io.debezium.pipeline.EventDispatcher;
+import io.debezium.pipeline.source.AbstractSnapshotChangeEventSource;
+import io.debezium.pipeline.source.spi.ChangeEventSource;
+import io.debezium.pipeline.source.spi.SnapshotProgressListener;
+import io.debezium.pipeline.spi.ChangeRecordEmitter;
+import io.debezium.pipeline.spi.SnapshotResult;
+import io.debezium.relational.RelationalSnapshotChangeEventSource;
+import io.debezium.relational.SnapshotChangeRecordEmitter;
+import io.debezium.relational.Table;
+import io.debezium.relational.TableId;
+import io.debezium.util.Clock;
+import io.debezium.util.ColumnUtils;
+import io.debezium.util.Strings;
+import io.debezium.util.Threads;
+import lombok.extern.slf4j.Slf4j;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.time.Duration;
+
+@Slf4j
+public class Db2SnapshotSplitReadTask
+ extends AbstractSnapshotChangeEventSource<Db2Partition, Db2OffsetContext> {
+
+ /** Interval for showing a log statement with the progress while scanning a single table. */
+ private static final Duration LOG_INTERVAL = Duration.ofMillis(10_000);
+
+ private final Db2ConnectorConfig connectorConfig;
+ private final Db2DatabaseSchema databaseSchema;
+ private final Db2Connection jdbcConnection;
+ private final JdbcSourceEventDispatcher<Db2Partition> dispatcher;
+ private final Clock clock;
+ private final SnapshotSplit snapshotSplit;
+ private final Db2OffsetContext offsetContext;
+ private final SnapshotProgressListener<Db2Partition> snapshotProgressListener;
+
+ public Db2SnapshotSplitReadTask(
+ Db2ConnectorConfig connectorConfig,
+ Db2OffsetContext previousOffset,
+ SnapshotProgressListener<Db2Partition> snapshotProgressListener,
+ Db2DatabaseSchema databaseSchema,
+ Db2Connection jdbcConnection,
+ JdbcSourceEventDispatcher<Db2Partition> dispatcher,
+ SnapshotSplit snapshotSplit) {
+ super(connectorConfig, snapshotProgressListener);
+ this.offsetContext = previousOffset;
+ this.connectorConfig = connectorConfig;
+ this.databaseSchema = databaseSchema;
+ this.jdbcConnection = jdbcConnection;
+ this.dispatcher = dispatcher;
+ this.clock = Clock.SYSTEM;
+ this.snapshotSplit = snapshotSplit;
+ this.snapshotProgressListener = snapshotProgressListener;
+ }
+
+ @Override
+ public SnapshotResult<Db2OffsetContext> execute(
+ ChangeEventSource.ChangeEventSourceContext context,
+ Db2Partition partition,
+ Db2OffsetContext previousOffset)
+ throws InterruptedException {
+ SnapshottingTask snapshottingTask = getSnapshottingTask(partition, previousOffset);
+ final SnapshotContext<Db2Partition, Db2OffsetContext> ctx;
+ try {
+ ctx = prepare(partition);
+ } catch (Exception e) {
+ log.error("Failed to initialize snapshot context.", e);
+ throw new RuntimeException(e);
+ }
+ try {
+ return doExecute(context, previousOffset, ctx, snapshottingTask);
+ } catch (InterruptedException e) {
+ log.warn("Snapshot was interrupted before completion");
+ throw e;
+ } catch (Exception t) {
+ throw new DebeziumException(t);
+ }
+ }
+
+ @Override
+ protected SnapshotResult doExecute(
+ ChangeEventSource.ChangeEventSourceContext context,
+ Db2OffsetContext previousOffset,
+ SnapshotContext<Db2Partition, Db2OffsetContext> snapshotContext,
+ AbstractSnapshotChangeEventSource.SnapshottingTask snapshottingTask)
+ throws Exception {
+ final Db2SnapshotContext ctx = (Db2SnapshotContext) snapshotContext;
+ ctx.offset = offsetContext;
+
+ final LsnOffset lowWatermark = Db2Utils.currentLsn(jdbcConnection);
+ log.info(
+ "Snapshot step 1 - Determining low watermark {} for split {}",
+ lowWatermark,
+ snapshotSplit);
+ ((SnapshotSplitChangeEventSourceContext) context).setLowWatermark(lowWatermark);
+ dispatcher.dispatchWatermarkEvent(
+ ctx.partition.getSourcePartition(), snapshotSplit, lowWatermark, WatermarkKind.LOW);
+
+ log.info("Snapshot step 2 - Snapshotting data");
+ createDataEvents(ctx, snapshotSplit.getTableId());
+
+ final LsnOffset highWatermark = Db2Utils.currentLsn(jdbcConnection);
+ log.info(
+ "Snapshot step 3 - Determining high watermark {} for split {}",
+ highWatermark,
+ snapshotSplit);
+ ((SnapshotSplitChangeEventSourceContext) context).setHighWatermark(highWatermark);
+ dispatcher.dispatchWatermarkEvent(
+ ctx.partition.getSourcePartition(),
+ snapshotSplit,
+ highWatermark,
+ WatermarkKind.HIGH);
+ return SnapshotResult.completed(ctx.offset);
+ }
+
+ @Override
+ protected AbstractSnapshotChangeEventSource.SnapshottingTask getSnapshottingTask(
+ Db2Partition partition, Db2OffsetContext previousOffset) {
+ return new SnapshottingTask(false, true);
+ }
+
+ @Override
+ protected Db2SnapshotContext prepare(Db2Partition partition) throws Exception {
+ return new Db2SnapshotContext(partition);
+ }
+
+ private void createDataEvents(Db2SnapshotContext snapshotContext, TableId tableId)
+ throws Exception {
+ EventDispatcher.SnapshotReceiver<Db2Partition> snapshotReceiver =
+ dispatcher.getSnapshotChangeEventReceiver();
+ log.debug("Snapshotting table {}", tableId);
+ createDataEventsForTable(
+ snapshotContext, snapshotReceiver, databaseSchema.tableFor(tableId));
+ snapshotReceiver.completeSnapshot();
+ }
+
+ /** Dispatches the data change events for the records of a single table. */
+ private void createDataEventsForTable(
+ Db2SnapshotContext snapshotContext,
+ EventDispatcher.SnapshotReceiver<Db2Partition> snapshotReceiver,
+ Table table)
+ throws InterruptedException {
+
+ long exportStart = clock.currentTimeInMillis();
+ log.info("Exporting data from split '{}' of table {}", snapshotSplit.splitId(), table.id());
+
+ final String selectSql =
+ Db2Utils.buildSplitScanQuery(
+ snapshotSplit.getTableId(),
+ snapshotSplit.getSplitKeyType(),
+ snapshotSplit.getSplitStart() == null,
+ snapshotSplit.getSplitEnd() == null);
+ log.info(
+ "For split '{}' of table {} using select statement: '{}'",
+ snapshotSplit.splitId(),
+ table.id(),
+ selectSql);
+
+ try (PreparedStatement selectStatement =
+ Db2Utils.readTableSplitDataStatement(
+ jdbcConnection,
+ selectSql,
+ snapshotSplit.getSplitStart() == null,
+ snapshotSplit.getSplitEnd() == null,
+ snapshotSplit.getSplitStart(),
+ snapshotSplit.getSplitEnd(),
+ snapshotSplit.getSplitKeyType(),
+ connectorConfig.getSnapshotFetchSize());
+ ResultSet rs = selectStatement.executeQuery()) {
+
+ ColumnUtils.ColumnArray columnArray = ColumnUtils.toArray(rs, table);
+ long rows = 0;
+ Threads.Timer logTimer = getTableScanLogTimer();
+
+ while (rs.next()) {
+ rows++;
+ final Object[] row =
+ jdbcConnection.rowToArray(table, databaseSchema, rs, columnArray);
+ if (logTimer.expired()) {
+ long stop = clock.currentTimeInMillis();
+ log.info(
+ "Exported {} records for split '{}' after {}",
+ rows,
+ snapshotSplit.splitId(),
+ Strings.duration(stop - exportStart));
+ snapshotProgressListener.rowsScanned(
+ snapshotContext.partition, table.id(), rows);
+ logTimer = getTableScanLogTimer();
+ }
+ dispatcher.dispatchSnapshotEvent(
+ snapshotContext.partition,
+ table.id(),
+ getChangeRecordEmitter(snapshotContext, table.id(), row),
+ snapshotReceiver);
+ }
+ log.info(
+ "Finished exporting {} records for split '{}', total duration '{}'",
+ rows,
+ snapshotSplit.splitId(),
+ Strings.duration(clock.currentTimeInMillis() - exportStart));
+ } catch (SQLException e) {
+ throw new ConnectException("Snapshotting of table " + table.id() + " failed", e);
+ }
+ }
+
+ protected ChangeRecordEmitter getChangeRecordEmitter(
+ Db2SnapshotContext snapshotContext, TableId tableId, Object[] row) {
+ snapshotContext.offset.event(tableId, clock.currentTime());
+ return new SnapshotChangeRecordEmitter(
+ snapshotContext.partition, snapshotContext.offset, row, clock);
+ }
+
+ private Threads.Timer getTableScanLogTimer() {
+ return Threads.timer(clock, LOG_INTERVAL);
+ }
+
+ private Object readField(ResultSet rs, int columnIndex) throws SQLException {
+ final ResultSetMetaData metaData = rs.getMetaData();
+ final int columnType = metaData.getColumnType(columnIndex);
+
+ if (columnType == Types.TIME) {
+ return rs.getTimestamp(columnIndex);
+ } else {
+ return rs.getObject(columnIndex);
+ }
+ }
+
+ private static class Db2SnapshotContext
+ extends RelationalSnapshotChangeEventSource.RelationalSnapshotContext<
+ Db2Partition, Db2OffsetContext> {
+
+ public Db2SnapshotContext(Db2Partition partition) throws SQLException {
+ super(partition, "");
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/scan/SnapshotSplitChangeEventSourceContext.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/scan/SnapshotSplitChangeEventSourceContext.java
new file mode 100644
index 0000000000..5cbe67f16c
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/scan/SnapshotSplitChangeEventSourceContext.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.scan;
+
+import org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset.LsnOffset;
+
+import io.debezium.pipeline.source.spi.ChangeEventSource;
+
+/**
+ * {@link ChangeEventSource.ChangeEventSourceContext} implementation that keeps low/high watermark
+ * for each {@link SnapshotSplit}.
+ */
+public class SnapshotSplitChangeEventSourceContext
+ implements ChangeEventSource.ChangeEventSourceContext {
+
+ private LsnOffset lowWatermark;
+ private LsnOffset highWatermark;
+
+ public LsnOffset getLowWatermark() {
+ return lowWatermark;
+ }
+
+ public void setLowWatermark(LsnOffset lowWatermark) {
+ this.lowWatermark = lowWatermark;
+ }
+
+ public LsnOffset getHighWatermark() {
+ return highWatermark;
+ }
+
+ public void setHighWatermark(LsnOffset highWatermark) {
+ this.highWatermark = highWatermark;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return lowWatermark != null && highWatermark != null;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/transactionlog/Db2TransactionLogFetchTask.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/transactionlog/Db2TransactionLogFetchTask.java
new file mode 100644
index 0000000000..a142793f90
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/reader/fetch/transactionlog/Db2TransactionLogFetchTask.java
@@ -0,0 +1,532 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.transactionlog;
+
+import org.apache.seatunnel.connectors.cdc.base.relational.JdbcSourceEventDispatcher;
+import org.apache.seatunnel.connectors.cdc.base.source.reader.external.FetchTask;
+import org.apache.seatunnel.connectors.cdc.base.source.split.IncrementalSplit;
+import org.apache.seatunnel.connectors.cdc.base.source.split.SourceSplitBase;
+import org.apache.seatunnel.connectors.cdc.base.source.split.wartermark.WatermarkKind;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset.LsnOffset;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.Db2SourceFetchTaskContext;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.reader.fetch.scan.Db2SnapshotFetchTask;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.debezium.connector.db2.Db2ChangeRecordEmitter;
+import io.debezium.connector.db2.Db2ChangeTable;
+import io.debezium.connector.db2.Db2Connection;
+import io.debezium.connector.db2.Db2ConnectorConfig;
+import io.debezium.connector.db2.Db2DatabaseSchema;
+import io.debezium.connector.db2.Db2OffsetContext;
+import io.debezium.connector.db2.Db2Partition;
+import io.debezium.connector.db2.Db2SchemaChangeEventEmitter;
+import io.debezium.connector.db2.Lsn;
+import io.debezium.connector.db2.TxLogPosition;
+import io.debezium.pipeline.ErrorHandler;
+import io.debezium.pipeline.source.spi.ChangeEventSource;
+import io.debezium.pipeline.source.spi.ChangeTableResultSet;
+import io.debezium.relational.TableId;
+import io.debezium.schema.DatabaseSchema;
+import io.debezium.schema.SchemaChangeEvent.SchemaChangeEventType;
+import io.debezium.util.Clock;
+import io.debezium.util.Metronome;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset.LsnOffset.NO_STOPPING_OFFSET;
+
+public class Db2TransactionLogFetchTask implements FetchTask<SourceSplitBase> {
+
+ private final IncrementalSplit split;
+ private volatile boolean taskRunning = false;
+
+ public Db2TransactionLogFetchTask(IncrementalSplit split) {
+ this.split = split;
+ }
+
+ @Override
+ public void execute(FetchTask.Context context) throws Exception {
+ Db2SourceFetchTaskContext sourceFetchContext = (Db2SourceFetchTaskContext) context;
+ taskRunning = true;
+
+ TransactionLogSplitReadTask transactionLogSplitReadTask =
+ new TransactionLogSplitReadTask(
+ sourceFetchContext.getDbzConnectorConfig(),
+ sourceFetchContext.getDataConnection(),
+ sourceFetchContext.getMetadataConnection(),
+ sourceFetchContext.getDispatcher(),
+ sourceFetchContext.getErrorHandler(),
+ sourceFetchContext.getDatabaseSchema(),
+ split);
+
+ transactionLogSplitReadTask.execute(
+ new TransactionLogSplitChangeEventSourceContext(),
+ sourceFetchContext.getPartition(),
+ sourceFetchContext.getOffsetContext());
+ }
+
+ @Override
+ public boolean isRunning() {
+ return taskRunning;
+ }
+
+ @Override
+ public void shutdown() {
+ taskRunning = false;
+ }
+
+ @Override
+ public SourceSplitBase getSplit() {
+ return split;
+ }
+
+ /**
+ * Reads Db2 change tables for an incremental split. Debezium's Db2 streaming source has no hook
+ * after each handled LSN, so SeaTunnel keeps this small wrapper to emit the snapshot backfill
+ * END watermark as soon as the bounded stop LSN is reached.
+ */
+ public static class TransactionLogSplitReadTask {
+
+ private static final int COL_COMMIT_LSN = 2;
+ private static final int COL_ROW_LSN = 3;
+ private static final int COL_OPERATION = 1;
+ private static final int COL_DATA = 5;
+ private static final Pattern MISSING_CDC_FUNCTION_CHANGES_ERROR =
+ Pattern.compile("Invalid object name 'cdc.fn_cdc_get_all_changes_(.*)'\\.");
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TransactionLogSplitReadTask.class);
+
+ private final Db2ConnectorConfig connectorConfig;
+ private final Db2Connection dataConnection;
+ private final Db2Connection metadataConnection;
+ private final JdbcSourceEventDispatcher<Db2Partition> dispatcher;
+ private final ErrorHandler errorHandler;
+ private final Clock clock;
+ private final Db2DatabaseSchema schema;
+ private final Duration pollInterval;
+ private final IncrementalSplit lsnSplit;
+
+ public TransactionLogSplitReadTask(
+ Db2ConnectorConfig connectorConfig,
+ Db2Connection dataConnection,
+ Db2Connection metadataConnection,
+ JdbcSourceEventDispatcher<Db2Partition> dispatcher,
+ ErrorHandler errorHandler,
+ Db2DatabaseSchema schema,
+ IncrementalSplit lsnSplit) {
+ this.connectorConfig = connectorConfig;
+ this.dataConnection = dataConnection;
+ this.metadataConnection = metadataConnection;
+ this.dispatcher = dispatcher;
+ this.errorHandler = errorHandler;
+ this.clock = Clock.system();
+ this.schema = schema;
+ this.pollInterval = connectorConfig.getPollInterval();
+ this.lsnSplit = lsnSplit;
+ }
+
+ public void execute(
+ ChangeEventSource.ChangeEventSourceContext context,
+ Db2Partition partition,
+ Db2OffsetContext offsetContext)
+ throws InterruptedException {
+ if (!connectorConfig.getSnapshotMode().shouldStream()) {
+ LOG.info("Streaming is not enabled in current configuration");
+ return;
+ }
+
+ Metronome metronome = Metronome.sleeper(pollInterval, clock);
+ Queue<Db2ChangeTable> schemaChangeCheckpoints =
+ new PriorityQueue<>((x, y) -> x.getStopLsn().compareTo(y.getStopLsn()));
+ try {
+ AtomicReference<Db2ChangeTable[]> tablesSlot =
+ new AtomicReference<>(getCdcTablesToQuery(partition, offsetContext));
+ TxLogPosition lastProcessedPositionOnStart = offsetContext.getChangePosition();
+ long lastProcessedEventSerialNoOnStart = offsetContext.getEventSerialNo();
+ LOG.info(
+ "Last position recorded in offsets is {}[{}]",
+ lastProcessedPositionOnStart,
+ lastProcessedEventSerialNoOnStart);
+
+ TxLogPosition lastProcessedPosition = lastProcessedPositionOnStart;
+ boolean shouldIncreaseFromLsn = offsetContext.isSnapshotCompleted();
+ while (context.isRunning()) {
+ Lsn currentMaxLsn = dataConnection.getMaxLsn();
+ if (!currentMaxLsn.isAvailable()) {
+ LOG.warn(
+ "No maximum LSN recorded in Db2. Please ensure that the Db2 capture agent is running");
+ metronome.pause();
+ continue;
+ }
+ if (currentMaxLsn.equals(lastProcessedPosition.getCommitLsn())
+ && shouldIncreaseFromLsn) {
+ metronome.pause();
+ continue;
+ }
+
+ Lsn fromLsn =
+ lastProcessedPosition.getCommitLsn().isAvailable()
+ && shouldIncreaseFromLsn
+ ? dataConnection.incrementLsn(
+ lastProcessedPosition.getCommitLsn())
+ : lastProcessedPosition.getCommitLsn();
+ shouldIncreaseFromLsn = true;
+
+ while (!schemaChangeCheckpoints.isEmpty()) {
+ migrateTable(partition, offsetContext, schemaChangeCheckpoints);
+ }
+ if (!dataConnection.listOfNewChangeTables(fromLsn, currentMaxLsn).isEmpty()) {
+ Db2ChangeTable[] tables = getCdcTablesToQuery(partition, offsetContext);
+ tablesSlot.set(tables);
+ for (Db2ChangeTable table : tables) {
+ if (table.getStartLsn().isBetween(fromLsn, currentMaxLsn)) {
+ LOG.info("Schema will be changed for {}", table);
+ schemaChangeCheckpoints.add(table);
+ }
+ }
+ }
+
+ AtomicBoolean boundedReadFinished = new AtomicBoolean(false);
+ try {
+ dataConnection.getChangesForTables(
+ tablesSlot.get(),
+ fromLsn,
+ currentMaxLsn,
+ resultSets -> {
+ readChangeTableResultSets(
+ context,
+ partition,
+ offsetContext,
+ schemaChangeCheckpoints,
+ tablesSlot,
+ lastProcessedPositionOnStart,
+ lastProcessedEventSerialNoOnStart,
+ resultSets,
+ boundedReadFinished);
+ });
+ lastProcessedPosition = TxLogPosition.valueOf(currentMaxLsn);
+ dataConnection.rollback();
+ if (boundedReadFinished.get()) {
+ return;
+ }
+ } catch (SQLException e) {
+ tablesSlot.set(processErrorFromChangeTableQuery(e, tablesSlot.get()));
+ }
+ }
+ } catch (Exception e) {
+ errorHandler.setProducerThrowable(e);
+ }
+ }
+
+ private void readChangeTableResultSets(
+ ChangeEventSource.ChangeEventSourceContext context,
+ Db2Partition partition,
+ Db2OffsetContext offsetContext,
+ Queue<Db2ChangeTable> schemaChangeCheckpoints,
+ AtomicReference<Db2ChangeTable[]> tablesSlot,
+ TxLogPosition lastProcessedPositionOnStart,
+ long lastProcessedEventSerialNoOnStart,
+ ResultSet[] resultSets,
+ AtomicBoolean boundedReadFinished)
+ throws SQLException, InterruptedException {
+ long eventSerialNoInInitialTx = 1;
+ int tableCount = resultSets.length;
+ ChangeTablePointer[] changeTables = new ChangeTablePointer[tableCount];
+ Db2ChangeTable[] tables = tablesSlot.get();
+
+ for (int i = 0; i < tableCount; i++) {
+ changeTables[i] = new ChangeTablePointer(tables[i], resultSets[i]);
+ changeTables[i].next();
+ }
+
+ for (; ; ) {
+ ChangeTablePointer tableWithSmallestLsn = getTableWithSmallestLsn(changeTables);
+ if (tableWithSmallestLsn == null || boundedReadFinished.get()) {
+ break;
+ }
+
+ if (!tableWithSmallestLsn.getChangePosition().isAvailable()
+ || !tableWithSmallestLsn.getChangePosition().getInTxLsn().isAvailable()) {
+ LOG.error(
+ "Skipping change {} as its LSN is NULL which is not expected",
+ tableWithSmallestLsn);
+ tableWithSmallestLsn.next();
+ continue;
+ }
+ if (tableWithSmallestLsn.getChangePosition().compareTo(lastProcessedPositionOnStart)
+ < 0) {
+ tableWithSmallestLsn.next();
+ continue;
+ }
+ if (tableWithSmallestLsn.getChangePosition().compareTo(lastProcessedPositionOnStart)
+ == 0
+ && eventSerialNoInInitialTx <= lastProcessedEventSerialNoOnStart) {
+ eventSerialNoInInitialTx++;
+ tableWithSmallestLsn.next();
+ continue;
+ }
+ if (tableWithSmallestLsn.getChangeTable().getStopLsn().isAvailable()
+ && tableWithSmallestLsn
+ .getChangeTable()
+ .getStopLsn()
+ .compareTo(
+ tableWithSmallestLsn
+ .getChangePosition()
+ .getCommitLsn())
+ <= 0) {
+ tableWithSmallestLsn.next();
+ continue;
+ }
+ if (!schemaChangeCheckpoints.isEmpty()
+ && tableWithSmallestLsn
+ .getChangePosition()
+ .getCommitLsn()
+ .compareTo(schemaChangeCheckpoints.peek().getStopLsn())
+ >= 0) {
+ migrateTable(partition, offsetContext, schemaChangeCheckpoints);
+ }
+
+ TableId tableId = tableWithSmallestLsn.getChangeTable().getSourceTableId();
+ int operation = tableWithSmallestLsn.getOperation();
+ Object[] data = tableWithSmallestLsn.getData();
+
+ int eventCount = 1;
+ if (operation == Db2ChangeRecordEmitter.OP_UPDATE_BEFORE) {
+ if (!tableWithSmallestLsn.next()
+ || tableWithSmallestLsn.getOperation()
+ != Db2ChangeRecordEmitter.OP_UPDATE_AFTER) {
+ throw new IllegalStateException(
+ "The update before event at "
+ + tableWithSmallestLsn.getChangePosition()
+ + " for table "
+ + tableId
+ + " was not followed by after event.");
+ }
+ eventCount = 2;
+ }
+ Object[] dataNext =
+ operation == Db2ChangeRecordEmitter.OP_UPDATE_BEFORE
+ ? tableWithSmallestLsn.getData()
+ : null;
+
+ TxLogPosition currentPosition = tableWithSmallestLsn.getChangePosition();
+ offsetContext.setChangePosition(currentPosition, eventCount);
+ offsetContext.event(
+ tableId, metadataConnection.timestampOfLsn(currentPosition.getCommitLsn()));
+
+ dispatcher.dispatchDataChangeEvent(
+ partition,
+ tableId,
+ new Db2ChangeRecordEmitter(
+ partition, offsetContext, operation, data, dataNext, clock));
+
+ if (isBoundedRead()
+ && currentLsnOffset(currentPosition)
+ .isAtOrAfter(lsnSplit.getStopOffset())) {
+ dispatchBoundedReadEndEvent(partition, currentPosition, context);
+ boundedReadFinished.set(true);
+ break;
+ }
+ tableWithSmallestLsn.next();
+ }
+ }
+
+ private ChangeTablePointer getTableWithSmallestLsn(ChangeTablePointer[] changeTables)
+ throws SQLException {
+ ChangeTablePointer tableWithSmallestLsn = null;
+ for (ChangeTablePointer changeTable : changeTables) {
+ if (changeTable.isCompleted()) {
+ continue;
+ }
+ if (tableWithSmallestLsn == null
+ || changeTable.compareTo(tableWithSmallestLsn) < 0) {
+ tableWithSmallestLsn = changeTable;
+ }
+ }
+ return tableWithSmallestLsn;
+ }
+
+ private void migrateTable(
+ Db2Partition partition,
+ Db2OffsetContext offsetContext,
+ Queue<Db2ChangeTable> schemaChangeCheckpoints)
+ throws InterruptedException, SQLException {
+ Db2ChangeTable newTable = schemaChangeCheckpoints.poll();
+ LOG.info("Migrating schema to {}", newTable);
+ dispatcher.dispatchSchemaChangeEvent(
+ partition,
+ newTable.getSourceTableId(),
+ new Db2SchemaChangeEventEmitter(
+ partition,
+ offsetContext,
+ newTable,
+ metadataConnection.getTableSchemaFromTable(newTable),
+ SchemaChangeEventType.ALTER));
+ }
+
+ private Db2ChangeTable[] processErrorFromChangeTableQuery(
+ SQLException exception, Db2ChangeTable[] currentChangeTables) throws SQLException {
+ Matcher matcher = MISSING_CDC_FUNCTION_CHANGES_ERROR.matcher(exception.getMessage());
+ if (matcher.matches()) {
+ String captureName = matcher.group(1);
+ LOG.info("Table is no longer captured with capture instance {}", captureName);
+ return Arrays.stream(currentChangeTables)
+ .filter(
+ changeTable ->
+ !changeTable.getCaptureInstance().equals(captureName))
+ .collect(Collectors.toList())
+ .toArray(new Db2ChangeTable[0]);
+ }
+ throw exception;
+ }
+
+ private Db2ChangeTable[] getCdcTablesToQuery(
+ Db2Partition partition, Db2OffsetContext offsetContext)
+ throws SQLException, InterruptedException {
+ Set<Db2ChangeTable> cdcEnabledTables = dataConnection.listOfChangeTables();
+ if (cdcEnabledTables.isEmpty()) {
+ LOG.warn(
+ "No table has enabled CDC or security constraints prevent getting the list of change tables");
+ }
+
+ Map<TableId, List<Db2ChangeTable>> includedAndCdcEnabledTables =
+ cdcEnabledTables.stream()
+ .filter(
+ changeTable ->
+ connectorConfig
+ .getTableFilters()
+ .dataCollectionFilter()
+ .isIncluded(changeTable.getSourceTableId()))
+ .collect(Collectors.groupingBy(Db2ChangeTable::getSourceTableId));
+
+ if (includedAndCdcEnabledTables.isEmpty()) {
+ LOG.warn(DatabaseSchema.NO_CAPTURED_DATA_COLLECTIONS_WARNING);
+ }
+
+ List<Db2ChangeTable> tables = new ArrayList<>();
+ for (List<Db2ChangeTable> captures : includedAndCdcEnabledTables.values()) {
+ Db2ChangeTable currentTable = captures.get(0);
+ if (captures.size() > 1) {
+ Db2ChangeTable futureTable;
+ if (captures.get(0).getStartLsn().compareTo(captures.get(1).getStartLsn())
+ < 0) {
+ futureTable = captures.get(1);
+ } else {
+ currentTable = captures.get(1);
+ futureTable = captures.get(0);
+ }
+ currentTable.setStopLsn(futureTable.getStartLsn());
+ tables.add(futureTable);
+ LOG.info(
+ "Multiple capture instances present for the same table: {} and {}",
+ currentTable,
+ futureTable);
+ }
+ if (schema.tableFor(currentTable.getSourceTableId()) == null) {
+ LOG.info(
+ "Table {} is new to be monitored by capture instance {}",
+ currentTable.getSourceTableId(),
+ currentTable.getCaptureInstance());
+ offsetContext.event(currentTable.getSourceTableId(), Instant.now());
+ dispatcher.dispatchSchemaChangeEvent(
+ partition,
+ currentTable.getSourceTableId(),
+ new Db2SchemaChangeEventEmitter(
+ partition,
+ offsetContext,
+ currentTable,
+ dataConnection.getTableSchemaFromTable(currentTable),
+ SchemaChangeEventType.CREATE));
+ }
+ tables.add(currentTable);
+ }
+ return tables.toArray(new Db2ChangeTable[tables.size()]);
+ }
+
+ private boolean isBoundedRead() {
+ return !NO_STOPPING_OFFSET.equals(lsnSplit.getStopOffset());
+ }
+
+ private LsnOffset currentLsnOffset(TxLogPosition position) {
+ return LsnOffset.valueOf(
+ position.getCommitLsn().toString(), position.getInTxLsn().toString());
+ }
+
+ private void dispatchBoundedReadEndEvent(
+ Db2Partition partition,
+ TxLogPosition currentPosition,
+ ChangeEventSource.ChangeEventSourceContext context)
+ throws InterruptedException {
+ LsnOffset currentOffset = currentLsnOffset(currentPosition);
+ dispatcher.dispatchWatermarkEvent(
+ partition.getSourcePartition(), lsnSplit, currentOffset, WatermarkKind.END);
+ if (context
+ instanceof Db2SnapshotFetchTask.SnapshotBinlogSplitChangeEventSourceContext) {
+ ((Db2SnapshotFetchTask.SnapshotBinlogSplitChangeEventSourceContext) context)
+ .finished();
+ }
+ }
+
+ private static class ChangeTablePointer
+ extends ChangeTableResultSet<Db2ChangeTable, TxLogPosition> {
+
+ private ChangeTablePointer(Db2ChangeTable changeTable, ResultSet resultSet) {
+ super(changeTable, resultSet, COL_DATA);
+ }
+
+ @Override
+ protected int getOperation(ResultSet resultSet) throws SQLException {
+ return resultSet.getInt(COL_OPERATION);
+ }
+
+ @Override
+ protected TxLogPosition getNextChangePosition(ResultSet resultSet) throws SQLException {
+ return isCompleted()
+ ? TxLogPosition.NULL
+ : TxLogPosition.valueOf(
+ Lsn.valueOf(resultSet.getBytes(COL_COMMIT_LSN)),
+ Lsn.valueOf(resultSet.getBytes(COL_ROW_LSN)));
+ }
+ }
+ }
+
+ private class TransactionLogSplitChangeEventSourceContext
+ implements ChangeEventSource.ChangeEventSourceContext {
+ @Override
+ public boolean isRunning() {
+ return taskRunning;
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2ConnectionUtils.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2ConnectionUtils.java
new file mode 100644
index 0000000000..69aeeddda2
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2ConnectionUtils.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils;
+
+import io.debezium.config.Configuration;
+import io.debezium.connector.db2.Db2Connection;
+import io.debezium.connector.db2.Db2ConnectorConfig;
+import io.debezium.jdbc.JdbcConfiguration;
+
+/** Utils for Db2 connection. */
+public class Db2ConnectionUtils {
+
+ public static Db2Connection createDb2Connection(Configuration dbzConfiguration) {
+ final Db2ConnectorConfig connectorConfig = new Db2ConnectorConfig(dbzConfiguration);
+ return new Db2Connection(JdbcConfiguration.adapt(connectorConfig.getJdbcConfig()));
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2Schema.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2Schema.java
new file mode 100644
index 0000000000..3b61c9c605
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2Schema.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils;
+
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.common.utils.SeaTunnelException;
+import org.apache.seatunnel.connectors.cdc.base.utils.CatalogTableUtils;
+
+import io.debezium.connector.db2.Db2Connection;
+import io.debezium.connector.db2.Db2ConnectorConfig;
+import io.debezium.jdbc.JdbcConnection;
+import io.debezium.relational.Table;
+import io.debezium.relational.TableId;
+import io.debezium.relational.Tables;
+import io.debezium.relational.history.TableChanges;
+import io.debezium.relational.history.TableChanges.TableChange;
+
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/** A component used to get schema by table path. */
+public class Db2Schema {
+
+ private final Db2ConnectorConfig connectorConfig;
+ private final Map<TableId, TableChange> schemasByTableId;
+ private final Map<TableId, CatalogTable> tableMap;
+
+ public Db2Schema(Db2ConnectorConfig connectorConfig, Map<TableId, CatalogTable> tableMap) {
+ this.schemasByTableId = new ConcurrentHashMap<>();
+ this.connectorConfig = connectorConfig;
+ this.tableMap = tableMap;
+ }
+
+ public TableChange getTableSchema(JdbcConnection jdbc, TableId tableId) {
+ // read schema from cache first
+ TableChange schema = schemasByTableId.get(tableId);
+ if (schema == null) {
+ schema = readTableSchema(jdbc, tableId);
+ }
+ return schema;
+ }
+
+ private TableChange readTableSchema(JdbcConnection jdbc, TableId tableId) {
+ Db2Connection db2Connection = (Db2Connection) jdbc;
+ Tables tables = new Tables();
+ try {
+ db2Connection.readSchema(
+ tables,
+ null,
+ tableId.schema(),
+ connectorConfig.getTableFilters().dataCollectionFilter(),
+ null,
+ false);
+ for (TableId id : tables.tableIds()) {
+ if (tableMap.containsKey(id)) {
+ Table table =
+ CatalogTableUtils.mergeCatalogTableConfig(
+ tables.forTable(id), tableMap.get(id));
+ TableChanges.TableChange tableChange =
+ new TableChanges.TableChange(
+ TableChanges.TableChangeType.CREATE, table);
+ schemasByTableId.put(id, tableChange);
+ }
+ }
+ } catch (SQLException e) {
+ throw new SeaTunnelException(
+ String.format("Failed to read schema for table %s ", tableId), e);
+ }
+
+ if (!schemasByTableId.containsKey(tableId)) {
+ throw new SeaTunnelException(
+ String.format("Can't obtain schema for table %s ", tableId));
+ }
+
+ return schemasByTableId.get(tableId);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2TypeUtils.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2TypeUtils.java
new file mode 100644
index 0000000000..af6d41d5ab
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2TypeUtils.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils;
+
+import org.apache.seatunnel.api.table.converter.BasicTypeDefine;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2.DB2TypeConverter;
+
+import io.debezium.relational.Column;
+
+/** Utilities for converting from Db2 types to SeaTunnel types. */
+public class Db2TypeUtils {
+
+ public static SeaTunnelDataType<?> convertFromColumn(Column column) {
+ BasicTypeDefine typeDefine =
+ BasicTypeDefine.builder()
+ .name(column.name())
+ .columnType(column.typeName())
+ .dataType(column.typeName())
+ .length((long) column.length())
+ .precision((long) column.length())
+ .scale(column.scale().orElse(0))
+ .build();
+ org.apache.seatunnel.api.table.catalog.Column seaTunnelColumn =
+ DB2TypeConverter.INSTANCE.convert(typeDefine);
+ return seaTunnelColumn.getDataType();
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2Utils.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2Utils.java
new file mode 100644
index 0000000000..9f605f864e
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2Utils.java
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.common.utils.SeaTunnelException;
+import org.apache.seatunnel.connectors.cdc.base.source.offset.Offset;
+import org.apache.seatunnel.connectors.cdc.base.utils.SourceRecordUtils;
+import org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset.LsnOffset;
+
+import org.apache.kafka.connect.source.SourceRecord;
+
+import io.debezium.connector.db2.Db2Connection;
+import io.debezium.connector.db2.Db2ConnectorConfig;
+import io.debezium.connector.db2.Db2DatabaseSchema;
+import io.debezium.connector.db2.Db2TopicSelector;
+import io.debezium.connector.db2.Lsn;
+import io.debezium.jdbc.JdbcConnection;
+import io.debezium.relational.Column;
+import io.debezium.relational.Table;
+import io.debezium.relational.TableId;
+import io.debezium.schema.TopicSelector;
+import io.debezium.util.SchemaNameAdjuster;
+import lombok.extern.slf4j.Slf4j;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/** The utils for Db2 data source. */
+@Slf4j
+public class Db2Utils {
+
+ public Db2Utils() {}
+
+ public static Object[] queryMinMax(JdbcConnection jdbc, TableId tableId, String columnName)
+ throws SQLException {
+ final String minMaxQuery =
+ String.format(
+ "SELECT MIN(%s), MAX(%s) FROM %s",
+ quote(columnName), quote(columnName), quote(tableId));
+ return jdbc.queryAndMap(
+ minMaxQuery,
+ rs -> {
+ if (!rs.next()) {
+ // this should never happen
+ throw new SQLException(
+ String.format(
+ "No result returned after running query [%s]",
+ minMaxQuery));
+ }
+ return SourceRecordUtils.rowToArray(rs, 2);
+ });
+ }
+
+ public static long queryApproximateRowCnt(JdbcConnection jdbc, TableId tableId)
+ throws SQLException {
+ // SYSCAT.TABLES.CARD is the optimizer's cardinality estimate. It avoids COUNT(*) during
+ // split planning and falls back to 0 when RUNSTATS has not populated the estimate yet.
+ final String rowCountQuery =
+ String.format(
+ "SELECT COALESCE(MAX(CARD), 0) FROM SYSCAT.TABLES "
+ + "WHERE TABSCHEMA = '%s' AND TABNAME = '%s'",
+ tableId.schema(), tableId.table());
+ return jdbc.queryAndMap(
+ rowCountQuery,
+ rs -> {
+ if (!rs.next()) {
+ throw new SQLException(
+ String.format(
+ "No result returned after running query [%s]",
+ rowCountQuery));
+ }
+ return rs.getLong(1);
+ });
+ }
+
+ public static Object queryMin(
+ JdbcConnection jdbc, TableId tableId, String columnName, Object excludedLowerBound)
+ throws SQLException {
+ final String minQuery =
+ String.format(
+ "SELECT MIN(%s) FROM %s WHERE %s > ?",
+ quote(columnName), quote(tableId), quote(columnName));
+ return jdbc.prepareQueryAndMap(
+ minQuery,
+ ps -> ps.setObject(1, excludedLowerBound),
+ rs -> {
+ if (!rs.next()) {
+ // this should never happen
+ throw new SQLException(
+ String.format(
+ "No result returned after running query [%s]", minQuery));
+ }
+ return rs.getObject(1);
+ });
+ }
+
+ public static Object[] sampleDataFromColumn(
+ JdbcConnection jdbc, TableId tableId, String columnName, int inverseSamplingRate)
+ throws SQLException {
+ final String minQuery =
+ String.format(
+ "SELECT %s FROM %s WHERE MOD((%s - (SELECT MIN(%s) FROM %s)), %s) = 0 ORDER BY %s",
+ quote(columnName),
+ quote(tableId),
+ quote(columnName),
+ quote(columnName),
+ quote(tableId),
+ inverseSamplingRate,
+ quote(columnName));
+ return jdbc.queryAndMap(
+ minQuery,
+ resultSet -> {
+ List<Object> results = new ArrayList<>();
+ while (resultSet.next()) {
+ results.add(resultSet.getObject(1));
+ }
+ return results.toArray();
+ });
+ }
+
+ public static Object[] skipReadAndSortSampleData(
+ JdbcConnection jdbc, TableId tableId, String columnName, int inverseSamplingRate)
+ throws Exception {
+ final String sampleQuery =
+ String.format("SELECT %s FROM %s", quote(columnName), quote(tableId));
+
+ Statement stmt = null;
+ ResultSet rs = null;
+
+ List<Object> results = new ArrayList<>();
+ try {
+ stmt =
+ jdbc.connection()
+ .createStatement(
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+
+ stmt.setFetchSize(1024);
+ rs = stmt.executeQuery(sampleQuery);
+
+ int count = 0;
+ while (rs.next()) {
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Processing row index: {}", count);
+ }
+ if (count % inverseSamplingRate == 0) {
+ results.add(rs.getObject(1));
+ }
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException("Thread interrupted");
+ }
+ }
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ log.error("Failed to close ResultSet", e);
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ log.error("Failed to close Statement", e);
+ }
+ }
+ }
+ Object[] resultsArray = results.toArray();
+ Arrays.sort(resultsArray);
+ return resultsArray;
+ }
+
+ /**
+ * Returns the next LSN to be read from the database. This is the LSN of the last record that
+ * was read from the database.
+ */
+ public static Object queryNextChunkMax(
+ JdbcConnection jdbc,
+ TableId tableId,
+ String splitColumnName,
+ int chunkSize,
+ Object includedLowerBound)
+ throws SQLException {
+ String quotedColumn = quote(splitColumnName);
+ String query =
+ String.format(
+ "SELECT MAX(%s) FROM ("
+ + "SELECT %s FROM %s WHERE %s >= ? ORDER BY %s ASC "
+ + "FETCH FIRST %s ROWS ONLY"
+ + ") AS T",
+ quotedColumn,
+ quotedColumn,
+ quote(tableId),
+ quotedColumn,
+ quotedColumn,
+ chunkSize);
+ return jdbc.prepareQueryAndMap(
+ query,
+ ps -> ps.setObject(1, includedLowerBound),
+ rs -> {
+ if (!rs.next()) {
+ // this should never happen
+ throw new SQLException(
+ String.format(
+ "No result returned after running query [%s]", query));
+ }
+ return rs.getObject(1);
+ });
+ }
+
+ public static SeaTunnelRowType getSplitType(Table table) {
+ List<Column> primaryKeys = table.primaryKeyColumns();
+ if (primaryKeys.isEmpty()) {
+ throw new SeaTunnelException(
+ String.format(
+ "Incremental snapshot for tables requires primary key,"
+ + " but table %s doesn't have primary key.",
+ table.id()));
+ }
+
+ // use first field in primary key as the split key
+ return getSplitType(primaryKeys.get(0));
+ }
+
+ public static SeaTunnelRowType getSplitType(Column splitColumn) {
+ return new SeaTunnelRowType(
+ new String[] {splitColumn.name()},
+ new SeaTunnelDataType<?>[] {Db2TypeUtils.convertFromColumn(splitColumn)});
+ }
+
+ public static Offset getLsn(SourceRecord record) {
+ return getLsnPosition(record.sourceOffset());
+ }
+
+ public static LsnOffset getLsnPosition(Map<String, ?> offset) {
+ Map<String, String> offsetStrMap = new HashMap<>();
+ for (Map.Entry<String, ?> entry : offset.entrySet()) {
+ offsetStrMap.put(
+ entry.getKey(), entry.getValue() == null ? null : entry.getValue().toString());
+ }
+ return LsnOffset.valueOf(offsetStrMap);
+ }
+
+ /** Fetch current largest log sequence number (LSN) of the database. */
+ public static LsnOffset currentLsn(Db2Connection connection) {
+ try {
+ Lsn commitLsn = connection.getMaxLsn();
+ return LsnOffset.valueOf(commitLsn.toString());
+ } catch (SQLException e) {
+ throw new SeaTunnelException(e.getMessage(), e);
+ }
+ }
+
+ /** Get split scan query for the given table. */
+ public static String buildSplitScanQuery(
+ TableId tableId, SeaTunnelRowType rowType, boolean isFirstSplit, boolean isLastSplit) {
+ return buildSplitQuery(tableId, rowType, isFirstSplit, isLastSplit, -1, true);
+ }
+
+ /** Get table split data PreparedStatement. */
+ public static PreparedStatement readTableSplitDataStatement(
+ JdbcConnection jdbc,
+ String sql,
+ boolean isFirstSplit,
+ boolean isLastSplit,
+ Object[] splitStart,
+ Object[] splitEnd,
+ SeaTunnelRowType splitKeyType,
+ int fetchSize) {
+ try {
+ final PreparedStatement statement = initStatement(jdbc, sql, fetchSize);
+ if (isFirstSplit && isLastSplit) {
+ return statement;
+ }
+ int primaryKeyNum = splitKeyType.getTotalFields();
+ if (isFirstSplit) {
+ for (int i = 0; i < primaryKeyNum; i++) {
+ statement.setObject(i + 1, splitEnd[i]);
+ statement.setObject(i + 1 + primaryKeyNum, splitEnd[i]);
+ }
+ } else if (isLastSplit) {
+ for (int i = 0; i < primaryKeyNum; i++) {
+ statement.setObject(i + 1, splitStart[i]);
+ }
+ } else {
+ for (int i = 0; i < primaryKeyNum; i++) {
+ statement.setObject(i + 1, splitStart[i]);
+ statement.setObject(i + 1 + primaryKeyNum, splitEnd[i]);
+ statement.setObject(i + 1 + 2 * primaryKeyNum, splitEnd[i]);
+ }
+ }
+ return statement;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to build the split data read statement.", e);
+ }
+ }
+
+ public static Db2DatabaseSchema createDb2DatabaseSchema(
+ Db2ConnectorConfig connectorConfig, Db2Connection connection) {
+ TopicSelector<TableId> topicSelector = Db2TopicSelector.defaultSelector(connectorConfig);
+ SchemaNameAdjuster schemaNameAdjuster = SchemaNameAdjuster.create();
+ return new Db2DatabaseSchema(
+ connectorConfig, schemaNameAdjuster, topicSelector, connection);
+ }
+
+ private static String getPrimaryKeyColumnsProjection(SeaTunnelRowType rowType) {
+ StringBuilder sql = new StringBuilder();
+ for (Iterator<String> fieldNamesIt = Arrays.stream(rowType.getFieldNames()).iterator();
+ fieldNamesIt.hasNext(); ) {
+ sql.append(quote(fieldNamesIt.next()));
+ if (fieldNamesIt.hasNext()) {
+ sql.append(" , ");
+ }
+ }
+ return sql.toString();
+ }
+
+ private static String buildSplitQuery(
+ TableId tableId,
+ SeaTunnelRowType rowType,
+ boolean isFirstSplit,
+ boolean isLastSplit,
+ int limitSize,
+ boolean isScanningData) {
+ final String condition;
+
+ if (isFirstSplit && isLastSplit) {
+ condition = null;
+ } else if (isFirstSplit) {
+ final StringBuilder sql = new StringBuilder();
+ addPrimaryKeyColumnsToCondition(rowType, sql, " <= ?");
+ if (isScanningData) {
+ sql.append(" AND NOT (");
+ addPrimaryKeyColumnsToCondition(rowType, sql, " = ?");
+ sql.append(")");
+ }
+ condition = sql.toString();
+ } else if (isLastSplit) {
+ final StringBuilder sql = new StringBuilder();
+ addPrimaryKeyColumnsToCondition(rowType, sql, " >= ?");
+ condition = sql.toString();
+ } else {
+ final StringBuilder sql = new StringBuilder();
+ addPrimaryKeyColumnsToCondition(rowType, sql, " >= ?");
+ if (isScanningData) {
+ sql.append(" AND NOT (");
+ addPrimaryKeyColumnsToCondition(rowType, sql, " = ?");
+ sql.append(")");
+ }
+ sql.append(" AND ");
+ addPrimaryKeyColumnsToCondition(rowType, sql, " <= ?");
+ condition = sql.toString();
+ }
+
+ if (isScanningData) {
+ return buildSelectWithRowLimits(
+ tableId, limitSize, "*", Optional.ofNullable(condition), Optional.empty());
+ } else {
+ final String orderBy =
+ String.join(
+ ", ",
+ Arrays.stream(rowType.getFieldNames())
+ .map(Db2Utils::quote)
+ .toArray(String[]::new));
+ return buildSelectWithBoundaryRowLimits(
+ tableId,
+ limitSize,
+ getPrimaryKeyColumnsProjection(rowType),
+ getMaxPrimaryKeyColumnsProjection(rowType),
+ Optional.ofNullable(condition),
+ orderBy);
+ }
+ }
+
+ private static PreparedStatement initStatement(JdbcConnection jdbc, String sql, int fetchSize)
+ throws SQLException {
+ final Connection connection = jdbc.connection();
+ connection.setAutoCommit(false);
+ final PreparedStatement statement = connection.prepareStatement(sql);
+ statement.setFetchSize(fetchSize);
+ return statement;
+ }
+
+ private static String getMaxPrimaryKeyColumnsProjection(SeaTunnelRowType rowType) {
+ StringBuilder sql = new StringBuilder();
+ for (Iterator<String> fieldNamesIt = Arrays.stream(rowType.getFieldNames()).iterator();
+ fieldNamesIt.hasNext(); ) {
+ sql.append("MAX(").append(quote(fieldNamesIt.next())).append(")");
+ if (fieldNamesIt.hasNext()) {
+ sql.append(" , ");
+ }
+ }
+ return sql.toString();
+ }
+
+ private static String buildSelectWithRowLimits(
+ TableId tableId,
+ int limit,
+ String projection,
+ Optional<String> condition,
+ Optional<String> orderBy) {
+ final StringBuilder sql = new StringBuilder("SELECT ");
+ sql.append(projection).append(" FROM ");
+ sql.append(quoteSchemaAndTable(tableId));
+ if (condition.isPresent()) {
+ sql.append(" WHERE ").append(condition.get());
+ }
+ if (orderBy.isPresent()) {
+ sql.append(" ORDER BY ").append(orderBy.get());
+ }
+ if (limit > 0) {
+ sql.append(" FETCH FIRST ").append(limit).append(" ROWS ONLY");
+ }
+ return sql.toString();
+ }
+
+ private static String quoteSchemaAndTable(TableId tableId) {
+ StringBuilder quoted = new StringBuilder();
+
+ if (tableId.schema() != null && !tableId.schema().isEmpty()) {
+ quoted.append(quote(tableId.schema())).append(".");
+ }
+
+ quoted.append(quote(tableId.table()));
+ return quoted.toString();
+ }
+
+ public static String quote(String dbOrTableName) {
+ return "\"" + dbOrTableName + "\"";
+ }
+
+ public static String quote(TableId tableId) {
+ return quoteSchemaAndTable(tableId);
+ }
+
+ private static void addPrimaryKeyColumnsToCondition(
+ SeaTunnelRowType rowType, StringBuilder sql, String predicate) {
+ for (Iterator<String> fieldNamesIt = Arrays.stream(rowType.getFieldNames()).iterator();
+ fieldNamesIt.hasNext(); ) {
+ sql.append(quote(fieldNamesIt.next())).append(predicate);
+ if (fieldNamesIt.hasNext()) {
+ sql.append(" AND ");
+ }
+ }
+ }
+
+ private static String buildSelectWithBoundaryRowLimits(
+ TableId tableId,
+ int limit,
+ String projection,
+ String maxColumnProjection,
+ Optional<String> condition,
+ String orderBy) {
+ final StringBuilder sql = new StringBuilder("SELECT ");
+ sql.append(maxColumnProjection);
+ sql.append(" FROM (");
+ sql.append("SELECT ");
+ sql.append(projection);
+ sql.append(" FROM ");
+ sql.append(quoteSchemaAndTable(tableId));
+ if (condition.isPresent()) {
+ sql.append(" WHERE ").append(condition.get());
+ }
+ sql.append(" ORDER BY ").append(orderBy);
+ if (limit > 0) {
+ sql.append(" FETCH FIRST ").append(limit).append(" ROWS ONLY");
+ }
+ sql.append(") T");
+ return sql.toString();
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtils.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtils.java
new file mode 100644
index 0000000000..ee4a7e8bd8
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtils.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils;
+
+import org.apache.seatunnel.common.utils.SeaTunnelException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.debezium.jdbc.JdbcConnection;
+import io.debezium.relational.RelationalTableFilters;
+import io.debezium.relational.TableId;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/** Utilities to discovery matched tables. */
+public class TableDiscoveryUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(TableDiscoveryUtils.class);
+
+ @SuppressWarnings("MagicNumber")
+ public static List<TableId> listTables(JdbcConnection jdbc, RelationalTableFilters tableFilters)
+ throws SQLException {
+ LOG.info("Read list of Db2 CDC enabled tables");
+ List<TableId> capturedTableIds =
+ ((io.debezium.connector.db2.Db2Connection) jdbc)
+ .listOfChangeTables().stream()
+ .map(io.debezium.connector.db2.Db2ChangeTable::getSourceTableId)
+ .filter(tableFilters.dataCollectionFilter()::isIncluded)
+ .collect(Collectors.toList());
+ capturedTableIds.forEach(
+ tableId -> LOG.info("\t including '{}' for further processing", tableId));
+ return capturedTableIds;
+ }
+
+ /**
+ * Validates that every explicitly configured DB2 table is present in the capture-enabled table
+ * set returned by Debezium discovery.
+ *
+ * <p>Db2 CDC discovery starts from ASN change-table metadata, so a requested table that is not
+ * capture-enabled would otherwise disappear before split assignment. This guard keeps explicit
+ * `table-names` semantics fail-fast instead of silently continuing with only the captured
+ * subset.
+ */
+ public static void validateExplicitCaptureTables(
+ List<String> configuredTables, List<TableId> capturedTableIds) {
+ if (configuredTables == null || configuredTables.isEmpty()) {
+ return;
+ }
+
+ Set<TableId> normalizedCapturedTables =
+ capturedTableIds.stream()
+ .map(TableDiscoveryUtils::normalizeCapturedTableId)
+ .collect(Collectors.toSet());
+ List<String> missingTables = new ArrayList<>();
+ for (String configuredTable : configuredTables) {
+ if (!normalizedCapturedTables.contains(toConfiguredDb2TableId(configuredTable))) {
+ missingTables.add(configuredTable);
+ }
+ }
+ if (!missingTables.isEmpty()) {
+ throw new SeaTunnelException(
+ "The following configured DB2 tables are not enabled for capture: "
+ + String.join(", ", missingTables));
+ }
+ }
+
+ /**
+ * Db2 Debezium metadata always uses an empty catalog because one connector instance captures a
+ * single configured database. Explicit SeaTunnel table names still carry the database segment,
+ * so startup validation needs to drop that catalog part before comparing with capture tables.
+ */
+ static TableId toConfiguredDb2TableId(String configuredTable) {
+ int firstDot = configuredTable.indexOf('.');
+ int secondDot = configuredTable.indexOf('.', firstDot + 1);
+ if (firstDot < 0 || secondDot < 0 || secondDot == configuredTable.length() - 1) {
+ throw new SeaTunnelException(
+ "DB2 CDC table-names must use database.schema.table format, but found: "
+ + configuredTable);
+ }
+ return new TableId(
+ "",
+ configuredTable.substring(firstDot + 1, secondDot),
+ configuredTable.substring(secondDot + 1));
+ }
+
+ /**
+ * Debezium change-table discovery emits empty-catalog {@link TableId}s for Db2, so startup
+ * validation normalizes both sides to the same shape before comparing them.
+ */
+ static TableId normalizeCapturedTableId(TableId tableId) {
+ return new TableId("", tableId.schema(), tableId.table());
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactoryTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactoryTest.java
new file mode 100644
index 0000000000..853215c542
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/Db2IncrementalSourceFactoryTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class Db2IncrementalSourceFactoryTest {
+ @Test
+ public void testOptionRule() {
+ Assertions.assertNotNull((new Db2IncrementalSourceFactory()).optionRule());
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/offset/LsnOffsetTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/offset/LsnOffsetTest.java
new file mode 100644
index 0000000000..679859635d
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/source/offset/LsnOffsetTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.source.offset;
+
+import org.apache.seatunnel.connectors.cdc.base.source.offset.Offset;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import io.debezium.connector.db2.Lsn;
+
+class LsnOffsetTest {
+
+ @Test
+ void testInitialOffsetRepresentsNoLsn() {
+ LsnOffset initial = LsnOffset.INITIAL_OFFSET;
+
+ // no LSN keys should be present in the offset map
+ Assertions.assertTrue(initial.getOffset().isEmpty());
+
+ // commit LSN resolved from the empty map should be Debezium's NULL LSN
+ Lsn commitLsn = initial.getCommitLsn();
+ Assertions.assertFalse(commitLsn.isAvailable());
+ }
+
+ @Test
+ void testCommitOnlyWatermarkCanStopChangeLsnOffset() {
+ String commitLsn = Lsn.valueOf(new byte[] {1}).toString();
+ String laterChangeLsn = Lsn.valueOf(new byte[] {2}).toString();
+
+ Offset stopOffset = LsnOffset.valueOf(commitLsn);
+ Offset currentOffset = LsnOffset.valueOf(commitLsn, laterChangeLsn);
+
+ Assertions.assertTrue(currentOffset.isAtOrAfter(stopOffset));
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2UtilsTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2UtilsTest.java
new file mode 100644
index 0000000000..6b50183c59
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/Db2UtilsTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils;
+
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import io.debezium.relational.TableId;
+
+public class Db2UtilsTest {
+ @Test
+ public void testSplitScanQuery() {
+ String splitScanSQL =
+ Db2Utils.buildSplitScanQuery(
+ TableId.parse("db1.schema1.table1"),
+ new SeaTunnelRowType(
+ new String[] {"id"}, new SeaTunnelDataType[] {BasicType.LONG_TYPE}),
+ false,
+ false);
+ Assertions.assertEquals(
+ "SELECT * FROM \"schema1\".\"table1\" WHERE \"id\" >= ? AND NOT (\"id\" = ?) AND \"id\" <= ?",
+ splitScanSQL);
+
+ splitScanSQL =
+ Db2Utils.buildSplitScanQuery(
+ TableId.parse("db1.schema1.table1"),
+ new SeaTunnelRowType(
+ new String[] {"id"}, new SeaTunnelDataType[] {BasicType.LONG_TYPE}),
+ true,
+ true);
+ Assertions.assertEquals("SELECT * FROM \"schema1\".\"table1\"", splitScanSQL);
+
+ splitScanSQL =
+ Db2Utils.buildSplitScanQuery(
+ TableId.parse("db1.schema1.table1"),
+ new SeaTunnelRowType(
+ new String[] {"id"}, new SeaTunnelDataType[] {BasicType.LONG_TYPE}),
+ true,
+ false);
+ Assertions.assertEquals(
+ "SELECT * FROM \"schema1\".\"table1\" WHERE \"id\" <= ? AND NOT (\"id\" = ?)",
+ splitScanSQL);
+
+ splitScanSQL =
+ Db2Utils.buildSplitScanQuery(
+ TableId.parse("db1.schema1.table1"),
+ new SeaTunnelRowType(
+ new String[] {"id"}, new SeaTunnelDataType[] {BasicType.LONG_TYPE}),
+ false,
+ true);
+ Assertions.assertEquals(
+ "SELECT * FROM \"schema1\".\"table1\" WHERE \"id\" >= ?", splitScanSQL);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtilsTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtilsTest.java
new file mode 100644
index 0000000000..c41634bac6
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-db2/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/db2/utils/TableDiscoveryUtilsTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.cdc.db2.utils;
+
+import org.apache.seatunnel.common.utils.SeaTunnelException;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import io.debezium.relational.TableId;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+/**
+ * Covers the explicit table validation that keeps DB2 CDC startup fail-fast for non-captured
+ * tables.
+ */
+class TableDiscoveryUtilsTest {
+
+ /**
+ * Explicitly configured tables must not disappear silently when Debezium only reports the
+ * capture-enabled subset.
+ */
+ @Test
+ void shouldFailWhenConfiguredTableIsMissingFromCaptureSet() {
+ SeaTunnelException exception =
+ Assertions.assertThrows(
+ SeaTunnelException.class,
+ () ->
+ TableDiscoveryUtils.validateExplicitCaptureTables(
+ Arrays.asList(
+ "testdb.DB2INST1.CUSTOMERS",
+ "testdb.DB2INST1.ORDERS"),
+ Collections.singletonList(
+ new TableId("", "DB2INST1", "CUSTOMERS"))));
+
+ Assertions.assertTrue(exception.getMessage().contains("testdb.DB2INST1.ORDERS"));
+ }
+
+ /**
+ * Startup validation should ignore the catalog/database segment because Debezium Db2 capture
+ * metadata uses empty-catalog table ids for the captured tables.
+ */
+ @Test
+ void shouldMatchConfiguredTablesAfterDroppingDatabaseSegment() {
+ Assertions.assertDoesNotThrow(
+ () ->
+ TableDiscoveryUtils.validateExplicitCaptureTables(
+ Collections.singletonList("testdb.DB2INST1.CUSTOMERS"),
+ Collections.singletonList(
+ new TableId("", "DB2INST1", "CUSTOMERS"))));
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-cdc/pom.xml b/seatunnel-connectors-v2/connector-cdc/pom.xml
index 652201859e..1e90099893 100644
--- a/seatunnel-connectors-v2/connector-cdc/pom.xml
+++ b/seatunnel-connectors-v2/connector-cdc/pom.xml
@@ -31,6 +31,7 @@
<modules>
<module>connector-cdc-base</module>
+ <module>connector-cdc-db2</module>
<module>connector-cdc-mysql</module>
<module>connector-cdc-sqlserver</module>
<module>connector-cdc-mongodb</module>
diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/db2/DB2Catalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/db2/DB2Catalog.java
new file mode 100644
index 0000000000..30d056646d
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/db2/DB2Catalog.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.db2;
+
+import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils;
+
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.Column;
+import org.apache.seatunnel.api.table.catalog.ConstraintKey;
+import org.apache.seatunnel.api.table.catalog.PrimaryKey;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.catalog.exception.CatalogException;
+import org.apache.seatunnel.api.table.converter.BasicTypeDefine;
+import org.apache.seatunnel.common.utils.JdbcUrlUtil;
+import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.AbstractJdbcCatalog;
+import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.utils.CatalogUtils;
+import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2.DB2TypeConverter;
+import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.db2.DB2TypeMapper;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class DB2Catalog extends AbstractJdbcCatalog {
+
+ private static final String SELECT_COLUMNS_SQL_TEMPLATE =
+ "SELECT COLNAME AS COLUMN_NAME,\n"
+ + " TYPENAME AS DATA_TYPE,\n"
+ + " LENGTH,\n"
+ + " SCALE,\n"
+ + " NULLS,\n"
+ + " \"DEFAULT\" AS DEFAULT_VALUE,\n"
+ + " REMARKS AS COMMENT,\n"
+ + " COLNO\n"
+ + "FROM SYSCAT.COLUMNS\n"
+ + "WHERE TABSCHEMA = '%s' AND TABNAME = '%s'\n"
+ + "ORDER BY COLNO";
+
+ public DB2Catalog(
+ String catalogName,
+ String username,
+ String pwd,
+ JdbcUrlUtil.UrlInfo urlInfo,
+ String defaultSchema,
+ String driverClass) {
+ super(catalogName, username, pwd, urlInfo, defaultSchema, driverClass);
+ }
+
+ @Override
+ protected String getDatabaseWithConditionSql(String databaseName) {
+ return String.format(
+ "SELECT CURRENT SERVER FROM SYSIBM.SYSDUMMY1 WHERE UPPER(CURRENT SERVER) = UPPER('%s')",
+ databaseName);
+ }
+
+ @Override
+ protected String getTableWithConditionSql(TablePath tablePath) {
+ return String.format(
+ "SELECT TABSCHEMA, TABNAME FROM SYSCAT.TABLES "
+ + "WHERE TYPE = 'T' AND TABSCHEMA = '%s' AND TABNAME = '%s'",
+ resolveSchema(tablePath), tablePath.getTableName());
+ }
+
+ @Override
+ protected String getListDatabaseSql() {
+ return "SELECT CURRENT SERVER FROM SYSIBM.SYSDUMMY1";
+ }
+
+ @Override
+ protected String getListTableSql(String databaseName) {
+ return "SELECT TABSCHEMA, TABNAME FROM SYSCAT.TABLES "
+ + "WHERE TYPE = 'T' AND TABSCHEMA NOT LIKE 'SYS%' "
+ + "ORDER BY TABSCHEMA, TABNAME";
+ }
+
+ @Override
+ protected String getSelectColumnsSql(TablePath tablePath) {
+ return String.format(
+ SELECT_COLUMNS_SQL_TEMPLATE, resolveSchema(tablePath), tablePath.getTableName());
+ }
+
+ @Override
+ protected Optional<String> getTableComment(DatabaseMetaData metaData, TablePath tablePath)
+ throws SQLException {
+ String sql =
+ String.format(
+ "SELECT REMARKS FROM SYSCAT.TABLES "
+ + "WHERE TYPE = 'T' AND TABSCHEMA = '%s' AND TABNAME = '%s'",
+ resolveSchema(tablePath), tablePath.getTableName());
+ List<String> comments =
+ queryString(
+ getUrlFromDatabaseName(tablePath.getDatabaseName()),
+ sql,
+ rs -> rs.getString("REMARKS"));
+ if (comments.isEmpty()) {
+ return Optional.empty();
+ }
+ return Optional.ofNullable(comments.get(0));
+ }
+
+ @Override
+ protected Column buildColumn(ResultSet resultSet) throws SQLException {
+ String columnName = resultSet.getString("COLUMN_NAME");
+ String dataType = resultSet.getString("DATA_TYPE");
+ long length = resultSet.getLong("LENGTH");
+ int scale = resultSet.getInt("SCALE");
+ boolean nullable = "Y".equalsIgnoreCase(resultSet.getString("NULLS"));
+
+ BasicTypeDefine typeDefine =
+ BasicTypeDefine.builder()
+ .name(columnName)
+ .columnType(dataType)
+ .dataType(dataType)
+ .length(length)
+ .precision(length)
+ .scale(scale)
+ .nullable(nullable)
+ .defaultValue(resultSet.getObject("DEFAULT_VALUE"))
+ .comment(resultSet.getString("COMMENT"))
+ .build();
+ return DB2TypeConverter.INSTANCE.convert(typeDefine);
+ }
+
+ @Override
+ protected String getCreateTableSql(
+ TablePath tablePath, CatalogTable table, boolean createIndex) {
+ PrimaryKey primaryKey = table.getTableSchema().getPrimaryKey();
+ Set<String> primaryKeyColumns =
+ primaryKey == null || primaryKey.getColumnNames() == null
+ ? Collections.emptySet()
+ : new HashSet<>(primaryKey.getColumnNames());
+ String columnSql =
+ table.getTableSchema().getColumns().stream()
+ .filter(Column::isPhysical)
+ .map(column -> buildCreateColumnSql(column, primaryKeyColumns))
+ .collect(Collectors.joining(", "));
+ if (primaryKey != null && primaryKey.getColumnNames() != null) {
+ String primaryKeySql =
+ primaryKey.getColumnNames().stream()
+ .map(this::quoteIdentifier)
+ .collect(Collectors.joining(", "));
+ columnSql = columnSql + ", PRIMARY KEY (" + primaryKeySql + ")";
+ }
+ return String.format("CREATE TABLE %s (%s)", quoteSchemaTable(tablePath), columnSql);
+ }
+
+ @Override
+ protected String getDropTableSql(TablePath tablePath) {
+ return String.format("DROP TABLE %s", quoteSchemaTable(tablePath));
+ }
+
+ @Override
+ protected String getCreateDatabaseSql(String databaseName) {
+ return String.format("CREATE DATABASE %s", quoteIdentifier(databaseName));
+ }
+
+ @Override
+ protected String getDropDatabaseSql(String databaseName) {
+ return String.format("DROP DATABASE %s", quoteIdentifier(databaseName));
+ }
+
+ @Override
+ public CatalogTable getTable(String sqlQuery) throws SQLException {
+ Connection defaultConnection = getConnection(defaultUrl);
+ return CatalogUtils.getCatalogTable(defaultConnection, sqlQuery, new DB2TypeMapper());
+ }
+
+ @Override
+ protected String getTruncateTableSql(TablePath tablePath) throws CatalogException {
+ return String.format("TRUNCATE TABLE %s IMMEDIATE", quoteSchemaTable(tablePath));
+ }
+
+ @Override
+ public String getExistDataSql(TablePath tablePath) {
+ return String.format(
+ "SELECT 1 FROM %s FETCH FIRST 1 ROW ONLY", quoteSchemaTable(tablePath));
+ }
+
+ private String buildCreateColumnSql(Column column, Set<String> primaryKeyColumns) {
+ BasicTypeDefine typeDefine = DB2TypeConverter.INSTANCE.reconvert(column);
+ // DB2 rejects a primary key if any key column is nullable, while JDBC query metadata can
+ // lose the original NOT NULL flag when generate_sink_sql builds the sink schema.
+ boolean nullable = column.isNullable() && !primaryKeyColumns.contains(column.getName());
+ return String.format(
+ "%s %s%s",
+ quoteIdentifier(column.getName()),
+ typeDefine.getColumnType(),
+ nullable ? "" : " NOT NULL");
+ }
+
+ private String quoteSchemaTable(TablePath tablePath) {
+ return quoteIdentifier(resolveSchema(tablePath))
+ + "."
+ + quoteIdentifier(tablePath.getTableName());
+ }
+
+ private String quoteIdentifier(String identifier) {
+ return "\"" + identifier.replace("\"", "\"\"") + "\"";
+ }
+
+ private String resolveSchema(TablePath tablePath) {
+ if (StringUtils.isNotBlank(tablePath.getSchemaName())) {
+ return tablePath.getSchemaName();
+ }
+ Optional<String> schema = defaultSchema.filter(StringUtils::isNotBlank);
+ if (schema.isPresent()) {
+ return schema.get();
+ }
+ return username.toUpperCase(Locale.ROOT);
+ }
+
+ @Override
+ protected Optional<PrimaryKey> getPrimaryKey(DatabaseMetaData metaData, TablePath tablePath)
+ throws SQLException {
+ String sql =
+ String.format(
+ "SELECT k.COLNAME, c.CONSTNAME "
+ + "FROM SYSCAT.KEYCOLUSE k "
+ + "JOIN SYSCAT.TABCONST c "
+ + "ON k.CONSTNAME = c.CONSTNAME "
+ + "AND k.TABSCHEMA = c.TABSCHEMA "
+ + "AND k.TABNAME = c.TABNAME "
+ + "WHERE c.TYPE = 'P' "
+ + "AND k.TABSCHEMA = '%s' "
+ + "AND k.TABNAME = '%s' "
+ + "ORDER BY k.COLSEQ",
+ resolveSchema(tablePath), tablePath.getTableName());
+ String primaryKeyName = null;
+ List<String> primaryKeyColumns = new ArrayList<>();
+ try (PreparedStatement statement =
+ getConnection(getUrlFromDatabaseName(tablePath.getDatabaseName()))
+ .prepareStatement(sql);
+ ResultSet resultSet = statement.executeQuery()) {
+ while (resultSet.next()) {
+ primaryKeyName = resultSet.getString("CONSTNAME");
+ primaryKeyColumns.add(resultSet.getString("COLNAME"));
+ }
+ }
+ if (primaryKeyColumns.isEmpty()) {
+ return Optional.empty();
+ }
+ return Optional.of(PrimaryKey.of(primaryKeyName, primaryKeyColumns));
+ }
+
+ @Override
+ protected List<ConstraintKey> getConstraintKeys(
+ DatabaseMetaData metaData, TablePath tablePath) {
+ return Collections.emptyList();
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/db2/DB2CatalogFactory.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/db2/DB2CatalogFactory.java
new file mode 100644
index 0000000000..0b392d4b85
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/db2/DB2CatalogFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.db2;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.table.catalog.Catalog;
+import org.apache.seatunnel.api.table.factory.CatalogFactory;
+import org.apache.seatunnel.api.table.factory.Factory;
+import org.apache.seatunnel.common.utils.JdbcUrlUtil;
+import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcCommonOptions;
+import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(Factory.class)
+public class DB2CatalogFactory implements CatalogFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return DatabaseIdentifier.DB_2;
+ }
+
+ @Override
+ public Catalog createCatalog(String catalogName, ReadonlyConfig options) {
+ String url = options.get(JdbcCommonOptions.URL);
+ JdbcUrlUtil.UrlInfo urlInfo = JdbcUrlUtil.getUrlInfo(url);
+ return new DB2Catalog(
+ catalogName,
+ options.get(JdbcCommonOptions.USERNAME),
+ options.get(JdbcCommonOptions.PASSWORD),
+ urlInfo,
+ options.get(JdbcCommonOptions.SCHEMA),
+ options.get(JdbcCommonOptions.DRIVER));
+ }
+
+ @Override
+ public OptionRule optionRule() {
+ return JdbcCommonOptions.baseCatalogRule().build();
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java
index 37b5ec66fc..80ec6709e2 100644
--- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java
+++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2Dialect.java
@@ -70,6 +70,12 @@ public class DB2Dialect implements JdbcDialect {
@Override
public String tableIdentifier(String database, String tableName) {
+ // DB2 connections are already bound to a database by the JDBC URL. When the table name
+ // carries a schema, prefixing the configured database would generate an invalid
+ // catalog.schema.table identifier.
+ if (tableName.contains(".")) {
+ return quoteIdentifier(tableName);
+ }
return quoteIdentifier(database) + "." + quoteIdentifier(tableName);
}
@@ -131,11 +137,10 @@ public class DB2Dialect implements JdbcDialect {
// Combine all parts to form the final SQL statement
String mergeStatement =
String.format(
- "MERGE INTO %s.%s AS target USING (VALUES (%s)) AS source (%s) ON %s "
+ "MERGE INTO %s AS target USING (VALUES (%s)) AS source (%s) ON %s "
+ "WHEN MATCHED AND (%s) THEN UPDATE SET %s "
+ "WHEN NOT MATCHED THEN %s",
- quoteIdentifier(database),
- quoteIdentifier(tableName),
+ tableIdentifier(database, tableName),
placeholderList,
fieldList,
onClause,
diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/db2/DB2CatalogTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/db2/DB2CatalogTest.java
new file mode 100644
index 0000000000..85ce56a8f7
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/db2/DB2CatalogTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.db2;
+
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
+import org.apache.seatunnel.api.table.catalog.PrimaryKey;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.common.utils.JdbcUrlUtil;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+
+public class DB2CatalogTest {
+
+ @Test
+ void testCreateTableSqlMarksPrimaryKeyColumnsNotNull() {
+ TablePath tablePath = TablePath.of("E2E", "SINK");
+ TableSchema tableSchema =
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "C_INT", BasicType.INT_TYPE, (Long) null, true, null, null))
+ .column(
+ PhysicalColumn.of(
+ "C_INTEGER",
+ BasicType.INT_TYPE,
+ (Long) null,
+ true,
+ null,
+ null))
+ .primaryKey(PrimaryKey.of("PK_SINK", Arrays.asList("C_INT")))
+ .build();
+ CatalogTable catalogTable =
+ CatalogTable.of(
+ TableIdentifier.of("db2", "E2E", "SINK"),
+ tableSchema,
+ new HashMap<>(),
+ new ArrayList<>(),
+ null);
+ DB2Catalog catalog =
+ new DB2Catalog(
+ "db2",
+ "db2inst1",
+ "123456",
+ JdbcUrlUtil.getUrlInfo("jdbc:db2://127.0.0.1:50000/E2E"),
+ "E2E",
+ null);
+
+ String createTableSql = catalog.getCreateTableSql(tablePath, catalogTable, true);
+
+ Assertions.assertEquals(
+ "CREATE TABLE \"E2E\".\"SINK\" (\"C_INT\" INT NOT NULL, "
+ + "\"C_INTEGER\" INT, PRIMARY KEY (\"C_INT\"))",
+ createTableSql);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectTest.java
index 090a5a448e..253577dafa 100644
--- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectTest.java
+++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/db2/DB2DialectTest.java
@@ -39,14 +39,19 @@ public class DB2DialectTest {
() ->
new AssertionError(
"Expected doUpdateSql String to be present"));
+ // DB2 JDBC URLs select the database, so schema-qualified table names must stay
+ // schema.table in DML statements.
Assertions.assertEquals(
- "MERGE INTO \"seatunnel\".\"test_schema\".\"role\" AS target USING (VALUES (?, ?, ?, ?, ?, ?, ?)) AS source (\"id\", \"type\", \"role_name\", \"description\", \"create_time\", \"update_time\", \"id_2\") ON target.\"id\" = source.\"id\" AND target.\"id_2\" = source.\"id_2\" WHEN MATCHED AND (target.\"id\" <> source.\"id\" OR target.\"type\" <> source.\"type\" OR target.\"role_name\" <> source.\"role_name\" OR target.\"description\" <> source.\"description\" OR target.\"cre [...]
+ "MERGE INTO \"test_schema\".\"role\" AS target USING (VALUES (?, ?, ?, ?, ?, ?, ?)) AS source (\"id\", \"type\", \"role_name\", \"description\", \"create_time\", \"update_time\", \"id_2\") ON target.\"id\" = source.\"id\" AND target.\"id_2\" = source.\"id_2\" WHEN MATCHED AND (target.\"id\" <> source.\"id\" OR target.\"type\" <> source.\"type\" OR target.\"role_name\" <> source.\"role_name\" OR target.\"description\" <> source.\"description\" OR target.\"create_time\" <> [...]
upsertSql);
// test insert sql
String insertSql = dialect.getInsertIntoStatement(database, tableName, fieldNames);
Assertions.assertEquals(
- "INSERT INTO \"seatunnel\".\"test_schema\".\"role\" (\"id\", \"type\", \"role_name\", \"description\", \"create_time\", \"update_time\", \"id_2\") VALUES (:id, :type, :role_name, :description, :create_time, :update_time, :id_2)",
+ "INSERT INTO \"test_schema\".\"role\" (\"id\", \"type\", \"role_name\", \"description\", \"create_time\", \"update_time\", \"id_2\") VALUES (:id, :type, :role_name, :description, :create_time, :update_time, :id_2)",
insertSql);
+
+ Assertions.assertEquals(
+ "\"seatunnel\".\"role\"", dialect.tableIdentifier(database, "role"));
}
}
diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml
index 885871cddb..3ba758a202 100644
--- a/seatunnel-dist/pom.xml
+++ b/seatunnel-dist/pom.xml
@@ -638,6 +638,12 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-cdc-db2</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>connector-cdc-mysql</artifactId>
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/pom.xml
new file mode 100644
index 0000000000..3fea264196
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/pom.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-connector-v2-e2e</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>connector-cdc-db2-e2e</artifactId>
+ <name>SeaTunnel : E2E : Connector V2 : CDC DB2</name>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-cdc-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-cdc-db2</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-transforms-v2</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-starter</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>db2</artifactId>
+ <version>${testcontainer.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.ibm.db2.jcc</groupId>
+ <artifactId>db2jcc</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/db2/Db2CDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/db2/Db2CDCIT.java
new file mode 100644
index 0000000000..d31ca7f5d1
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/db2/Db2CDCIT.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.e2e.connector.cdc.db2;
+
+import org.apache.seatunnel.e2e.common.TestResource;
+import org.apache.seatunnel.e2e.common.TestSuiteBase;
+import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
+import org.apache.seatunnel.e2e.common.container.EngineType;
+import org.apache.seatunnel.e2e.common.container.TestContainer;
+import org.apache.seatunnel.e2e.common.container.seatunnel.SeaTunnelContainer;
+import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
+import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
+import org.apache.seatunnel.e2e.common.util.JdbcUtil;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestTemplate;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.Db2Container;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.lifecycle.Startables;
+import org.testcontainers.utility.DockerLoggerFactory;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Types;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import static org.awaitility.Awaitility.await;
+
+/**
+ * Verifies DB2 CDC snapshot and stream replay against a JDBC sink using a minimal synthetic ASNCDC
+ * capture catalog.
+ */
+@Slf4j
+@DisabledOnContainer(
+ value = {},
+ type = {EngineType.SPARK, EngineType.FLINK},
+ disabledReason =
+ "DB2 CDC E2E runs on the SeaTunnel engine to avoid heavy DB2 and Flink containers competing for memory")
+public class Db2CDCIT extends TestSuiteBase implements TestResource {
+
+ private static final String HOST = "db2-cdc-e2e";
+ private static final int PORT = 50000;
+ private static final String DATABASE = "E2E";
+ private static final String USERNAME = "db2inst1";
+ private static final String PASSWORD = "123456";
+ private static final String SCHEMA = "DB2INST1";
+ private static final String CDC_SCHEMA = "ASNCDC";
+ private static final String SOURCE_TABLE = SCHEMA + ".CUSTOMERS";
+ private static final String SINK_TABLE = SCHEMA + ".CUSTOMERS_SINK";
+ private static final String CAPTURE_TABLE = "DB2INST1_CUSTOMERS";
+ private static final String CHANGE_TABLE = CDC_SCHEMA + "." + CAPTURE_TABLE;
+ private static final String SELECT_CUSTOMERS =
+ "SELECT ID, NAME, DESCRIPTION FROM %s ORDER BY ID";
+ private static final String SEATUNNEL_SERVER_JVM_OPTION =
+ "-Xms256m -Xmx768m -XX:MaxMetaspaceSize=512m";
+ private static final String SEATUNNEL_CLIENT_JVM_OPTION = "-Xms128m -Xmx256m";
+
+ /**
+ * Use the fixed Testcontainers-supported DB2 tag instead of unversioned `latest` so the E2E
+ * does not drift as IBM updates the image.
+ */
+ private static final String DB2_IMAGE = "ibmcom/db2:11.5.0.0a";
+
+ public static final Db2Container DB2_CONTAINER =
+ new Db2Container(DB2_IMAGE)
+ .withDatabaseName(DATABASE)
+ .withUsername(USERNAME)
+ .withPassword(PASSWORD)
+ // IBM's DB2 image is currently published only for amd64, so Apple Silicon
+ // developers need an explicit platform override to run this E2E locally.
+ .withCreateContainerCmdModifier(cmd -> cmd.withPlatform("linux/amd64"))
+ .withNetwork(NETWORK)
+ .withNetworkAliases(HOST)
+ .withExposedPorts(PORT)
+ // Apple Silicon runs the amd64 DB2 image under emulation, which can take
+ // significantly longer than the default DB2Container wait budget.
+ .withStartupTimeout(Duration.ofMinutes(30))
+ .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(DB2_IMAGE)))
+ .acceptLicense();
+
+ private String driverUrl() {
+ return "https://repo1.maven.org/maven2/com/ibm/db2/jcc/db2jcc/db2jcc4/db2jcc-db2jcc4.jar";
+ }
+
+ @TestContainerExtension
+ protected final ContainerExtendedFactory extendedFactory =
+ container -> {
+ Container.ExecResult extraCommands =
+ container.execInContainer(
+ "bash",
+ "-c",
+ "mkdir -p /tmp/seatunnel/plugins/DB2-CDC/lib "
+ + "/tmp/seatunnel/plugins/Jdbc/lib && "
+ + "wget --no-check-certificate -O "
+ + "/tmp/seatunnel/plugins/DB2-CDC/lib/db2jcc.jar "
+ + driverUrl()
+ + " && cp /tmp/seatunnel/plugins/DB2-CDC/lib/db2jcc.jar "
+ + "/tmp/seatunnel/plugins/Jdbc/lib/db2jcc.jar");
+ Assertions.assertEquals(0, extraCommands.getExitCode(), extraCommands.getStderr());
+ };
+
+ @Override
+ @BeforeAll
+ public void startUp() {
+ configureSeaTunnelJvmOptions();
+ log.info("Starting DB2 container...");
+ Startables.deepStart(Stream.of(DB2_CONTAINER)).join();
+ log.info("DB2 container is started.");
+ }
+
+ @Override
+ @AfterAll
+ public void tearDown() {
+ log.info("Stopping DB2 container...");
+ if (DB2_CONTAINER != null) {
+ DB2_CONTAINER.stop();
+ }
+ clearSeaTunnelJvmOptions();
+ log.info("DB2 container is stopped.");
+ }
+
+ @TestTemplate
+ public void testDb2CdcToDb2(TestContainer container) {
+ initializeDb2Tables();
+
+ CompletableFuture<Void> jobFuture =
+ CompletableFuture.runAsync(
+ () -> {
+ try {
+ Container.ExecResult result =
+ container.executeJob("/db2cdc_to_db2.conf");
+ Assertions.assertEquals(
+ 0, result.getExitCode(), result.getStderr());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ await().atMost(2, TimeUnit.MINUTES)
+ .untilAsserted(
+ () -> {
+ assertJobNotFailed(jobFuture);
+ Assertions.assertIterableEquals(
+ querySql(SELECT_CUSTOMERS, SOURCE_TABLE),
+ querySql(SELECT_CUSTOMERS, SINK_TABLE));
+ });
+
+ applyChangeEvents();
+
+ await().atMost(2, TimeUnit.MINUTES)
+ .untilAsserted(
+ () -> {
+ assertJobNotFailed(jobFuture);
+ Assertions.assertIterableEquals(
+ querySql(SELECT_CUSTOMERS, SOURCE_TABLE),
+ querySql(SELECT_CUSTOMERS, SINK_TABLE));
+ });
+ }
+
+ private void initializeDb2Tables() {
+ try (Connection connection = getJdbcConnection();
+ Statement statement = connection.createStatement()) {
+ createSchemaIfNotExists(statement, SCHEMA);
+ createSchemaIfNotExists(statement, CDC_SCHEMA);
+ dropTableIfExists(statement, CHANGE_TABLE);
+ dropTableIfExists(statement, CDC_SCHEMA + ".IBMSNAP_REGISTER");
+ dropTableIfExists(statement, SINK_TABLE);
+ dropTableIfExists(statement, SOURCE_TABLE);
+
+ statement.execute(
+ "CREATE TABLE "
+ + SOURCE_TABLE
+ + " (ID INTEGER NOT NULL, NAME VARCHAR(64) NOT NULL, "
+ + "DESCRIPTION VARCHAR(128), PRIMARY KEY (ID))");
+ statement.execute(
+ "CREATE TABLE "
+ + SINK_TABLE
+ + " (ID INTEGER NOT NULL, NAME VARCHAR(64) NOT NULL, "
+ + "DESCRIPTION VARCHAR(128), PRIMARY KEY (ID))");
+ statement.execute(
+ "CREATE TABLE "
+ + CDC_SCHEMA
+ + ".IBMSNAP_REGISTER (SOURCE_OWNER VARCHAR(128) NOT NULL, "
+ + "SOURCE_TABLE VARCHAR(128) NOT NULL, CD_OWNER VARCHAR(128) NOT NULL, "
+ + "CD_TABLE VARCHAR(128) NOT NULL, "
+ + "CD_NEW_SYNCHPOINT CHAR(16) FOR BIT DATA, "
+ + "CD_OLD_SYNCHPOINT CHAR(16) FOR BIT DATA, "
+ + "SYNCHPOINT CHAR(16) FOR BIT DATA)");
+ // Debezium's ASNCDC.ADDTABLE procedure creates three CDC metadata columns before
+ // the source columns. Adding extra metadata here would shift Debezium's result-set
+ // offset and make row data decode against the wrong schema.
+ statement.execute(
+ "CREATE TABLE "
+ + CHANGE_TABLE
+ + " (IBMSNAP_COMMITSEQ CHAR(16) FOR BIT DATA NOT NULL, "
+ + "IBMSNAP_INTENTSEQ CHAR(16) FOR BIT DATA NOT NULL, "
+ + "IBMSNAP_OPERATION CHAR(1) NOT NULL, "
+ + "ID INTEGER, NAME VARCHAR(64), DESCRIPTION VARCHAR(128))");
+
+ // The DB2 E2E image does not run IBM ASN capture. This minimal ASNCDC catalog
+ // exposes the source table to Debezium while the test appends deterministic change
+ // rows to the capture table below.
+ insertCustomer(connection, SOURCE_TABLE, 1, "Alice", "snapshot row 1");
+ insertCustomer(connection, SOURCE_TABLE, 2, "Bob", "snapshot row 2");
+ insertCaptureRegistration(connection);
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to initialize DB2 CDC E2E tables", e);
+ }
+ }
+
+ private void applyChangeEvents() {
+ try (Connection connection = getJdbcConnection()) {
+ connection.setAutoCommit(false);
+ insertCustomer(connection, SOURCE_TABLE, 3, "Carol", "stream insert row");
+ insertChangeEvent(connection, 1, "I", 3, "Carol", "stream insert row");
+
+ deleteCustomer(connection, SOURCE_TABLE, 1);
+ insertChangeEvent(connection, 2, "D", 1, "Alice", "snapshot row 1");
+ updateCaptureSynchpoint(connection, 2);
+ connection.commit();
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to apply DB2 CDC E2E change events", e);
+ }
+ }
+
+ private void createSchemaIfNotExists(Statement statement, String schema) throws SQLException {
+ try {
+ statement.execute("CREATE SCHEMA " + schema);
+ } catch (SQLException e) {
+ if (!"42710".equals(e.getSQLState())) {
+ throw e;
+ }
+ }
+ }
+
+ private void dropTableIfExists(Statement statement, String table) throws SQLException {
+ try {
+ statement.execute("DROP TABLE " + table);
+ } catch (SQLException e) {
+ if (!"42704".equals(e.getSQLState())) {
+ throw e;
+ }
+ }
+ }
+
+ private void insertCaptureRegistration(Connection connection) throws SQLException {
+ try (PreparedStatement statement =
+ connection.prepareStatement(
+ "INSERT INTO "
+ + CDC_SCHEMA
+ + ".IBMSNAP_REGISTER "
+ + "(SOURCE_OWNER, SOURCE_TABLE, CD_OWNER, CD_TABLE, "
+ + "CD_NEW_SYNCHPOINT, CD_OLD_SYNCHPOINT, SYNCHPOINT) "
+ + "VALUES (?, ?, ?, ?, ?, ?, ?)")) {
+ statement.setString(1, SCHEMA);
+ statement.setString(2, "CUSTOMERS");
+ statement.setString(3, CDC_SCHEMA);
+ statement.setString(4, CAPTURE_TABLE);
+ statement.setBytes(5, lsn(0));
+ // A single active capture instance must not have a stop LSN. Debezium treats a
+ // non-null CD_OLD_SYNCHPOINT as the stop point of a retired capture table.
+ statement.setNull(6, Types.BINARY);
+ statement.setBytes(7, lsn(0));
+ statement.executeUpdate();
+ }
+ }
+
+ private void updateCaptureSynchpoint(Connection connection, int commitLsn) throws SQLException {
+ try (PreparedStatement statement =
+ connection.prepareStatement(
+ "UPDATE "
+ + CDC_SCHEMA
+ + ".IBMSNAP_REGISTER SET SYNCHPOINT = ? "
+ + "WHERE SOURCE_OWNER = ? AND SOURCE_TABLE = ?")) {
+ // Debezium polls the max LSN from IBMSNAP_REGISTER, so advancing SYNCHPOINT makes the
+ // synthetic capture rows visible to the transaction-log reader.
+ statement.setBytes(1, lsn(commitLsn));
+ statement.setString(2, SCHEMA);
+ statement.setString(3, "CUSTOMERS");
+ statement.executeUpdate();
+ }
+ }
+
+ private void insertCustomer(
+ Connection connection, String table, int id, String name, String description)
+ throws SQLException {
+ try (PreparedStatement statement =
+ connection.prepareStatement(
+ "INSERT INTO " + table + " (ID, NAME, DESCRIPTION) VALUES (?, ?, ?)")) {
+ statement.setInt(1, id);
+ statement.setString(2, name);
+ statement.setString(3, description);
+ statement.executeUpdate();
+ }
+ }
+
+ private void deleteCustomer(Connection connection, String table, int id) throws SQLException {
+ try (PreparedStatement statement =
+ connection.prepareStatement("DELETE FROM " + table + " WHERE ID = ?")) {
+ statement.setInt(1, id);
+ statement.executeUpdate();
+ }
+ }
+
+ /**
+ * Inserts rows into the synthetic DB2 change table in the same shape that Debezium's DB2
+ * connector reads from ASNCDC capture tables.
+ */
+ private void insertChangeEvent(
+ Connection connection,
+ int commitLsn,
+ String operation,
+ int id,
+ String name,
+ String description)
+ throws SQLException {
+ try (PreparedStatement statement =
+ connection.prepareStatement(
+ "INSERT INTO "
+ + CHANGE_TABLE
+ + " (IBMSNAP_COMMITSEQ, IBMSNAP_INTENTSEQ, IBMSNAP_OPERATION, "
+ + "ID, NAME, DESCRIPTION) "
+ + "VALUES (?, ?, ?, ?, ?, ?)")) {
+ statement.setBytes(1, lsn(commitLsn));
+ statement.setBytes(2, lsn(commitLsn));
+ statement.setString(3, operation);
+ statement.setInt(4, id);
+ statement.setString(5, name);
+ statement.setString(6, description);
+ statement.executeUpdate();
+ }
+ }
+
+ private Connection getJdbcConnection() throws SQLException {
+ return DriverManager.getConnection(
+ DB2_CONTAINER.getJdbcUrl(),
+ DB2_CONTAINER.getUsername(),
+ DB2_CONTAINER.getPassword());
+ }
+
+ private List<List<Object>> querySql(String sql, String table) {
+ return JdbcUtil.querySql(
+ String.format(sql, table),
+ () -> {
+ try {
+ return this.getJdbcConnection();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ private void assertJobNotFailed(CompletableFuture<Void> jobFuture) {
+ if (jobFuture.isCompletedExceptionally()) {
+ jobFuture.join();
+ }
+ }
+
+ private void configureSeaTunnelJvmOptions() {
+ // DB2 is a memory-heavy container, so this E2E keeps Zeta's server and client JVMs
+ // smaller than the shared defaults to avoid resource competition on constrained runners.
+ System.setProperty(
+ SeaTunnelContainer.SERVER_JVM_OPTION_PROPERTY, SEATUNNEL_SERVER_JVM_OPTION);
+ System.setProperty(
+ SeaTunnelContainer.CLIENT_JVM_OPTION_PROPERTY, SEATUNNEL_CLIENT_JVM_OPTION);
+ }
+
+ private void clearSeaTunnelJvmOptions() {
+ System.clearProperty(SeaTunnelContainer.SERVER_JVM_OPTION_PROPERTY);
+ System.clearProperty(SeaTunnelContainer.CLIENT_JVM_OPTION_PROPERTY);
+ }
+
+ private byte[] lsn(int value) {
+ byte[] lsn = new byte[16];
+ lsn[15] = (byte) value;
+ return lsn;
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2.conf
new file mode 100644
index 0000000000..603577f37f
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-db2-e2e/src/test/resources/db2cdc_to_db2.conf
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "STREAMING"
+ checkpoint.interval = 5000
+}
+
+source {
+ DB2-CDC {
+ plugin_output = "customers"
+ username = "db2inst1"
+ password = "123456"
+ startup.mode = "initial"
+ database-names = ["E2E"]
+ table-names = ["E2E.DB2INST1.CUSTOMERS"]
+ url = "jdbc:db2://db2-cdc-e2e:50000/E2E"
+ snapshot.split.size = 2
+ }
+}
+
+sink {
+ Jdbc {
+ plugin_input = "customers"
+ driver = "com.ibm.db2.jcc.DB2Driver"
+ url = "jdbc:db2://db2-cdc-e2e:50000/E2E"
+ username = "db2inst1"
+ password = "123456"
+ generate_sink_sql = true
+ database = "E2E"
+ schema = "DB2INST1"
+ table = "CUSTOMERS_SINK"
+ batch_size = 1
+ primary_keys = ["ID"]
+ }
+}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml
index fd33321674..176bfb7bcf 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml
@@ -30,6 +30,7 @@
<module>connector-assert-e2e</module>
<module>connector-jdbc-e2e</module>
<module>connector-redis-e2e</module>
+ <module>connector-cdc-db2-e2e</module>
<module>connector-cdc-sqlserver-e2e</module>
<module>connector-clickhouse-e2e</module>
<module>connector-databend-e2e</module>
diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
index a0a1c14b61..648127c273 100644
--- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
+++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
@@ -79,6 +79,11 @@ import static org.apache.seatunnel.e2e.common.util.ContainerUtil.copyAllConnecto
@Slf4j
@AutoService(TestContainer.class)
public class SeaTunnelContainer extends AbstractTestContainer {
+ public static final String SERVER_JVM_OPTION_PROPERTY =
+ "seatunnel.e2e.seatunnel.server.jvm.option";
+ public static final String CLIENT_JVM_OPTION_PROPERTY =
+ "seatunnel.e2e.seatunnel.client.jvm.option";
+
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String REST_STOP_JOB_PATH = "/stop-job";
private static final String REST_CHECKPOINT_OVERVIEW_PATH = "/jobs/checkpoints";
@@ -146,9 +151,15 @@ public class SeaTunnelContainer extends AbstractTestContainer {
}
protected String[] buildStartCommand() {
- return new String[] {
- ContainerUtil.adaptPathForWin(Paths.get(SEATUNNEL_HOME, "bin", SERVER_SHELL).toString())
- };
+ List<String> command = new ArrayList<>();
+ command.add(
+ ContainerUtil.adaptPathForWin(
+ Paths.get(SEATUNNEL_HOME, "bin", SERVER_SHELL).toString()));
+ String serverJvmOption = System.getProperty(SERVER_JVM_OPTION_PROPERTY);
+ if (!isBlank(serverJvmOption)) {
+ command.add("-DJvmOption=" + serverJvmOption);
+ }
+ return command.toArray(new String[0]);
}
protected GenericContainer<?> createSeaTunnelContainerWithFakeSourceAndInMemorySink(
@@ -257,7 +268,15 @@ public class SeaTunnelContainer extends AbstractTestContainer {
@Override
protected List<String> getExtraStartShellCommands() {
- return Collections.emptyList();
+ String clientJvmOption = System.getProperty(CLIENT_JVM_OPTION_PROPERTY);
+ if (isBlank(clientJvmOption)) {
+ return Collections.emptyList();
+ }
+ return Collections.singletonList("-DJvmOption=" + clientJvmOption);
+ }
+
+ private static boolean isBlank(String value) {
+ return value == null || value.trim().isEmpty();
}
@Override