You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ga...@apache.org on 2023/05/15 06:37:46 UTC
[incubator-seatunnel-web] branch add_canvas_job_define updated: add datasource module (#54)
This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch add_canvas_job_define
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel-web.git
The following commit(s) were added to refs/heads/add_canvas_job_define by this push:
new 4398dc88 add datasource module (#54)
4398dc88 is described below
commit 4398dc88c2a2f6ed957ef3a9f400c7a66a616a43
Author: Eric <ga...@gmail.com>
AuthorDate: Mon May 15 14:37:41 2023 +0800
add datasource module (#54)
---
README.md | 2 +-
build.sh | 2 +-
pom.xml | 100 ++-
seatunnel-datasource/pom.xml | 40 ++
.../seatunnel-datasource-client/pom.xml | 71 ++
.../datasource/AbstractDataSourceClient.java | 135 ++++
.../seatunnel/datasource/DataSourceClient.java | 28 +
.../datasource/annotation/ThreadSafe.java | 29 +
.../exception/DataSourceSDKException.java | 37 +
.../datasource/request/PageQueryRequest.java | 42 ++
.../request/VirtualTableCreateRequest.java | 40 ++
.../request/VirtualTableFieldRequest.java | 42 ++
.../request/VirtualTableUpdateRequest.java | 36 +
.../datasource/response/BaseResponse.java | 38 ++
.../response/DataSourceDetailResponse.java | 38 ++
.../datasource/response/PageDataResponse.java | 32 +
.../datasource/response/PageResultResponse.java | 36 +
.../response/VirtualTableDetailResponse.java | 42 ++
.../response/VirtualTableFieldResponse.java | 42 ++
.../datasource/response/VirtualTableResponse.java | 36 +
.../datasource/service/DataSourceService.java | 116 ++++
.../datasource/s3/S3DatasourceChannelTest.java | 55 ++
.../seatunnel/datasource/DataSourceClientTest.java | 48 ++
.../datasource-all/pom.xml | 112 +++
.../datasource-elasticsearch/pom.xml | 62 ++
.../ElasticSearchDataSourceChannel.java | 135 ++++
.../ElasticSearchDataSourceConfig.java | 36 +
.../ElasticSearchDataSourceFactory.java | 48 ++
.../elasticsearch/ElasticSearchOptionRule.java | 109 +++
.../client/ElasticsearchClusterInfo.java | 30 +
.../plugin/elasticsearch/client/EsRestClient.java | 371 ++++++++++
.../elasticsearch/client/ResponseException.java | 45 ++
.../plugin/elasticsearch/client/SSLUtils.java | 165 +++++
.../ElasticSearchDataSourceChannelTest.java | 114 ++++
.../ElasticSearchDataSourceFactoryTest.java | 43 ++
.../datasource-jdbc-clickhouse/pom.xml | 59 ++
.../jdbc/ClickhouseDataSourceConfig.java | 59 ++
.../jdbc/ClickhouseJdbcDataSourceChannel.java | 177 +++++
.../jdbc/ClickhouseJdbcDataSourceFactory.java | 48 ++
.../clickhouse/jdbc/ClickhouseOptionRule.java | 68 ++
.../datasource-jdbc-hive/pom.xml | 35 +
.../plugin/hive/jdbc/HiveJdbcConstants.java | 29 +
.../hive/jdbc/HiveJdbcDataSourceChannel.java | 215 ++++++
.../hive/jdbc/HiveJdbcDataSourceFactory.java | 53 ++
.../plugin/hive/jdbc/HiveJdbcOptionRule.java | 42 ++
.../datasource-jdbc-mysql/pom.xml | 59 ++
.../plugin/mysql/jdbc/MysqlDataSourceConfig.java | 51 ++
.../mysql/jdbc/MysqlJdbcDataSourceChannel.java | 182 +++++
.../mysql/jdbc/MysqlJdbcDataSourceFactory.java | 48 ++
.../plugin/mysql/jdbc/MysqlOptionRule.java | 69 ++
.../datasource-jdbc-oracle/pom.xml | 59 ++
.../oracle/jdbc/OracleDataSourceChannel.java | 174 +++++
.../plugin/oracle/jdbc/OracleDataSourceConfig.java | 53 ++
.../oracle/jdbc/OracleJdbcDataSourceFactory.java | 47 ++
.../plugin/oracle/jdbc/OracleOptionRule.java | 67 ++
.../datasource-jdbc-postgresql/pom.xml | 58 ++
.../jdbc/PostgresqlDataSourceChannel.java | 184 +++++
.../jdbc/PostgresqlDataSourceConfig.java | 62 ++
.../jdbc/PostgresqlDataSourceFactory.java | 46 ++
.../postgresql/jdbc/PostgresqlOptionRule.java | 69 ++
.../datasource-jdbc-redshift/pom.xml | 59 ++
.../redshift/jdbc/RedshiftDataSourceChannel.java | 180 +++++
.../redshift/jdbc/RedshiftDataSourceConfig.java | 62 ++
.../redshift/jdbc/RedshiftDataSourceFactory.java | 48 ++
.../plugin/redshift/jdbc/RedshiftOptionRule.java | 69 ++
.../datasource-jdbc-sqlserver/pom.xml | 59 ++
.../sqlserver/jdbc/SqlServerDataSourceChannel.java | 175 +++++
.../sqlserver/jdbc/SqlServerDataSourceConfig.java | 60 ++
.../sqlserver/jdbc/SqlServerDataSourceFactory.java | 48 ++
.../plugin/sqlserver/jdbc/SqlServerOptionRule.java | 68 ++
.../datasource-jdbc-starrocks/pom.xml | 59 ++
.../starrocks/jdbc/StarRocksDataSourceConfig.java | 53 ++
.../jdbc/StarRocksJdbcDataSourceChannel.java | 175 +++++
.../jdbc/StarRocksJdbcDataSourceFactory.java | 46 ++
.../plugin/starrocks/jdbc/StarRocksOptionRule.java | 69 ++
.../datasource-jdbc-tidb/pom.xml | 36 +
.../plugin/tidb/jdbc/TidbDataSourceConfig.java | 51 ++
.../tidb/jdbc/TidbJdbcDataSourceChannel.java | 182 +++++
.../tidb/jdbc/TidbJdbcDataSourceFactory.java | 48 ++
.../plugin/tidb/jdbc/TidbOptionRule.java | 69 ++
.../datasource-kafka/pom.xml | 56 ++
.../plugin/kafka/KafkaDataSourceChannel.java | 115 ++++
.../plugin/kafka/KafkaDataSourceFactory.java | 58 ++
.../datasource/plugin/kafka/KafkaOptionRule.java | 67 ++
.../plugin/kafka/KafkaRequestParamsUtils.java | 55 ++
.../plugin/kafka/KafkaDataSourceChannelTest.java | 106 +++
.../plugin/kafka/KafkaRequestParamsUtilsTest.java | 66 ++
.../datasource-mysql-cdc/pom.xml | 57 ++
.../cdc/mysql/MysqlCDCDataSourceChannel.java | 249 +++++++
.../plugin/cdc/mysql/MysqlCDCDataSourceConfig.java | 34 +
.../cdc/mysql/MysqlCDCDataSourceFactory.java | 46 ++
.../plugin/cdc/mysql/MysqlCDCOptionRule.java | 74 ++
.../datasource-plugins-api/pom.xml | 37 +
.../datasource/plugin/api/DataSourceChannel.java | 83 +++
.../datasource/plugin/api/DataSourceFactory.java | 29 +
.../plugin/api/DataSourcePluginException.java | 37 +
.../plugin/api/DataSourcePluginInfo.java | 53 ++
.../plugin/api/DatasourcePluginTypeEnum.java | 53 ++
.../plugin/api/common/ParamtersUtils.java | 40 ++
.../datasource/plugin/api/model/TableField.java | 40 ++
.../datasource/plugin/api/utils/JdbcUtils.java | 43 ++
.../plugin/api/common/ParamtersUtilsTest.java | 49 ++
.../datasource-s3-redshift/pom.xml | 67 ++
.../plugin/redshift/s3/HadoopS3AConfiguration.java | 100 +++
.../redshift/s3/S3RedshiftDataSourceChannel.java | 268 ++++++++
.../redshift/s3/S3RedshiftDataSourceFactory.java | 56 ++
.../plugin/redshift/s3/S3RedshiftOptionRule.java | 191 ++++++
.../datasource-s3/pom.xml | 68 ++
.../plugin/s3/HadoopS3AConfiguration.java | 95 +++
.../datasource/plugin/s3/S3DataSourceFactory.java | 58 ++
.../datasource/plugin/s3/S3DatasourceChannel.java | 89 +++
.../datasource/plugin/s3/S3OptionRule.java | 166 +++++
.../datasource-sqlserver-cdc/pom.xml | 52 ++
.../sqlserver/SqlServerCDCDataSourceChannel.java | 231 +++++++
.../sqlserver/SqlServerCDCDataSourceConfig.java | 34 +
.../sqlserver/SqlServerCDCDataSourceFactory.java | 47 ++
.../cdc/sqlserver/SqlServerCDCOptionRule.java | 73 ++
.../test/TestSqlServerCDCDataSourceChannel.java | 48 ++
.../datasource-starrocks/pom.xml | 57 ++
.../plugin/starrocks/StarRocksCatalog.java | 224 ++++++
.../starrocks/StarRocksDataSourceChannel.java | 147 ++++
.../starrocks/StarRocksDataSourceConfig.java | 34 +
.../starrocks/StarRocksDataSourceFactory.java | 45 ++
.../plugin/starrocks/StarRocksOptionRule.java | 74 ++
.../seatunnel-datasource-plugins/pom.xml | 53 ++
seatunnel-server/pom.xml | 5 +
seatunnel-server/seatunnel-app/pom.xml | 154 +----
.../src/main/assembly/seatunnel-web.xml | 64 --
.../src/main/bin/seatunnel-backend-daemon.sh | 2 +-
.../seatunnel/app/controller/AuthController.java | 2 +-
.../src/main/resources/application.yml | 2 +-
seatunnel-web-dist/pom.xml | 349 +++++++++-
.../release-docs/{binary => }/LICENSE | 17 +-
.../release-docs/{binary => }/NOTICE | 0
.../{binary => }/licenses/LICENSE-Jetty-9.x.txt | 0
.../{binary => }/licenses/LICENSE-aspectj.txt | 0
.../LICENSE-cron-utils.txt} | 24 +-
.../{binary => }/licenses/LICENSE-h2.txt | 0
.../{binary => }/licenses/LICENSE-jakarta.txt | 0
.../release-docs/licenses/LICENSE-javassist.html | 373 ++++++++++
.../licenses/LICENSE-javax-annotation.txt | 0
.../release-docs/licenses/LICENSE-javax.el.txt | 759 +++++++++++++++++++++
.../{binary => }/licenses/LICENSE-jsoup.txt | 0
.../{binary => }/licenses/LICENSE-logback.txt | 0
.../{binary => }/licenses/LICENSE-mapstruct.txt | 0
.../{binary => }/licenses/LICENSE-slf4j.txt | 0
.../licenses/LICENSE-spring-plugin.txt | 0
.../{binary => }/licenses/LICENSE-swagger-core.txt | 0
.../src/main/assembly/seatunnel-web-ci.xml | 57 +-
.../src/main/assembly/seatunnel-web.xml | 48 +-
tools/dependencies/checkLicense.sh | 2 +-
tools/dependencies/known-dependencies.txt | 3 +-
152 files changed, 11316 insertions(+), 255 deletions(-)
diff --git a/README.md b/README.md
index 822472d8..4800fcff 100644
--- a/README.md
+++ b/README.md
@@ -106,7 +106,7 @@ cd incubator-seatunnel-web
sh build.sh code
```
-Then you can find the installer package in dir `incubator-seatunnel-web/seatunnel-server/seatunnel-app/target/apache-seatunnel-web-incubating-${project.version}.tar.gz`.
+Then you can find the installer package in dir `incubator-seatunnel-web/seatunnel-web-dist/target/apache-seatunnel-web-incubating-${project.version}.tar.gz`.
#### 3.2 Install
diff --git a/build.sh b/build.sh
index 6ce3ed7f..9a849703 100644
--- a/build.sh
+++ b/build.sh
@@ -29,7 +29,7 @@ DOCKER_VERSION=1.0.0-snapshot
code() {
/bin/sh $WORKDIR/mvnw clean package -DskipTests
# mv release zip
- mv $WORKDIR/seatunnel-server/seatunnel-app/target/apache-seatunnel-web-incubating-1.0.0-SNAPSHOT.zip $WORKDIR/
+ mv $WORKDIR/seatunnel-web-dist/target/apache-seatunnel-web-incubating-1.0.0-SNAPSHOT.zip $WORKDIR/
}
# build image
diff --git a/pom.xml b/pom.xml
index 6097e7c8..b2a4da10 100644
--- a/pom.xml
+++ b/pom.xml
@@ -73,6 +73,8 @@
<modules>
<module>seatunnel-server</module>
+ <module>seatunnel-datasource</module>
+ <module>seatunnel-web-dist</module>
</modules>
<properties>
@@ -97,6 +99,7 @@
<maven-scm-provider-jgit.version>1.9.5</maven-scm-provider-jgit.version>
<exec-maven-plugin.version>3.1.0</exec-maven-plugin.version>
<checkstyle.fails.on.error>true</checkstyle.fails.on.error>
+ <maven-shade-plugin.version>3.3.0</maven-shade-plugin.version>
<lombok.version>1.18.0</lombok.version>
<skipUT>false</skipUT>
@@ -108,12 +111,20 @@
<h2.version>2.1.214</h2.version>
<junit.version>5.9.0</junit.version>
<commons-collections4.version>4.4</commons-collections4.version>
- <commons-lang3.version>3.4</commons-lang3.version>
+ <commons-lang3.version>3.12.0</commons-lang3.version>
<guava.version>19.0</guava.version>
<checker.qual.version>3.10.0</checker.qual.version>
<log4j-core.version>2.17.1</log4j-core.version>
<awaitility.version>4.2.0</awaitility.version>
<seatunnel-framework.version>2.3.1</seatunnel-framework.version>
+ <redshift.version>2.1.0.9</redshift.version>
+ <hadoop-aws.version>3.1.4</hadoop-aws.version>
+ <aws-java-sdk-bundle.version>1.11.271</aws-java-sdk-bundle.version>
+ <auto-service.version>1.0.1</auto-service.version>
+ <lombok.version>1.18.24</lombok.version>
+ <e2e.dependency.skip>true</e2e.dependency.skip>
+ <maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version>
+ <flatten-maven-plugin.version>1.3.0</flatten-maven-plugin.version>
</properties>
<dependencyManagement>
@@ -237,6 +248,13 @@
<version>${seatunnel-framework.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>${lombok.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
<!-- seatunnel connector test -->
<dependency>
<groupId>org.apache.seatunnel</groupId>
@@ -402,6 +420,53 @@
<version>${maven-assembly-plugin.version}</version>
</plugin>
+ <!-- shade -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven-shade-plugin.version}</version>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <!-- Make sure the transitive dependencies are written to the generated pom under <dependencies> -->
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <artifactSet>
+ <excludes>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>ch.qos.logback:*</exclude>
+ <exclude>log4j:*</exclude>
+ <exclude>org.apache.logging.log4j:*</exclude>
+ <exclude>commons-logging:*</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+
+ <executions>
+ <execution>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <transformers combine.children="append">
+ <!-- The service transformer is needed to merge META-INF/services files -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
<!-- checkstyle (Start) -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -494,7 +559,7 @@
<artifactId>license-maven-plugin</artifactId>
<version>${maven-license-maven-plugin}</version>
<configuration>
- <outputDirectory>${project.basedir}/seatunnel-dist/target/</outputDirectory>
+ <outputDirectory>${project.basedir}/seatunnel-web-dist/target/</outputDirectory>
<thirdPartyFilename>THIRD-PARTY.txt</thirdPartyFilename>
<sortArtifactByName>false</sortArtifactByName>
<useMissingFile>false</useMissingFile>
@@ -505,6 +570,37 @@
<excludedScopes>test,provided</excludedScopes>
</configuration>
</plugin>
+ <!-- make sure that flatten runs after shaded -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>flatten-maven-plugin</artifactId>
+ <version>${flatten-maven-plugin.version}</version>
+ <configuration>
+ <updatePomFile>true</updatePomFile>
+ <flattenMode>resolveCiFriendliesOnly</flattenMode>
+ </configuration>
+ <executions>
+ <execution>
+ <id>flatten</id>
+ <goals>
+ <goal>flatten</goal>
+ </goals>
+ <phase>process-resources</phase>
+ </execution>
+ <execution>
+ <id>flatten.clean</id>
+ <goals>
+ <goal>clean</goal>
+ </goals>
+ <phase>clean</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>${maven-dependency-plugin.version}</version>
+ </plugin>
</plugins>
</pluginManagement>
diff --git a/seatunnel-datasource/pom.xml b/seatunnel-datasource/pom.xml
new file mode 100644
index 00000000..a6fcbc7a
--- /dev/null
+++ b/seatunnel-datasource/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-web</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>seatunnel-datasource</artifactId>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>seatunnel-datasource-client</module>
+ <module>seatunnel-datasource-plugins</module>
+ </modules>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-client/pom.xml b/seatunnel-datasource/seatunnel-datasource-client/pom.xml
new file mode 100644
index 00000000..e697c677
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>seatunnel-datasource-client</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-all</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <skip>${e2e.dependency.skip}</skip>
+ <appendOutput>true</appendOutput>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java
new file mode 100644
index 00000000..7cc2783e
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.exception.DataSourceSDKException;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.service.DataSourceService;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public abstract class AbstractDataSourceClient implements DataSourceService {
+
+ private Map<String, DataSourcePluginInfo> supportedDataSourceInfo = new HashMap<>();
+
+ private Map<String, Integer> supportedDataSourceIndex = new HashMap<>();
+
+ protected List<DataSourcePluginInfo> supportedDataSources = new ArrayList<>();
+
+ private List<DataSourceChannel> dataSourceChannels = new ArrayList<>();
+
+ protected AbstractDataSourceClient() {
+ AtomicInteger dataSourceIndex = new AtomicInteger();
+ ServiceLoader.load(DataSourceFactory.class)
+ .forEach(
+ seaTunnelDataSourceFactory -> {
+ seaTunnelDataSourceFactory
+ .supportedDataSources()
+ .forEach(
+ dataSourceInfo -> {
+ supportedDataSourceInfo.put(
+ dataSourceInfo.getName().toUpperCase(),
+ dataSourceInfo);
+ supportedDataSourceIndex.put(
+ dataSourceInfo.getName().toUpperCase(),
+ dataSourceIndex.get());
+ supportedDataSources.add(dataSourceInfo);
+ });
+ dataSourceChannels.add(seaTunnelDataSourceFactory.createChannel());
+ dataSourceIndex.getAndIncrement();
+ });
+ if (supportedDataSourceInfo.isEmpty()) {
+ throw new DataSourceSDKException("No supported data source found");
+ }
+ }
+
+ @Override
+ public Boolean checkDataSourceConnectivity(
+ String pluginName, Map<String, String> dataSourceParams) {
+ return getDataSourceChannel(pluginName)
+ .checkDataSourceConnectivity(pluginName, dataSourceParams);
+ }
+
+ @Override
+ public List<DataSourcePluginInfo> listAllDataSources() {
+ return supportedDataSources;
+ }
+
+ protected DataSourceChannel getDataSourceChannel(String pluginName) {
+ checkNotNull(pluginName, "pluginName cannot be null");
+ Integer index = supportedDataSourceIndex.get(pluginName.toUpperCase());
+ if (index == null) {
+ throw new DataSourceSDKException(
+ "The %s plugin is not supported or plugin not exist.", pluginName);
+ }
+ return dataSourceChannels.get(index);
+ }
+
+ @Override
+ public OptionRule queryDataSourceFieldByName(String pluginName) {
+ return getDataSourceChannel(pluginName).getDataSourceOptions(pluginName);
+ }
+
+ @Override
+ public OptionRule queryMetadataFieldByName(String pluginName) {
+ return getDataSourceChannel(pluginName)
+ .getDatasourceMetadataFieldsByDataSourceName(pluginName);
+ }
+
+ @Override
+ public List<String> getTables(
+ String pluginName, String databaseName, Map<String, String> requestParams) {
+ return getDataSourceChannel(pluginName).getTables(pluginName, requestParams, databaseName);
+ }
+
+ @Override
+ public List<String> getDatabases(String pluginName, Map<String, String> requestParams) {
+ return getDataSourceChannel(pluginName).getDatabases(pluginName, requestParams);
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ String pluginName,
+ Map<String, String> requestParams,
+ String databaseName,
+ String tableName) {
+ return getDataSourceChannel(pluginName)
+ .getTableFields(pluginName, requestParams, databaseName, tableName);
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ String pluginName,
+ Map<String, String> requestParams,
+ String databaseName,
+ List<String> tableNames) {
+ return getDataSourceChannel(pluginName)
+ .getTableFields(pluginName, requestParams, databaseName, tableNames);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/DataSourceClient.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/DataSourceClient.java
new file mode 100644
index 00000000..4bf46baa
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/DataSourceClient.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource;
+
+import org.apache.seatunnel.datasource.annotation.ThreadSafe;
+
+@ThreadSafe
+public class DataSourceClient extends AbstractDataSourceClient {
+
+ public DataSourceClient() {
+ super();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/annotation/ThreadSafe.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/annotation/ThreadSafe.java
new file mode 100644
index 00000000..55794b81
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/annotation/ThreadSafe.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Documented
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.CLASS)
+public @interface ThreadSafe {}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/exception/DataSourceSDKException.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/exception/DataSourceSDKException.java
new file mode 100644
index 00000000..3f90eabc
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/exception/DataSourceSDKException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.exception;
+
+public class DataSourceSDKException extends RuntimeException {
+
+ public DataSourceSDKException(String message) {
+ super(message);
+ }
+
+ public DataSourceSDKException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public DataSourceSDKException(Throwable cause) {
+ super(cause);
+ }
+
+ public DataSourceSDKException(String message, Object... args) {
+ super(String.format(message, args));
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/PageQueryRequest.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/PageQueryRequest.java
new file mode 100644
index 00000000..8b09bc35
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/PageQueryRequest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.request;
+
+import lombok.Data;
+import lombok.experimental.SuperBuilder;
+
+import java.util.Map;
+
+@Data
+@SuperBuilder
+public class PageQueryRequest {
+
+ private Integer pageNum;
+
+ private Integer pageSize;
+
+ private String orderBy;
+
+ private String order;
+
+ private String search;
+
+ Map<String, Object> fuzzySearchParamMap;
+
+ Map<String, Object> exactSearchParamMap;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/VirtualTableCreateRequest.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/VirtualTableCreateRequest.java
new file mode 100644
index 00000000..3d7c4220
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/VirtualTableCreateRequest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.request;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+public class VirtualTableCreateRequest {
+
+ private Long id;
+
+ private Long dataSourceId;
+
+ private String description;
+
+ private String databaseName;
+
+ private String tableName;
+
+ private List<VirtualTableFieldRequest> fields;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/VirtualTableFieldRequest.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/VirtualTableFieldRequest.java
new file mode 100644
index 00000000..f2acb83c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/VirtualTableFieldRequest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.request;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.Map;
+
+@Data
+@Builder
+public class VirtualTableFieldRequest {
+
+ private String type;
+
+ private String name;
+
+ private String comment;
+
+ private Boolean primaryKey;
+
+ private String defaultValue;
+
+ private Boolean nullable;
+
+ private Map<String, String> properties;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/VirtualTableUpdateRequest.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/VirtualTableUpdateRequest.java
new file mode 100644
index 00000000..0939687c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/request/VirtualTableUpdateRequest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.request;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+public class VirtualTableUpdateRequest {
+
+ private String description;
+
+ private String databaseName;
+
+ private String tableName;
+
+ private List<VirtualTableFieldRequest> fields;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/BaseResponse.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/BaseResponse.java
new file mode 100644
index 00000000..831a2a64
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/BaseResponse.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.response;
+
+import lombok.Data;
+import lombok.experimental.SuperBuilder;
+
+import java.util.Date;
+
+@Data
+@SuperBuilder
+public class BaseResponse {
+
+ /** create time */
+ private Date createTime;
+
+ /** update time */
+ private Date updateTime;
+
+ private String createUserId;
+
+ private String updateUserId;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/DataSourceDetailResponse.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/DataSourceDetailResponse.java
new file mode 100644
index 00000000..f489ad45
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/DataSourceDetailResponse.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.response;
+
+import lombok.Data;
+import lombok.experimental.SuperBuilder;
+
+import java.util.Map;
+
+@Data
+@SuperBuilder
+public class DataSourceDetailResponse extends BaseResponse {
+
+ private Long id;
+
+ private String dataSourceName;
+
+ private String pluginName;
+
+ private String comment;
+
+ private Map<String, String> params;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/PageDataResponse.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/PageDataResponse.java
new file mode 100644
index 00000000..24e801cc
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/PageDataResponse.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.response;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+public class PageDataResponse<T> {
+ private Integer pageNum;
+ private Integer pageSize;
+ private Integer total;
+ private List<T> datas;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/PageResultResponse.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/PageResultResponse.java
new file mode 100644
index 00000000..10bab20c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/PageResultResponse.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.response;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class PageResultResponse<T> {
+
+ private Integer pageNum;
+
+ private Integer pageSize;
+
+ private Integer total;
+
+ private Integer pages;
+
+ private List<T> dataList;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/VirtualTableDetailResponse.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/VirtualTableDetailResponse.java
new file mode 100644
index 00000000..63ab5d57
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/VirtualTableDetailResponse.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.response;
+
+import lombok.Data;
+import lombok.experimental.SuperBuilder;
+
+import java.util.List;
+
+@Data
+@SuperBuilder
+public class VirtualTableDetailResponse extends BaseResponse {
+
+ private Long id;
+
+ private Long dataSourceId;
+
+ private String description;
+
+ private String type;
+
+ private String databaseName;
+
+ private String tableName;
+
+ private List<VirtualTableFieldResponse> fields;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/VirtualTableFieldResponse.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/VirtualTableFieldResponse.java
new file mode 100644
index 00000000..5e8583ef
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/VirtualTableFieldResponse.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.response;
+
+import lombok.Data;
+import lombok.experimental.SuperBuilder;
+
+import java.util.Map;
+
+@Data
+@SuperBuilder
+public class VirtualTableFieldResponse {
+
+ private String type;
+
+ private String name;
+
+ private String comment;
+
+ private Boolean primaryKey;
+
+ private String defaultValue;
+
+ private Boolean nullable;
+
+ private Map<String, String> properties;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/VirtualTableResponse.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/VirtualTableResponse.java
new file mode 100644
index 00000000..8e061ad4
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/response/VirtualTableResponse.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.response;
+
+import lombok.Data;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+public class VirtualTableResponse extends BaseResponse {
+
+ private Long id;
+
+ private Long dataSourceId;
+
+ private String description;
+
+ private String databaseName;
+
+ private String tableName;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/service/DataSourceService.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/service/DataSourceService.java
new file mode 100644
index 00000000..9ab18325
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/service/DataSourceService.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.service;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import java.util.List;
+import java.util.Map;
+
+public interface DataSourceService {
+
+ /**
+ * get all data source plugins
+ *
+ * @return data source plugins info
+ */
+ List<DataSourcePluginInfo> listAllDataSources();
+
+ /**
+ * get data source plugin fields
+ *
+ * @param pluginName data source name
+ * @return data source plugin fields
+ */
+ OptionRule queryDataSourceFieldByName(String pluginName);
+
+ /**
+ * get data source metadata fields
+ *
+ * @param pluginName data source name
+ * @return data source metadata fields
+ */
+ OptionRule queryMetadataFieldByName(String pluginName);
+
+ /**
+ * check data source params is valid and connectable
+ *
+ * @param parameters data source params eg mysql plugin key: url // jdbc url key: username key:
+ * password other key...
+ * @return true if valid, false if invalid
+ */
+ /**
+ * we can use this method to check data source connectivity
+ *
+ * @param dataSourceParams data source params
+ * @return check result
+ */
+ Boolean checkDataSourceConnectivity(String pluginName, Map<String, String> datasourceParams);
+
+ /**
+ * get data source table names by database name
+ *
+ * @param pluginName plugin name
+ * @param databaseName database name
+ * @param requestParams connection params
+ * @return table names
+ */
+ List<String> getTables(
+ String pluginName, String databaseName, Map<String, String> requestParams);
+
+ /**
+ * get data source database names
+ *
+ * @param pluginName plugin name
+ * @param requestParams connection params
+ * @return database names
+ */
+ List<String> getDatabases(String pluginName, Map<String, String> requestParams);
+
+ /**
+ * get data source table fields
+ *
+ * @param pluginName plugin name
+ * @param requestParams connection params
+ * @param databaseName database name
+ * @param tableName table name
+ * @return table fields
+ */
+ List<TableField> getTableFields(
+ String pluginName,
+ Map<String, String> requestParams,
+ String databaseName,
+ String tableName);
+
+ /**
+ * get data source table fields
+ *
+ * @param pluginName plugin name
+ * @param requestParams connection params
+ * @param databaseName database name
+ * @param tableNames table names
+ * @return table fields
+ */
+ Map<String, List<TableField>> getTableFields(
+ String pluginName,
+ Map<String, String> requestParams,
+ String databaseName,
+ List<String> tableNames);
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/test/java/com/whaleops/datasource/s3/S3DatasourceChannelTest.java b/seatunnel-datasource/seatunnel-datasource-client/src/test/java/com/whaleops/datasource/s3/S3DatasourceChannelTest.java
new file mode 100644
index 00000000..ece18206
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/test/java/com/whaleops/datasource/s3/S3DatasourceChannelTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.whaleops.datasource.s3;
+
+import org.apache.seatunnel.datasource.plugin.s3.S3DatasourceChannel;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+@Disabled
+class S3DatasourceChannelTest {
+ private static S3DatasourceChannel S3_DATASOURCE_CHANNEL = new S3DatasourceChannel();
+
+ @Test
+ void checkDataSourceConnectivity() {
+ Assertions.assertDoesNotThrow(
+ () -> {
+ S3_DATASOURCE_CHANNEL.checkDataSourceConnectivity("S3", createRequestParams());
+ });
+ }
+
+ private Map<String, String> createRequestParams() {
+ Map<String, String> requestParams =
+ new ImmutableMap.Builder<String, String>()
+ .put("bucket", "s3a://poc-kuke")
+ .put("fs.s3a.endpoint", "s3.cn-north-1.amazonaws.com.cn")
+ .put(
+ "fs.s3a.aws.credentials.provider",
+ "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider")
+ .put("access_key", "AKIAYYUV5DMXADXRBGTA")
+ .put("secret_key", "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")
+ .put("hadoop_s3_properties", "")
+ .build();
+ return requestParams;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/test/java/org/apache/seatunnel/datasource/DataSourceClientTest.java b/seatunnel-datasource/seatunnel-datasource-client/src/test/java/org/apache/seatunnel/datasource/DataSourceClientTest.java
new file mode 100644
index 00000000..e2877e88
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/test/java/org/apache/seatunnel/datasource/DataSourceClientTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class DataSourceClientTest {
+ private static final DataSourceClient DATA_SOURCE_CLIENT = new DataSourceClient();
+
+ @Test
+ public void listAllDataSources() {
+ Assertions.assertTrue(
+ DATA_SOURCE_CLIENT.listAllDataSources().stream()
+ .anyMatch(
+ dataSourcePluginInfo ->
+ StringUtils.equalsAnyIgnoreCase(
+ dataSourcePluginInfo.getName(), "jdbc-mysql")));
+ Assertions.assertTrue(
+ DATA_SOURCE_CLIENT.listAllDataSources().stream()
+ .anyMatch(
+ dataSourcePluginInfo ->
+ StringUtils.equalsAnyIgnoreCase(
+ dataSourcePluginInfo.getName(), "kafka")));
+ Assertions.assertTrue(
+ DATA_SOURCE_CLIENT.listAllDataSources().stream()
+ .anyMatch(
+ dataSourcePluginInfo ->
+ StringUtils.equalsAnyIgnoreCase(
+ dataSourcePluginInfo.getName(), "elasticsearch")));
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml
new file mode 100644
index 00000000..64d904df
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-all</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-clickhouse</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-hive</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-mysql</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-oracle</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-postgresql</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-tidb</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-redshift</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-sqlserver</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-starrocks</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-kafka</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-elasticsearch</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-s3-redshift</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-starrocks</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-mysql-cdc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-s3</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-sqlserver-cdc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml
new file mode 100644
index 00000000..7d2272b7
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-elasticsearch</artifactId>
+
+ <properties>
+ <elasticsearch-rest-client.version>7.5.1</elasticsearch-rest-client.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch-rest-client.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>security</artifactId>
+ <version>206</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java
new file mode 100644
index 00000000..17495487
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.elasticsearch.client.EsRestClient;
+
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+
+import lombok.NonNull;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ElasticSearchDataSourceChannel implements DataSourceChannel {
+
+ private static final String DATABASE = "default";
+
+ @Override
+ public boolean canAbleGetSchema() {
+ return true;
+ }
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return ElasticSearchOptionRule.optionRule();
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return ElasticSearchOptionRule.metadataRule();
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ databaseCheck(database);
+ try (EsRestClient client =
+ EsRestClient.createInstance(ConfigFactory.parseMap(requestParams))) {
+ return client.listIndex();
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ return DEFAULT_DATABASES;
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (EsRestClient client =
+ EsRestClient.createInstance(ConfigFactory.parseMap(requestParams))) {
+ client.getClusterInfo();
+ return true;
+ } catch (Throwable e) {
+ throw new DataSourcePluginException(
+ "check ElasticSearch connectivity failed, " + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ databaseCheck(database);
+ try (EsRestClient client =
+ EsRestClient.createInstance(ConfigFactory.parseMap(requestParams))) {
+ Map<String, String> fieldTypeMapping = client.getFieldTypeMapping(table);
+ List<TableField> fields = new ArrayList<>();
+ fieldTypeMapping.forEach(
+ (fieldName, fieldType) ->
+ fields.add(convertToTableField(fieldName, fieldType)));
+ return fields;
+ } catch (Exception ex) {
+ throw new DataSourcePluginException("Get table fields failed", ex);
+ }
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ databaseCheck(database);
+ Map<String, List<TableField>> tableFields = new HashMap<>();
+ tables.forEach(
+ table ->
+ tableFields.put(
+ table, getTableFields(pluginName, requestParams, database, table)));
+ return tableFields;
+ }
+
+ private static void databaseCheck(@NonNull String database) {
+ if (!StringUtils.equalsAnyIgnoreCase(database, DATABASE)) {
+ throw new IllegalArgumentException("database not found: " + database);
+ }
+ }
+
+ private TableField convertToTableField(String fieldName, String fieldType) {
+ TableField tableField = new TableField();
+ tableField.setName(fieldName);
+ tableField.setType(fieldType);
+ tableField.setComment(null);
+ tableField.setNullable(true);
+ tableField.setPrimaryKey(fieldName.equals("_id"));
+ tableField.setDefaultValue(null);
+ return tableField;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceConfig.java
new file mode 100644
index 00000000..97eec189
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceConfig.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+public class ElasticSearchDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "ElasticSearch";
+
+ public static final String PLUGIN_VERSION = "1.0.0";
+
+ public static final DataSourcePluginInfo ELASTICSEARCH_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version(PLUGIN_VERSION)
+ .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode())
+ .build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactory.java
new file mode 100644
index 00000000..f9d01cae
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+@AutoService(DataSourceFactory.class)
+public class ElasticSearchDataSourceFactory implements DataSourceFactory {
+
+ public static final String PLUGIN_NAME = "ElasticSearch";
+
+ @Override
+ public String factoryIdentifier() {
+ return PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(ElasticSearchDataSourceConfig.ELASTICSEARCH_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new ElasticSearchDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchOptionRule.java
new file mode 100644
index 00000000..51097879
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchOptionRule.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+import java.util.List;
+
+public class ElasticSearchOptionRule {
+
+ public static final Option<List<String>> HOSTS =
+ Options.key("hosts")
+ .listType()
+ .noDefaultValue()
+ .withDescription(
+ "Elasticsearch cluster http address, the format is host:port, allowing multiple hosts to be specified. Such as [\"host1:9200\", \"host2:9200\"]");
+
+ public static final Option<String> INDEX =
+ Options.key("index")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Elasticsearch index name, support * fuzzy matching");
+
+ public static final Option<String> USERNAME =
+ Options.key("username")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("x-pack username");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("x-pack password");
+
+ public static final Option<Boolean> TLS_VERIFY_CERTIFICATE =
+ Options.key("tls_verify_certificate")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription("Enable certificates validation for HTTPS endpoints");
+
+ public static final Option<Boolean> TLS_VERIFY_HOSTNAME =
+ Options.key("tls_verify_hostname")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription("Enable hostname validation for HTTPS endpoints");
+
+ public static final Option<String> TLS_KEY_STORE_PATH =
+ Options.key("tls_keystore_path")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The path to the PEM or JKS key store. This file must be readable by the operating system user running SeaTunnel.");
+
+ public static final Option<String> TLS_KEY_STORE_PASSWORD =
+ Options.key("tls_keystore_password")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The key password for the key store specified");
+
+ public static final Option<String> TLS_TRUST_STORE_PATH =
+ Options.key("tls_truststore_path")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The path to PEM or JKS trust store. This file must be readable by the operating system user running SeaTunnel.");
+
+ public static final Option<String> TLS_TRUST_STORE_PASSWORD =
+ Options.key("tls_truststore_password")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The key password for the trust store specified");
+
+ public static OptionRule optionRule() {
+ return OptionRule.builder()
+ .required(HOSTS)
+ .optional(
+ USERNAME,
+ PASSWORD,
+ TLS_VERIFY_CERTIFICATE,
+ TLS_VERIFY_HOSTNAME,
+ TLS_KEY_STORE_PATH,
+ TLS_KEY_STORE_PASSWORD,
+ TLS_TRUST_STORE_PATH,
+ TLS_TRUST_STORE_PASSWORD)
+ .build();
+ }
+
+ public static OptionRule metadataRule() {
+ return OptionRule.builder().required(INDEX).build();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/ElasticsearchClusterInfo.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/ElasticsearchClusterInfo.java
new file mode 100644
index 00000000..f02516ce
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/ElasticsearchClusterInfo.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch.client;
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.ToString;
+
+@Getter
+@Builder
+@ToString
+public class ElasticsearchClusterInfo {
+ private String distribution;
+ private String clusterVersion;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/EsRestClient.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/EsRestClient.java
new file mode 100644
index 00000000..1ef2e705
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/EsRestClient.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch.client;
+
+import org.apache.seatunnel.common.utils.JsonUtils;
+import org.apache.seatunnel.datasource.plugin.elasticsearch.ElasticSearchOptionRule;
+
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpStatus;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.TrustAllStrategy;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.ssl.SSLContexts;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+
+import javax.net.ssl.SSLContext;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class EsRestClient implements AutoCloseable {
+
+ private static final int CONNECTION_REQUEST_TIMEOUT = 10 * 1000;
+
+ private static final int SOCKET_TIMEOUT = 5 * 60 * 1000;
+
+ private final RestClient restClient;
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private EsRestClient(RestClient restClient) {
+ this.restClient = restClient;
+ }
+
+ public static EsRestClient createInstance(Config pluginConfig) {
+ try {
+ List<String> hosts =
+ OBJECT_MAPPER.readValue(
+ pluginConfig.getString(ElasticSearchOptionRule.HOSTS.key()),
+ List.class);
+ Optional<String> username = Optional.empty();
+ Optional<String> password = Optional.empty();
+ if (pluginConfig.hasPath(ElasticSearchOptionRule.USERNAME.key())) {
+ username =
+ Optional.of(pluginConfig.getString(ElasticSearchOptionRule.USERNAME.key()));
+ if (pluginConfig.hasPath(ElasticSearchOptionRule.PASSWORD.key())) {
+ password =
+ Optional.of(
+ pluginConfig.getString(ElasticSearchOptionRule.PASSWORD.key()));
+ }
+ }
+ Optional<String> keystorePath = Optional.empty();
+ Optional<String> keystorePassword = Optional.empty();
+ Optional<String> truststorePath = Optional.empty();
+ Optional<String> truststorePassword = Optional.empty();
+ boolean tlsVerifyCertificate =
+ ElasticSearchOptionRule.TLS_VERIFY_CERTIFICATE.defaultValue();
+ if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_VERIFY_CERTIFICATE.key())) {
+ tlsVerifyCertificate =
+ pluginConfig.getBoolean(
+ ElasticSearchOptionRule.TLS_VERIFY_CERTIFICATE.key());
+ }
+ if (tlsVerifyCertificate) {
+ if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_KEY_STORE_PATH.key())) {
+ keystorePath =
+ Optional.of(
+ pluginConfig.getString(
+ ElasticSearchOptionRule.TLS_KEY_STORE_PATH.key()));
+ }
+ if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_KEY_STORE_PASSWORD.key())) {
+ keystorePassword =
+ Optional.of(
+ pluginConfig.getString(
+ ElasticSearchOptionRule.TLS_KEY_STORE_PASSWORD.key()));
+ }
+ if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_TRUST_STORE_PATH.key())) {
+ truststorePath =
+ Optional.of(
+ pluginConfig.getString(
+ ElasticSearchOptionRule.TLS_TRUST_STORE_PATH.key()));
+ }
+ if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_TRUST_STORE_PASSWORD.key())) {
+ truststorePassword =
+ Optional.of(
+ pluginConfig.getString(
+ ElasticSearchOptionRule.TLS_TRUST_STORE_PASSWORD
+ .key()));
+ }
+ }
+ boolean tlsVerifyHostnames = ElasticSearchOptionRule.TLS_VERIFY_HOSTNAME.defaultValue();
+ if (pluginConfig.hasPath(ElasticSearchOptionRule.TLS_VERIFY_HOSTNAME.key())) {
+ tlsVerifyHostnames =
+ pluginConfig.getBoolean(ElasticSearchOptionRule.TLS_VERIFY_HOSTNAME.key());
+ }
+ return createInstance(
+ hosts,
+ username,
+ password,
+ tlsVerifyCertificate,
+ tlsVerifyHostnames,
+ keystorePath,
+ keystorePassword,
+ truststorePath,
+ truststorePassword);
+ } catch (Exception e) {
+ throw new RuntimeException("Create EsRestClient failed", e);
+ }
+ }
+
+ public static EsRestClient createInstance(
+ List<String> hosts,
+ Optional<String> username,
+ Optional<String> password,
+ boolean tlsVerifyCertificate,
+ boolean tlsVerifyHostnames,
+ Optional<String> keystorePath,
+ Optional<String> keystorePassword,
+ Optional<String> truststorePath,
+ Optional<String> truststorePassword) {
+ RestClientBuilder restClientBuilder =
+ getRestClientBuilder(
+ hosts,
+ username,
+ password,
+ tlsVerifyCertificate,
+ tlsVerifyHostnames,
+ keystorePath,
+ keystorePassword,
+ truststorePath,
+ truststorePassword);
+ return new EsRestClient(restClientBuilder.build());
+ }
+
+ private static RestClientBuilder getRestClientBuilder(
+ List<String> hosts,
+ Optional<String> username,
+ Optional<String> password,
+ boolean tlsVerifyCertificate,
+ boolean tlsVerifyHostnames,
+ Optional<String> keystorePath,
+ Optional<String> keystorePassword,
+ Optional<String> truststorePath,
+ Optional<String> truststorePassword) {
+ HttpHost[] httpHosts = new HttpHost[hosts.size()];
+ for (int i = 0; i < hosts.size(); i++) {
+ httpHosts[i] = HttpHost.create(hosts.get(i));
+ }
+
+ RestClientBuilder restClientBuilder =
+ RestClient.builder(httpHosts)
+ .setRequestConfigCallback(
+ requestConfigBuilder ->
+ requestConfigBuilder
+ .setConnectionRequestTimeout(
+ CONNECTION_REQUEST_TIMEOUT)
+ .setSocketTimeout(SOCKET_TIMEOUT));
+
+ restClientBuilder.setHttpClientConfigCallback(
+ httpClientBuilder -> {
+ if (username.isPresent()) {
+ CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+ credentialsProvider.setCredentials(
+ AuthScope.ANY,
+ new UsernamePasswordCredentials(username.get(), password.get()));
+ httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+ }
+
+ try {
+ if (tlsVerifyCertificate) {
+ Optional<SSLContext> sslContext =
+ SSLUtils.buildSSLContext(
+ keystorePath,
+ keystorePassword,
+ truststorePath,
+ truststorePassword);
+ sslContext.ifPresent(e -> httpClientBuilder.setSSLContext(e));
+ } else {
+ SSLContext sslContext =
+ SSLContexts.custom()
+ .loadTrustMaterial(new TrustAllStrategy())
+ .build();
+ httpClientBuilder.setSSLContext(sslContext);
+ }
+ if (!tlsVerifyHostnames) {
+ httpClientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return httpClientBuilder;
+ });
+ return restClientBuilder;
+ }
+
+ public ElasticsearchClusterInfo getClusterInfo() {
+ Request request = new Request("GET", "/");
+ try {
+ Response response = restClient.performRequest(request);
+ String result = EntityUtils.toString(response.getEntity());
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonNode jsonNode = objectMapper.readTree(result);
+ JsonNode versionNode = jsonNode.get("version");
+ return ElasticsearchClusterInfo.builder()
+ .clusterVersion(versionNode.get("number").asText())
+ .distribution(
+ Optional.ofNullable(versionNode.get("distribution"))
+ .map(JsonNode::asText)
+ .orElse(null))
+ .build();
+ } catch (IOException e) {
+ throw new ResponseException("fail to get elasticsearch version.", e);
+ }
+ }
+
+ public void close() {
+ try {
+ restClient.close();
+ } catch (IOException e) {
+ log.warn("close elasticsearch connection error", e);
+ }
+ }
+
+ public List<String> listIndex() {
+ String endpoint = "/_cat/indices?format=json";
+ Request request = new Request("GET", endpoint);
+ try {
+ Response response = restClient.performRequest(request);
+ if (response == null) {
+ throw new ResponseException("GET " + endpoint + " response null");
+ }
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ String entity = EntityUtils.toString(response.getEntity());
+ return JsonUtils.toList(entity, Map.class).stream()
+ .map(map -> map.get("index").toString())
+ .collect(Collectors.toList());
+ } else {
+ throw new ResponseException(
+ String.format(
+ "GET %s response status code=%d",
+ endpoint, response.getStatusLine().getStatusCode()));
+ }
+ } catch (IOException ex) {
+ throw new ResponseException(ex);
+ }
+ }
+
+ public void dropIndex(String tableName) {
+ String endpoint = String.format("/%s", tableName);
+ Request request = new Request("DELETE", endpoint);
+ try {
+ Response response = restClient.performRequest(request);
+ if (response == null) {
+ throw new ResponseException("DELETE " + endpoint + " response null");
+ }
+ // todo: if the index doesn't exist, the response status code is 200?
+ if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new ResponseException(
+ String.format(
+ "DELETE %s response status code=%d",
+ endpoint, response.getStatusLine().getStatusCode()));
+ }
+ } catch (IOException ex) {
+ throw new ResponseException(ex);
+ }
+ }
+
+ /**
+ * get es field name and type mapping relation
+ *
+ * @param index index name
+ * @return {key-> field name,value->es type}
+ */
+ public Map<String, String> getFieldTypeMapping(String index) {
+ String endpoint = String.format("/%s/_mappings", index);
+ Request request = new Request("GET", endpoint);
+ Map<String, String> mapping = new HashMap<>();
+ try {
+ Response response = restClient.performRequest(request);
+ if (response == null) {
+ throw new ResponseException("GET " + endpoint + " response null");
+ }
+ if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new ResponseException(
+ String.format(
+ "GET %s response status code=%d",
+ endpoint, response.getStatusLine().getStatusCode()));
+ }
+ String entity = EntityUtils.toString(response.getEntity());
+ log.info(String.format("GET %s response=%s", endpoint, entity));
+ ObjectNode responseJson = JsonUtils.parseObject(entity);
+ for (Iterator<JsonNode> it = responseJson.elements(); it.hasNext(); ) {
+ JsonNode indexProperty = it.next();
+ JsonNode mappingsProperty = indexProperty.get("mappings");
+ if (mappingsProperty.has("mappingsProperty")) {
+ JsonNode properties = mappingsProperty.get("properties");
+ mapping = getFieldTypeMappingFromProperties(properties);
+ } else {
+ for (JsonNode typeNode : mappingsProperty) {
+ JsonNode properties;
+ if (typeNode.has("properties")) {
+ properties = typeNode.get("properties");
+ } else {
+ properties = typeNode;
+ }
+ mapping.putAll(getFieldTypeMappingFromProperties(properties));
+ }
+ }
+ }
+ } catch (IOException ex) {
+ throw new ResponseException(ex);
+ }
+ return mapping;
+ }
+
+ private static Map<String, String> getFieldTypeMappingFromProperties(JsonNode properties) {
+ Map<String, String> mapping = new HashMap<>();
+ for (Iterator<String> it = properties.fieldNames(); it.hasNext(); ) {
+ String field = it.next();
+ JsonNode fieldProperty = properties.get(field);
+ if (fieldProperty == null) {
+ mapping.put(field, "text");
+ } else {
+ if (fieldProperty.has("type")) {
+ String type = fieldProperty.get("type").asText();
+ mapping.put(field, type);
+ } else {
+ log.warn(
+ String.format(
+ "fail to get elasticsearch field %s mapping type,so give a default type text",
+ field));
+ mapping.put(field, "text");
+ }
+ }
+ }
+ return mapping;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/ResponseException.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/ResponseException.java
new file mode 100644
index 00000000..59417cb7
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/ResponseException.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch.client;
+
+public class ResponseException extends RuntimeException {
+
+ public ResponseException() {
+ super();
+ }
+
+ public ResponseException(String message) {
+ super(message);
+ }
+
+ public ResponseException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ResponseException(Throwable cause) {
+ super(cause);
+ }
+
+ protected ResponseException(
+ String message,
+ Throwable cause,
+ boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/SSLUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/SSLUtils.java
new file mode 100644
index 00000000..9c10cd3b
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/client/SSLUtils.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch.client;
+
+import static java.util.Collections.list;
+
+import io.airlift.security.pem.PemReader;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+import javax.security.auth.x500.X500Principal;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateExpiredException;
+import java.security.cert.CertificateNotYetValidException;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+@SuppressWarnings("MagicNumber")
+public final class SSLUtils {
+
+ public static Optional<SSLContext> buildSSLContext(
+ Optional<String> keyStorePath,
+ Optional<String> keyStorePassword,
+ Optional<String> trustStorePath,
+ Optional<String> trustStorePassword)
+ throws GeneralSecurityException, IOException {
+ if (!keyStorePath.isPresent() && !trustStorePath.isPresent()) {
+ return Optional.empty();
+ }
+ return Optional.of(
+ createSSLContext(
+ keyStorePath, keyStorePassword, trustStorePath, trustStorePassword));
+ }
+
+ private static SSLContext createSSLContext(
+ Optional<String> keyStorePath,
+ Optional<String> keyStorePassword,
+ Optional<String> trustStorePath,
+ Optional<String> trustStorePassword)
+ throws GeneralSecurityException, IOException {
+ // load KeyStore if configured and get KeyManagers
+ KeyStore keyStore = null;
+ KeyManager[] keyManagers = null;
+ if (keyStorePath.isPresent()) {
+ File keyStoreFile = new File(keyStorePath.get());
+ char[] keyManagerPassword;
+ try {
+ // attempt to read the key store as a PEM file
+ keyStore = PemReader.loadKeyStore(keyStoreFile, keyStoreFile, keyStorePassword);
+ // for PEM encoded keys, the password is used to decrypt the specific key (and does
+ // not protect the keystore itself)
+ keyManagerPassword = new char[0];
+ } catch (IOException | GeneralSecurityException ignored) {
+ keyManagerPassword = keyStorePassword.map(String::toCharArray).orElse(null);
+
+ keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
+ try (InputStream in = new FileInputStream(keyStoreFile)) {
+ keyStore.load(in, keyManagerPassword);
+ }
+ }
+ validateCertificates(keyStore);
+ KeyManagerFactory keyManagerFactory =
+ KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ keyManagerFactory.init(keyStore, keyManagerPassword);
+ keyManagers = keyManagerFactory.getKeyManagers();
+ }
+
+ // load TrustStore if configured, otherwise use KeyStore
+ KeyStore trustStore = keyStore;
+ if (trustStorePath.isPresent()) {
+ File trustStoreFile = new File(trustStorePath.get());
+ trustStore = loadTrustStore(trustStoreFile, trustStorePassword);
+ }
+
+ // create TrustManagerFactory
+ TrustManagerFactory trustManagerFactory =
+ TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ trustManagerFactory.init(trustStore);
+
+ // get X509TrustManager
+ TrustManager[] trustManagers = trustManagerFactory.getTrustManagers();
+ if (trustManagers.length != 1 || !(trustManagers[0] instanceof X509TrustManager)) {
+ throw new RuntimeException(
+ "Unexpected default trust managers:" + Arrays.toString(trustManagers));
+ }
+ // create SSLContext
+ SSLContext result = SSLContext.getInstance("SSL");
+ result.init(keyManagers, trustManagers, null);
+ return result;
+ }
+
+ private static KeyStore loadTrustStore(File trustStorePath, Optional<String> trustStorePassword)
+ throws IOException, GeneralSecurityException {
+ KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
+ try {
+ // attempt to read the trust store as a PEM file
+ List<X509Certificate> certificateChain = PemReader.readCertificateChain(trustStorePath);
+ if (!certificateChain.isEmpty()) {
+ trustStore.load(null, null);
+ for (X509Certificate certificate : certificateChain) {
+ X500Principal principal = certificate.getSubjectX500Principal();
+ trustStore.setCertificateEntry(principal.getName(), certificate);
+ }
+ return trustStore;
+ }
+ } catch (IOException | GeneralSecurityException ignored) {
+ // ignored
+ }
+
+ try (InputStream in = new FileInputStream(trustStorePath)) {
+ trustStore.load(in, trustStorePassword.map(String::toCharArray).orElse(null));
+ }
+ return trustStore;
+ }
+
+ private static void validateCertificates(KeyStore keyStore) throws GeneralSecurityException {
+ for (String alias : list(keyStore.aliases())) {
+ if (!keyStore.isKeyEntry(alias)) {
+ continue;
+ }
+ Certificate certificate = keyStore.getCertificate(alias);
+ if (!(certificate instanceof X509Certificate)) {
+ continue;
+ }
+
+ try {
+ ((X509Certificate) certificate).checkValidity();
+ } catch (CertificateExpiredException e) {
+ throw new CertificateExpiredException(
+ "KeyStore certificate is expired: " + e.getMessage());
+ } catch (CertificateNotYetValidException e) {
+ throw new CertificateNotYetValidException(
+ "KeyStore certificate is not yet valid: " + e.getMessage());
+ }
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java
new file mode 100644
index 00000000..b8b25d84
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch;
+
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+// todo: use testcontainer to create container
+@Disabled
+class ElasticSearchDataSourceChannelTest {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(ElasticSearchDataSourceChannelTest.class);
+
+ private static final ElasticSearchDataSourceChannel ELASTIC_SEARCH_DATA_SOURCE_CHANNEL =
+ new ElasticSearchDataSourceChannel();
+
+ private static final String PLUGIN_NAME = "ElasticSearch";
+
+ private static final String DATABASE = "Default";
+
+ private static final Map<String, String> REQUEST_MAP =
+ new ImmutableMap.Builder<String, String>()
+ .put(ElasticSearchOptionRule.HOSTS.key(), "[\"http://localhost:9200\"]")
+ .build();
+
+ @Test
+ void canAbleGetSchema() {
+ Assertions.assertTrue(ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.canAbleGetSchema());
+ }
+
+ @Test
+ void getDataSourceOptions() {
+ Assertions.assertNotNull(ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDataSourceOptions(PLUGIN_NAME));
+ }
+
+ @Test
+ void getDatasourceMetadataFieldsByDataSourceName() {
+ Assertions.assertNotNull(
+ ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDatasourceMetadataFieldsByDataSourceName(
+ PLUGIN_NAME));
+ }
+
+ @Test
+ void getTables() {
+ Assertions.assertDoesNotThrow(
+ () -> {
+ List<String> tables =
+ ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getTables(
+ PLUGIN_NAME, REQUEST_MAP, DATABASE);
+ LOGGER.info("{}", tables);
+ });
+ }
+
+ @Test
+ void getDatabases() {
+ Assertions.assertLinesMatch(
+ Lists.newArrayList("default"),
+ ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDatabases(PLUGIN_NAME, REQUEST_MAP));
+ }
+
+ @Test
+ void checkDataSourceConnectivity() {
+ Assertions.assertTrue(
+ ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.checkDataSourceConnectivity(
+ PLUGIN_NAME, REQUEST_MAP));
+ }
+
+ @Test
+ void getTableFields() {
+ Assertions.assertDoesNotThrow(
+ () -> {
+ List<TableField> tableFields =
+ ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getTableFields(
+ PLUGIN_NAME, REQUEST_MAP, DATABASE, "");
+ LOGGER.info("{}", tableFields);
+ });
+ }
+
+ @Test
+ void testGetTableFields() {
+ Assertions.assertDoesNotThrow(
+ () -> {
+ Map<String, List<TableField>> tableFields =
+ ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getTableFields(
+ PLUGIN_NAME, REQUEST_MAP, DATABASE, Lists.newArrayList(""));
+ LOGGER.info("{}", tableFields);
+ });
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactoryTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactoryTest.java
new file mode 100644
index 00000000..85a49e77
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceFactoryTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.elasticsearch;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class ElasticSearchDataSourceFactoryTest {
+
+ private static final ElasticSearchDataSourceFactory ELASTIC_SEARCH_DATA_SOURCE_FACTORY =
+ new ElasticSearchDataSourceFactory();
+
+ @Test
+ void factoryIdentifier() {
+ Assertions.assertEquals(
+ "ElasticSearch", ELASTIC_SEARCH_DATA_SOURCE_FACTORY.factoryIdentifier());
+ }
+
+ @Test
+ void supportedDataSources() {
+ Assertions.assertFalse(ELASTIC_SEARCH_DATA_SOURCE_FACTORY.supportedDataSources().isEmpty());
+ }
+
+ @Test
+ void createChannel() {
+ Assertions.assertNotNull(ELASTIC_SEARCH_DATA_SOURCE_FACTORY.createChannel());
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml
new file mode 100644
index 00000000..1ca92dfb
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-jdbc-clickhouse</artifactId>
+
+ <properties>
+ <clickhouse.version>0.3.2-patch11</clickhouse.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+
+ <!-- driver -->
+ <dependency>
+ <groupId>com.clickhouse</groupId>
+ <artifactId>clickhouse-jdbc</artifactId>
+ <version>${clickhouse.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseDataSourceConfig.java
new file mode 100644
index 00000000..4862d41d
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseDataSourceConfig.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.clickhouse.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class ClickhouseDataSourceConfig {
+ public static final String PLUGIN_NAME = "JDBC-ClickHouse";
+
+ public static final DataSourcePluginInfo CLICKHOUSE_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .supportVirtualTables(false)
+ .build();
+
+ public static final Set<String> CLICKHOUSE_SYSTEM_DATABASES =
+ Sets.newHashSet(
+ "system",
+ "default",
+ "information_schema",
+ "mysql",
+ "performance_schema",
+ "sys");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder()
+ .required(ClickhouseOptionRule.URL, ClickhouseOptionRule.DRIVER)
+ .optional(ClickhouseOptionRule.USER, ClickhouseOptionRule.PASSWORD)
+ .build();
+
+ public static final OptionRule METADATA_RULE =
+ OptionRule.builder()
+ .required(ClickhouseOptionRule.DATABASE, ClickhouseOptionRule.TABLE)
+ .build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java
new file mode 100644
index 00000000..37141174
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.clickhouse.jdbc;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+public class ClickhouseJdbcDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return ClickhouseDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return ClickhouseDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);) {
+ ResultSet resultSet =
+ connection
+ .getMetaData()
+ .getTables(database, null, null, new String[]{"TABLE"});
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ }
+ }
+ return tableNames;
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ Statement statement = connection.createStatement();
+ ResultSet re = statement.executeQuery("SHOW DATABASES;")) {
+ // filter system databases
+ while (re.next()) {
+ String dbName = re.getString("name");
+ if (StringUtils.isNotBlank(dbName)
+ && !ClickhouseDataSourceConfig.CLICKHOUSE_SYSTEM_DATABASES.contains(
+ dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ } catch (Exception ex) {
+ throw new RuntimeException("get databases failed", ex);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (Connection ignored = getConnection(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException("Check jdbc connectivity failed", e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams, database)) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, database, table);
+ try (ResultSet resultSet = metaData.getColumns(database, null, table, null)) {
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString()));
+ tableFields.add(tableField);
+ }
+ }
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("Get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ return null;
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private Connection getConnection(Map<String, String> requestParams)
+ throws SQLException, ClassNotFoundException {
+ return getConnection(requestParams, null);
+ }
+
+ private Connection getConnection(Map<String, String> requestParams, String databaseName)
+ throws SQLException, ClassNotFoundException {
+ checkNotNull(requestParams.get(ClickhouseOptionRule.DRIVER.key()));
+ checkNotNull(requestParams.get(ClickhouseOptionRule.URL.key()), "Jdbc url cannot be null");
+ String url =
+ JdbcUtils.replaceDatabase(
+ requestParams.get(ClickhouseOptionRule.URL.key()), databaseName);
+ if (requestParams.containsKey(ClickhouseOptionRule.USER.key())) {
+ String username = requestParams.get(ClickhouseOptionRule.USER.key());
+ String password = requestParams.get(ClickhouseOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceFactory.java
new file mode 100644
index 00000000..f8b97986
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.clickhouse.jdbc;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+
+@Slf4j
+@AutoService(DataSourceFactory.class)
+public class ClickhouseJdbcDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return ClickhouseDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(ClickhouseDataSourceConfig.CLICKHOUSE_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new ClickhouseJdbcDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseOptionRule.java
new file mode 100644
index 00000000..3beff605
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseOptionRule.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.clickhouse.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public class ClickhouseOptionRule {
+
+ public static final Option<String> URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "jdbc url, eg:"
+ + "jdbc:clickhouse://localhost:8123/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
+
+ public static final Option<String> USER =
+ Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
+
+ public static final Option<String> DATABASE =
+ Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
+
+ public static final Option<String> TABLE =
+ Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
+
+ public static final Option<DriverType> DRIVER =
+ Options.key("driver")
+ .enumType(DriverType.class)
+ .noDefaultValue()
+ .withDescription("driver");
+
+ public enum DriverType {
+ ClickHouse("ru.yandex.clickhouse.ClickHouseDriver");
+ private final String driverClassName;
+
+ DriverType(String driverClassName) {
+ this.driverClassName = driverClassName;
+ }
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ @Override
+ public String toString() {
+ return driverClassName;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml
new file mode 100644
index 00000000..c73b4502
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-jdbc-hive</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcConstants.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcConstants.java
new file mode 100644
index 00000000..8b133ac7
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcConstants.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.hive.jdbc;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class HiveJdbcConstants {
+
+ public static final Set<String> HIVE_SYSTEM_DATABASES =
+ Sets.newHashSet(
+ "information_schema", "mysql", "performance_schema", "sys", "test", "hivedb");
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java
new file mode 100644
index 00000000..32543c42
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.hive.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+public class HiveJdbcDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return HiveJdbcOptionRule.optionRule();
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return HiveJdbcOptionRule.metadataRule();
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ return getTables(pluginName, requestParams, database);
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try {
+ return getDataBaseNames(pluginName, requestParams);
+ } catch (SQLException e) {
+ log.error("Query Hive databases error, request params is {}", requestParams, e);
+ throw new DataSourcePluginException("Query Hive databases error,", e);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ return checkJdbcConnectivity(requestParams);
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ return getTableFields(requestParams, database, table);
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ Map<String, List<TableField>> tableFields = new HashMap<>(tables.size());
+ for (String table : tables) {
+ tableFields.put(table, getTableFields(requestParams, database, table));
+ }
+ return tableFields;
+ }
+
+ protected boolean checkJdbcConnectivity(Map<String, String> requestParams) {
+ try (Connection ignored = init(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException(
+ "check jdbc connectivity failed, " + e.getMessage(), e);
+ }
+ }
+
+ protected Connection init(Map<String, String> requestParams) throws SQLException {
+ if (MapUtils.isEmpty(requestParams)) {
+ throw new DataSourcePluginException(
+ "Hive jdbc request params is null, please check your config");
+ }
+ String url = requestParams.get(HiveJdbcOptionRule.URL.key());
+ return DriverManager.getConnection(url);
+ }
+
+ protected List<String> getDataBaseNames(String pluginName, Map<String, String> requestParams)
+ throws SQLException {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = init(requestParams);
+ Statement statement = connection.createStatement();) {
+ ResultSet re = statement.executeQuery("SHOW DATABASES;");
+ // filter system databases
+ while (re.next()) {
+ String dbName = re.getString("database");
+ if (StringUtils.isNotBlank(dbName) && isNotSystemDatabase(pluginName, dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ }
+ }
+
+ protected List<String> getTableNames(Map<String, String> requestParams, String dbName) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = init(requestParams);) {
+ ResultSet resultSet =
+ connection.getMetaData().getTables(dbName, null, null, new String[]{"TABLE"});
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ }
+ }
+ return tableNames;
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ protected List<TableField> getTableFields(
+ Map<String, String> requestParams, String dbName, String tableName) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = init(requestParams);) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, dbName, tableName);
+ ResultSet resultSet = metaData.getColumns(dbName, null, tableName, null);
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ boolean isNullable = convertToBoolean(nullable);
+ tableField.setNullable(isNullable);
+ tableFields.add(tableField);
+ }
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ @SuppressWarnings("checkstyle:MagicNumber")
+ private static boolean checkHostConnectable(String host, int port) {
+ try (Socket socket = new Socket()) {
+ socket.connect(new InetSocketAddress(host, port), 1000);
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ private boolean isNotSystemDatabase(String pluginName, String dbName) {
+ // FIXME,filters system databases
+ return true;
+ }
+
+ private boolean convertToBoolean(Object value) {
+ if (value instanceof Boolean) {
+ return (Boolean) value;
+ }
+ if (value instanceof String) {
+ return value.equals("TRUE");
+ }
+ return false;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceFactory.java
new file mode 100644
index 00000000..b149cc3b
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceFactory.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.hive.jdbc;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class HiveJdbcDataSourceFactory implements DataSourceFactory {
+ @Override
+ public String factoryIdentifier() {
+ return "Hive-JDBC";
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ DataSourcePluginInfo dataSourcePluginInfo =
+ DataSourcePluginInfo.builder()
+ .name("Hive-JDBC")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .version("1.0.0")
+ .icon("Hive-JDBC")
+ .supportVirtualTables(false)
+ .build();
+ Set<DataSourcePluginInfo> dataSourceInfos = new HashSet<>();
+ dataSourceInfos.add(dataSourcePluginInfo);
+ return dataSourceInfos;
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new HiveJdbcDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcOptionRule.java
new file mode 100644
index 00000000..fb004b69
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcOptionRule.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.hive.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+public class HiveJdbcOptionRule {
+
+ public static final Option<String> URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "jdbc url, eg:"
+ + "jdbc:hive2://localhost:10000/default?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
+
+ public static OptionRule optionRule() {
+ return OptionRule.builder().required(URL).build();
+ }
+
+ public static OptionRule metadataRule() {
+ // todo
+ return OptionRule.builder().build();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml
new file mode 100644
index 00000000..11e0ea6b
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-jdbc-mysql</artifactId>
+
+ <properties>
+ <mysql-connector.version>8.0.28</mysql-connector.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+
+ <!-- driver -->
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql-connector.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlDataSourceConfig.java
new file mode 100644
index 00000000..2e2de3e4
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlDataSourceConfig.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.mysql.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class MysqlDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "JDBC-Mysql";
+
+ public static final DataSourcePluginInfo MYSQL_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+
+ public static final Set<String> MYSQL_SYSTEM_DATABASES =
+ Sets.newHashSet("information_schema", "mysql", "performance_schema", "sys");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder()
+ .required(MysqlOptionRule.URL, MysqlOptionRule.DRIVER)
+ .optional(MysqlOptionRule.USER, MysqlOptionRule.PASSWORD)
+ .build();
+
+ public static final OptionRule METADATA_RULE =
+ OptionRule.builder().required(MysqlOptionRule.DATABASE, MysqlOptionRule.TABLE).build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java
new file mode 100644
index 00000000..1fe420d5
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.mysql.jdbc;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
+
+import lombok.NonNull;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class MysqlJdbcDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return MysqlDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return MysqlDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ ResultSet resultSet =
+ connection
+ .getMetaData()
+ .getTables(database, null, null, new String[]{"TABLE"})) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ }
+ }
+ return tableNames;
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;");
+ ResultSet re = statement.executeQuery()) {
+ // filter system databases
+ while (re.next()) {
+ String dbName = re.getString("database");
+ if (StringUtils.isNotBlank(dbName)
+ && !MysqlDataSourceConfig.MYSQL_SYSTEM_DATABASES.contains(dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new DataSourcePluginException("Get databases failed", e);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (Connection ignored = getConnection(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException("check jdbc connectivity failed", e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams, database)) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, database, table);
+ try (ResultSet resultSet = metaData.getColumns(database, null, table, null)) {
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString()));
+ tableFields.add(tableField);
+ }
+ }
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ return tables.parallelStream()
+ .collect(
+ Collectors.toMap(
+ Function.identity(),
+ table ->
+ getTableFields(
+ pluginName, requestParams, database, table)));
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private Connection getConnection(Map<String, String> requestParams)
+ throws SQLException, ClassNotFoundException {
+ return getConnection(requestParams, null);
+ }
+
+ private Connection getConnection(Map<String, String> requestParams, String databaseName)
+ throws SQLException, ClassNotFoundException {
+ checkNotNull(requestParams.get(MysqlOptionRule.DRIVER.key()));
+ checkNotNull(requestParams.get(MysqlOptionRule.URL.key()), "Jdbc url cannot be null");
+ String url =
+ JdbcUtils.replaceDatabase(
+ requestParams.get(MysqlOptionRule.URL.key()), databaseName);
+ if (requestParams.containsKey(MysqlOptionRule.USER.key())) {
+ String username = requestParams.get(MysqlOptionRule.USER.key());
+ String password = requestParams.get(MysqlOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceFactory.java
new file mode 100644
index 00000000..31f6439b
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.mysql.jdbc;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+
+@Slf4j
+@AutoService(DataSourceFactory.class)
+public class MysqlJdbcDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return MysqlDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(MysqlDataSourceConfig.MYSQL_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new MysqlJdbcDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlOptionRule.java
new file mode 100644
index 00000000..f666251f
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlOptionRule.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.mysql.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public class MysqlOptionRule {
+
+ public static final Option<String> URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "jdbc url, eg:"
+ + " jdbc:mysql://localhost:3306/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
+
+ public static final Option<String> USER =
+ Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
+
+ public static final Option<String> DATABASE =
+ Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
+
+ public static final Option<String> TABLE =
+ Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
+
+ public static final Option<DriverType> DRIVER =
+ Options.key("driver")
+ .enumType(DriverType.class)
+ .defaultValue(DriverType.MYSQL)
+ .withDescription("driver");
+
+ public enum DriverType {
+ MYSQL("com.mysql.cj.jdbc.Driver"),
+ ;
+ private final String driverClassName;
+
+ DriverType(String driverClassName) {
+ this.driverClassName = driverClassName;
+ }
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ @Override
+ public String toString() {
+ return driverClassName;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml
new file mode 100644
index 00000000..1e48c9b8
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-jdbc-oracle</artifactId>
+
+ <properties>
+ <oracle-jdbc.version>21.5.0.0</oracle-jdbc.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+
+ <!-- driver -->
+ <dependency>
+ <groupId>com.oracle.database.jdbc</groupId>
+ <artifactId>ojdbc8</artifactId>
+ <version>${oracle-jdbc.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java
new file mode 100644
index 00000000..91ccc661
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.oracle.jdbc;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
+
+import lombok.NonNull;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class OracleDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return OracleDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return OracleDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ ResultSet resultSet =
+ connection
+ .getMetaData()
+ .getTables(database, null, null, new String[]{"TABLE"});) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ }
+ }
+ return tableNames;
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;");
+ ResultSet re = statement.executeQuery()) {
+ // filter system databases
+ while (re.next()) {
+ String dbName = re.getString("database");
+ if (StringUtils.isNotBlank(dbName)
+ && !OracleDataSourceConfig.ORACLE_SYSTEM_DATABASES.contains(dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ } catch (Exception ex) {
+ throw new RuntimeException("get databases failed", ex);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (Connection ignored = getConnection(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException("check jdbc connectivity failed", e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams, database)) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, database, table);
+ try (ResultSet resultSet = metaData.getColumns(database, null, table, null)) {
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString()));
+ tableFields.add(tableField);
+ }
+ }
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ return null;
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private Connection getConnection(Map<String, String> requestParams)
+ throws SQLException, ClassNotFoundException {
+ return getConnection(requestParams, null);
+ }
+
+ private Connection getConnection(Map<String, String> requestParams, String databaseName)
+ throws SQLException, ClassNotFoundException {
+ checkNotNull(requestParams.get(OracleOptionRule.DRIVER.key()));
+ checkNotNull(requestParams.get(OracleOptionRule.URL.key()), "Jdbc url cannot be null");
+ String url =
+ JdbcUtils.replaceDatabase(
+ requestParams.get(OracleOptionRule.URL.key()), databaseName);
+ if (requestParams.containsKey(OracleOptionRule.USER.key())) {
+ String username = requestParams.get(OracleOptionRule.USER.key());
+ String password = requestParams.get(OracleOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceConfig.java
new file mode 100644
index 00000000..96f09fbe
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceConfig.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.oracle.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class OracleDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "JDBC-Oracle";
+
+ public static final DataSourcePluginInfo ORACLE_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+
+ public static final Set<String> ORACLE_SYSTEM_DATABASES =
+ Sets.newHashSet("SYS", "SYSTEM", "SYSDBA", "SYSOPER", "HR", "SCOTT");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder()
+ .required(OracleOptionRule.URL, OracleOptionRule.DRIVER)
+ .optional(OracleOptionRule.USER, OracleOptionRule.PASSWORD)
+ .build();
+
+ public static final OptionRule METADATA_RULE =
+ OptionRule.builder()
+ .required(OracleOptionRule.DATABASE, OracleOptionRule.TABLE)
+ .build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleJdbcDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleJdbcDataSourceFactory.java
new file mode 100644
index 00000000..b6a0aec9
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleJdbcDataSourceFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.oracle.jdbc;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+
+@Slf4j
+@AutoService(DataSourceFactory.class)
+public class OracleJdbcDataSourceFactory implements DataSourceFactory {
+ @Override
+ public String factoryIdentifier() {
+ return OracleDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(OracleDataSourceConfig.ORACLE_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new OracleDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleOptionRule.java
new file mode 100644
index 00000000..517a8c32
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleOptionRule.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.oracle.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public class OracleOptionRule {
+
+ public static final Option<String> URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("jdbc url, eg:" + "jdbc:oracle:thin:@localhost:1521:XE");
+
+ public static final Option<String> USER =
+ Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
+
+ public static final Option<String> DATABASE =
+ Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
+
+ public static final Option<String> TABLE =
+ Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
+
+ public static final Option<DriverType> DRIVER =
+ Options.key("driver")
+ .enumType(DriverType.class)
+ .defaultValue(DriverType.ORACLE)
+ .withDescription("driver");
+
+ public enum DriverType {
+ ORACLE("oracle.jdbc.driver.OracleDriver"),
+ ;
+ private final String driverClassName;
+
+ DriverType(String driverClassName) {
+ this.driverClassName = driverClassName;
+ }
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ @Override
+ public String toString() {
+ return driverClassName;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml
new file mode 100644
index 00000000..cffbf251
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-jdbc-postgresql</artifactId>
+
+ <properties>
+ <postgresql.version>42.4.3</postgresql.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+
+ <!-- driver -->
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${postgresql.version}</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java
new file mode 100644
index 00000000..b331d61d
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.postgresql.jdbc;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
+
+import lombok.NonNull;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class PostgresqlDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return PostgresqlDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return PostgresqlDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ List<String> tableNames = new ArrayList<>();
+ String query = "SELECT table_schema, table_name FROM information_schema.tables";
+ try (Connection connection = getConnection(requestParams, database)) {
+ try (Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(query)) {
+ while (resultSet.next()) {
+ String schemaName = resultSet.getString("table_schema");
+ String tableName = resultSet.getString("table_name");
+ if (StringUtils.isNotBlank(schemaName)
+ && !PostgresqlDataSourceConfig.POSTGRESQL_SYSTEM_DATABASES.contains(
+ schemaName)) {
+ tableNames.add(schemaName + "." + tableName);
+ }
+ }
+ }
+ return tableNames;
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ PreparedStatement statement =
+ connection.prepareStatement("select datname from pg_database;");
+ ResultSet re = statement.executeQuery()) {
+ while (re.next()) {
+ String dbName = re.getString("datname");
+ if (StringUtils.isNotBlank(dbName)
+ && !PostgresqlDataSourceConfig.POSTGRESQL_SYSTEM_DATABASES.contains(
+ dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new DataSourcePluginException("get databases failed", e);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (Connection ignored = getConnection(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException("check jdbc connectivity failed", e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams, database);) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, database, table);
+ String[] split = table.split("\\.");
+ if (split.length != 2) {
+ throw new DataSourcePluginException(
+ "Postgresql tableName should composed by schemaName.tableName");
+ }
+ try (ResultSet resultSet = metaData.getColumns(database, split[0], split[1], null)) {
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString()));
+ tableFields.add(tableField);
+ }
+ }
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ return null;
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private Connection getConnection(Map<String, String> requestParams)
+ throws SQLException, ClassNotFoundException {
+ return getConnection(requestParams, null);
+ }
+
+ private Connection getConnection(Map<String, String> requestParams, String databaseName)
+ throws SQLException, ClassNotFoundException {
+ checkNotNull(requestParams.get(PostgresqlOptionRule.DRIVER.key()));
+ checkNotNull(requestParams.get(PostgresqlOptionRule.URL.key()), "Jdbc url cannot be null");
+ String url =
+ JdbcUtils.replaceDatabase(
+ requestParams.get(PostgresqlOptionRule.URL.key()), databaseName);
+ if (requestParams.containsKey(PostgresqlOptionRule.USER.key())) {
+ String username = requestParams.get(PostgresqlOptionRule.USER.key());
+ String password = requestParams.get(PostgresqlOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceConfig.java
new file mode 100644
index 00000000..2aa0538b
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceConfig.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.postgresql.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class PostgresqlDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "JDBC-Postgres";
+
+ public static final DataSourcePluginInfo POSTGRESQL_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+
+ public static final Set<String> POSTGRESQL_SYSTEM_DATABASES =
+ Sets.newHashSet(
+ "information_schema",
+ "pg_catalog",
+ "root",
+ "pg_toast",
+ "pg_temp_1",
+ "pg_toast_temp_1",
+ "postgres",
+ "template0",
+ "template1");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder()
+ .required(PostgresqlOptionRule.URL, PostgresqlOptionRule.DRIVER)
+ .optional(PostgresqlOptionRule.USER, PostgresqlOptionRule.PASSWORD)
+ .build();
+
+ public static final OptionRule METADATA_RULE =
+ OptionRule.builder()
+ .required(PostgresqlOptionRule.DATABASE, PostgresqlOptionRule.TABLE)
+ .build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceFactory.java
new file mode 100644
index 00000000..13dd7847
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.postgresql.jdbc;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+@AutoService(DataSourceFactory.class)
+public class PostgresqlDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return PostgresqlDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(PostgresqlDataSourceConfig.POSTGRESQL_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new PostgresqlDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlOptionRule.java
new file mode 100644
index 00000000..16e8b85b
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlOptionRule.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.postgresql.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public class PostgresqlOptionRule {
+
+ public static final Option<String> URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "jdbc url, eg:"
+ + "jdbc:postgresql://localhost:5432//test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
+
+ public static final Option<String> USER =
+ Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
+
+ public static final Option<String> DATABASE =
+ Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
+
+ public static final Option<String> TABLE =
+ Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
+
+ public static final Option<DriverType> DRIVER =
+ Options.key("driver")
+ .enumType(DriverType.class)
+ .defaultValue(DriverType.POSTGRESQL)
+ .withDescription("driver");
+
+ public enum DriverType {
+ POSTGRESQL("org.postgresql.Driver"),
+ ;
+ private final String driverClassName;
+
+ DriverType(String driverClassName) {
+ this.driverClassName = driverClassName;
+ }
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ @Override
+ public String toString() {
+ return driverClassName;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml
new file mode 100644
index 00000000..fc369127
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-jdbc-redshift</artifactId>
+
+ <properties>
+ <redshift.version>2.1.0.10</redshift.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+
+ <!-- driver -->
+ <dependency>
+ <groupId>com.amazon.redshift</groupId>
+ <artifactId>redshift-jdbc42</artifactId>
+ <version>${redshift.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java
new file mode 100644
index 00000000..32f77e98
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.redshift.jdbc;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
+
+import lombok.NonNull;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class RedshiftDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return RedshiftDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return RedshiftDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams, database);
+ ResultSet resultSet =
+ connection.getMetaData().getTables(database, null, null, null);) {
+ while (resultSet.next()) {
+ String schemaName = resultSet.getString("TABLE_SCHEM");
+ String tableName = resultSet.getString("TABLE_NAME");
+ // todo: use isNotSystemSchemaName
+ if (StringUtils.isNotBlank(schemaName)
+ && !RedshiftDataSourceConfig.REDSHIFT_SYSTEM_TABLES.contains(schemaName)) {
+ tableNames.add(schemaName + "." + tableName);
+ }
+ }
+ return tableNames;
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ PreparedStatement statement =
+ connection.prepareStatement("select datname from pg_database;");
+ ResultSet re = statement.executeQuery()) {
+ while (re.next()) {
+ String dbName = re.getString("datname");
+ if (StringUtils.isNotBlank(dbName)
+ && !RedshiftDataSourceConfig.REDSHIFT_SYSTEM_TABLES.contains(dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new DataSourcePluginException("get databases failed", e);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (Connection ignored = getConnection(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException("check jdbc connectivity failed", e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams, database);) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, database, table);
+ String[] split = table.split("\\.");
+ if (split.length != 2) {
+ throw new DataSourcePluginException(
+ "Postgresql tableName should composed by schemaName.tableName");
+ }
+ try (ResultSet resultSet = metaData.getColumns(database, split[0], split[1], null)) {
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString()));
+ tableFields.add(tableField);
+ }
+ }
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ return null;
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private Connection getConnection(Map<String, String> requestParams)
+ throws SQLException, ClassNotFoundException {
+ return getConnection(requestParams, null);
+ }
+
+ private Connection getConnection(Map<String, String> requestParams, String databaseName)
+ throws SQLException, ClassNotFoundException {
+ checkNotNull(requestParams.get(RedshiftOptionRule.DRIVER.key()));
+ checkNotNull(requestParams.get(RedshiftOptionRule.URL.key()), "Jdbc url cannot be null");
+ String url =
+ JdbcUtils.replaceDatabase(
+ requestParams.get(RedshiftOptionRule.URL.key()), databaseName);
+ if (requestParams.containsKey(RedshiftOptionRule.USER.key())) {
+ String username = requestParams.get(RedshiftOptionRule.USER.key());
+ String password = requestParams.get(RedshiftOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceConfig.java
new file mode 100644
index 00000000..450c8d95
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceConfig.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.redshift.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class RedshiftDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "JDBC-Redshift";
+
+ public static final DataSourcePluginInfo REDSHIFT_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon("redshift")
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+
+ public static final Set<String> REDSHIFT_SYSTEM_TABLES =
+ Sets.newHashSet(
+ "information_schema",
+ "pg_catalog",
+ "root",
+ "pg_toast",
+ "pg_temp_1",
+ "pg_toast_temp_1",
+ "postgres",
+ "template0",
+ "template1");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder()
+ .required(RedshiftOptionRule.URL, RedshiftOptionRule.DRIVER)
+ .optional(RedshiftOptionRule.USER, RedshiftOptionRule.PASSWORD)
+ .build();
+
+ public static final OptionRule METADATA_RULE =
+ OptionRule.builder()
+ .required(RedshiftOptionRule.DATABASE, RedshiftOptionRule.TABLE)
+ .build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceFactory.java
new file mode 100644
index 00000000..bf33a24c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.redshift.jdbc;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+
+@Slf4j
+@AutoService(DataSourceFactory.class)
+public class RedshiftDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return RedshiftDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(RedshiftDataSourceConfig.REDSHIFT_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new RedshiftDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftOptionRule.java
new file mode 100644
index 00000000..ba4c11c0
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftOptionRule.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.redshift.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public class RedshiftOptionRule {
+
+ public static final Option<String> URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "jdbc url, eg:"
+ + "jdbc:redshift://server.redshift.amazonaws.com:5439/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
+
+ public static final Option<String> USER =
+ Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
+
+ public static final Option<String> DATABASE =
+ Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
+
+ public static final Option<String> TABLE =
+ Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
+
+ public static final Option<DriverType> DRIVER =
+ Options.key("driver")
+ .enumType(DriverType.class)
+ .defaultValue(DriverType.JDBC42_REDSHIFT)
+ .withDescription("driver");
+
+ public enum DriverType {
+ JDBC42_REDSHIFT("com.amazon.redshift.jdbc42.Driver"),
+ ;
+ private final String driverClassName;
+
+ DriverType(String driverClassName) {
+ this.driverClassName = driverClassName;
+ }
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ @Override
+ public String toString() {
+ return driverClassName;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml
new file mode 100644
index 00000000..5452a7e8
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-jdbc-sqlserver</artifactId>
+
+ <properties>
+ <sqlserver.version>9.2.1.jre8</sqlserver.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+
+ <!-- driver -->
+ <dependency>
+ <groupId>com.microsoft.sqlserver</groupId>
+ <artifactId>mssql-jdbc</artifactId>
+ <version>${sqlserver.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java
new file mode 100644
index 00000000..bde7550c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.sqlserver.jdbc;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+public class SqlServerDataSourceChannel implements DataSourceChannel {
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return SqlServerDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return SqlServerDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ ResultSet resultSet =
+ connection
+ .getMetaData()
+ .getTables(database, null, null, new String[]{"TABLE"})) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ }
+ }
+ return tableNames;
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;");
+ ResultSet re = statement.executeQuery()) {
+ // filter system databases
+ while (re.next()) {
+ String dbName = re.getString("database");
+ if (StringUtils.isNotBlank(dbName)
+ && !SqlServerDataSourceConfig.SQLSERVER_SYSTEM_DATABASES.contains(dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ } catch (Exception ex) {
+ throw new RuntimeException("get databases failed", ex);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (Connection ignored = getConnection(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException("check jdbc connectivity failed", e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams, database)) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, database, table);
+ try (ResultSet resultSet = metaData.getColumns(database, null, table, null)) {
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString()));
+ tableFields.add(tableField);
+ }
+ }
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ return null;
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private Connection getConnection(Map<String, String> requestParams)
+ throws SQLException, ClassNotFoundException {
+ return getConnection(requestParams, null);
+ }
+
+ private Connection getConnection(Map<String, String> requestParams, String databaseName)
+ throws SQLException, ClassNotFoundException {
+ checkNotNull(requestParams.get(SqlServerOptionRule.DRIVER.key()));
+ checkNotNull(requestParams.get(SqlServerOptionRule.URL.key()), "Jdbc url cannot be null");
+ String url =
+ JdbcUtils.replaceDatabase(
+ requestParams.get(SqlServerOptionRule.URL.key()), databaseName);
+ if (requestParams.containsKey(SqlServerOptionRule.USER.key())) {
+ String username = requestParams.get(SqlServerOptionRule.USER.key());
+ String password = requestParams.get(SqlServerOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceConfig.java
new file mode 100644
index 00000000..3bfc056c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceConfig.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.sqlserver.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class SqlServerDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "JDBC-SQLServer";
+
+ public static final DataSourcePluginInfo SQLSERVER_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+
+ public static final Set<String> SQLSERVER_SYSTEM_DATABASES =
+ Sets.newHashSet(
+ "master",
+ "tempdb",
+ "model",
+ "msdb",
+ "ReportServer",
+ "ReportServerTempDB",
+ "SSISDB");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder()
+ .required(SqlServerOptionRule.URL, SqlServerOptionRule.DRIVER)
+ .optional(SqlServerOptionRule.USER, SqlServerOptionRule.PASSWORD)
+ .build();
+
+ public static final OptionRule METADATA_RULE =
+ OptionRule.builder()
+ .required(SqlServerOptionRule.DATABASE, SqlServerOptionRule.TABLE)
+ .build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceFactory.java
new file mode 100644
index 00000000..b9a21ce4
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.sqlserver.jdbc;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+
+@Slf4j
+@AutoService(DataSourceFactory.class)
+public class SqlServerDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return SqlServerDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(SqlServerDataSourceConfig.SQLSERVER_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new SqlServerDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerOptionRule.java
new file mode 100644
index 00000000..ec157d39
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerOptionRule.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.sqlserver.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public class SqlServerOptionRule {
+
+ public static final Option<String> URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "jdbc url, eg:" + "jdbc:sqlserver://localhost:1433;database=xx");
+
+ public static final Option<String> USER =
+ Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
+
+ public static final Option<String> DATABASE =
+ Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
+
+ public static final Option<String> TABLE =
+ Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
+
+ public static final Option<DriverType> DRIVER =
+ Options.key("driver")
+ .enumType(DriverType.class)
+ .defaultValue(DriverType.SQL_SERVER)
+ .withDescription("driver");
+
+ public enum DriverType {
+ SQL_SERVER("com.microsoft.sqlserver.jdbc.SQLServerDriver"),
+ ;
+ private final String driverClassName;
+
+ DriverType(String driverClassName) {
+ this.driverClassName = driverClassName;
+ }
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ @Override
+ public String toString() {
+ return driverClassName;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/pom.xml
new file mode 100644
index 00000000..76a5885f
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-jdbc-starrocks</artifactId>
+
+ <properties>
+ <mysql-connector.version>8.0.28</mysql-connector.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+
+ <!-- driver -->
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql-connector.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksDataSourceConfig.java
new file mode 100644
index 00000000..427657cc
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksDataSourceConfig.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.starrocks.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class StarRocksDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "JDBC-StarRocks";
+
+ public static final DataSourcePluginInfo STAR_ROCKS_DATA_SOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+
+ public static final Set<String> STAR_ROCKS_SYSTEM_DATABASES =
+ Sets.newHashSet("_statistics_", "information_schema");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder()
+ .required(StarRocksOptionRule.URL, StarRocksOptionRule.DRIVER)
+ .optional(StarRocksOptionRule.USER, StarRocksOptionRule.PASSWORD)
+ .build();
+
+ public static final OptionRule METADATA_RULE =
+ OptionRule.builder()
+ .required(StarRocksOptionRule.DATABASE, StarRocksOptionRule.TABLE)
+ .build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java
new file mode 100644
index 00000000..3f429a7c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.starrocks.jdbc;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
+
+import lombok.NonNull;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class StarRocksJdbcDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return StarRocksDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return StarRocksDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ ResultSet resultSet =
+ connection
+ .getMetaData()
+ .getTables(database, null, null, new String[]{"TABLE"})) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ }
+ }
+ return tableNames;
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;");
+ ResultSet re = statement.executeQuery()) {
+ // filter system databases
+ while (re.next()) {
+ String dbName = re.getString("database");
+ if (StringUtils.isNotBlank(dbName)
+ && !StarRocksDataSourceConfig.STAR_ROCKS_SYSTEM_DATABASES.contains(
+ dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ } catch (Exception ex) {
+ throw new DataSourcePluginException("get databases failed", ex);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (Connection ignored = getConnection(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException("check jdbc connectivity failed", e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams, database)) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, database, table);
+ try (ResultSet resultSet = metaData.getColumns(database, null, table, null);) {
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString()));
+ tableFields.add(tableField);
+ }
+ }
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ return null;
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private Connection getConnection(Map<String, String> requestParams)
+ throws SQLException, ClassNotFoundException {
+ return getConnection(requestParams, null);
+ }
+
+ private Connection getConnection(Map<String, String> requestParams, String databaseName)
+ throws SQLException, ClassNotFoundException {
+ checkNotNull(requestParams.get(StarRocksOptionRule.DRIVER.key()));
+ checkNotNull(requestParams.get(StarRocksOptionRule.URL.key()), "Jdbc url cannot be null");
+ String url =
+ JdbcUtils.replaceDatabase(
+ requestParams.get(StarRocksOptionRule.URL.key()), databaseName);
+ if (requestParams.containsKey(StarRocksOptionRule.USER.key())) {
+ String username = requestParams.get(StarRocksOptionRule.USER.key());
+ String password = requestParams.get(StarRocksOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceFactory.java
new file mode 100644
index 00000000..65620d0e
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.starrocks.jdbc;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+@AutoService(DataSourceFactory.class)
+public class StarRocksJdbcDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return StarRocksDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(StarRocksDataSourceConfig.STAR_ROCKS_DATA_SOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new StarRocksJdbcDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksOptionRule.java
new file mode 100644
index 00000000..d67c2a30
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksOptionRule.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.starrocks.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public class StarRocksOptionRule {
+
+ public static final Option<String> URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "jdbc url, eg:"
+ + "jdbc:mysql://localhost:9030/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
+
+ public static final Option<String> USER =
+ Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
+
+ public static final Option<String> DATABASE =
+ Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
+
+ public static final Option<String> TABLE =
+ Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
+
+ public static final Option<DriverType> DRIVER =
+ Options.key("driver")
+ .enumType(DriverType.class)
+ .defaultValue(DriverType.MYSQL)
+ .withDescription("driver");
+
+ public enum DriverType {
+ MYSQL("com.mysql.cj.jdbc.Driver"),
+ ;
+ private final String driverClassName;
+
+ DriverType(String driverClassName) {
+ this.driverClassName = driverClassName;
+ }
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ @Override
+ public String toString() {
+ return driverClassName;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml
new file mode 100644
index 00000000..5797396d
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-jdbc-tidb</artifactId>
+
+ <properties />
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-jdbc-mysql</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbDataSourceConfig.java
new file mode 100644
index 00000000..f02caff2
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbDataSourceConfig.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.tidb.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class TidbDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "JDBC-TiDB";
+
+ public static final DataSourcePluginInfo TIDB_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+
+ public static final Set<String> TIDB_SYSTEM_DATABASES =
+ Sets.newHashSet("information_schema", "mysql", "performance_schema", "metrics_schema");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder()
+ .required(TidbOptionRule.URL, TidbOptionRule.DRIVER)
+ .optional(TidbOptionRule.USER, TidbOptionRule.PASSWORD)
+ .build();
+
+ public static final OptionRule METADATA_RULE =
+ OptionRule.builder().required(TidbOptionRule.DATABASE, TidbOptionRule.TABLE).build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java
new file mode 100644
index 00000000..8558f13c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.tidb.jdbc;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
+
+import lombok.NonNull;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class TidbJdbcDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return TidbDataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return TidbDataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ ResultSet resultSet =
+ connection
+ .getMetaData()
+ .getTables(database, null, null, new String[]{"TABLE"})) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ }
+ }
+ return tableNames;
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams);
+ PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;");
+ ResultSet re = statement.executeQuery()) {
+ // filter system databases
+ while (re.next()) {
+ String dbName = re.getString("database");
+ if (StringUtils.isNotBlank(dbName)
+ && !TidbDataSourceConfig.TIDB_SYSTEM_DATABASES.contains(dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new DataSourcePluginException("Get databases failed", e);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (Connection ignored = getConnection(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException("check jdbc connectivity failed", e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams, database)) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, database, table);
+ try (ResultSet resultSet = metaData.getColumns(database, null, table, null)) {
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString()));
+ tableFields.add(tableField);
+ }
+ }
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ return tables.parallelStream()
+ .collect(
+ Collectors.toMap(
+ Function.identity(),
+ table ->
+ getTableFields(
+ pluginName, requestParams, database, table)));
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private Connection getConnection(Map<String, String> requestParams)
+ throws SQLException, ClassNotFoundException {
+ return getConnection(requestParams, null);
+ }
+
+ private Connection getConnection(Map<String, String> requestParams, String databaseName)
+ throws SQLException, ClassNotFoundException {
+ checkNotNull(requestParams.get(TidbOptionRule.DRIVER.key()));
+ checkNotNull(requestParams.get(TidbOptionRule.URL.key()), "Jdbc url cannot be null");
+ String url =
+ JdbcUtils.replaceDatabase(
+ requestParams.get(TidbOptionRule.URL.key()), databaseName);
+ if (requestParams.containsKey(TidbOptionRule.USER.key())) {
+ String username = requestParams.get(TidbOptionRule.USER.key());
+ String password = requestParams.get(TidbOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceFactory.java
new file mode 100644
index 00000000..49906bc9
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.tidb.jdbc;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+
+@Slf4j
+@AutoService(DataSourceFactory.class)
+public class TidbJdbcDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return TidbDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(TidbDataSourceConfig.TIDB_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new TidbJdbcDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbOptionRule.java
new file mode 100644
index 00000000..90af0f95
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbOptionRule.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.tidb.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public class TidbOptionRule {
+
+ public static final Option<String> URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "jdbc url, eg:"
+ + " jdbc:mysql://localhost:3306/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
+
+ public static final Option<String> USER =
+ Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
+
+ public static final Option<String> DATABASE =
+ Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
+
+ public static final Option<String> TABLE =
+ Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
+
+ public static final Option<DriverType> DRIVER =
+ Options.key("driver")
+ .enumType(DriverType.class)
+ .defaultValue(DriverType.MYSQL)
+ .withDescription("driver");
+
+ public enum DriverType {
+ MYSQL("com.mysql.cj.jdbc.Driver"),
+ ;
+ private final String driverClassName;
+
+ DriverType(String driverClassName) {
+ this.driverClassName = driverClassName;
+ }
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ @Override
+ public String toString() {
+ return driverClassName;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml
new file mode 100644
index 00000000..64a90557
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-kafka</artifactId>
+
+ <properties>
+ <kafka.client.version>3.2.0</kafka.client.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.client.version}</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java
new file mode 100644
index 00000000..72e67995
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.kafka;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.DescribeClusterOptions;
+import org.apache.kafka.clients.admin.DescribeClusterResult;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Slf4j
+public class KafkaDataSourceChannel implements DataSourceChannel {
+
+ private static final String DATABASE = "default";
+ private static final DescribeClusterOptions DEFAULT_TIMEOUT_OPTIONS =
+ new DescribeClusterOptions().timeoutMs(60 * 1000);
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return KafkaOptionRule.optionRule();
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return KafkaOptionRule.metadataRule();
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default");
+ try (AdminClient adminClient = createAdminClient(requestParams)) {
+ Set<String> strings = adminClient.listTopics().names().get();
+ return new ArrayList<>(strings);
+ } catch (Exception ex) {
+ throw new DataSourcePluginException(
+ "check kafka connectivity failed, " + ex.getMessage(), ex);
+ }
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ return DEFAULT_DATABASES;
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try (AdminClient adminClient = createAdminClient(requestParams)) {
+ // just test the connection
+ DescribeClusterResult describeClusterResult =
+ adminClient.describeCluster(DEFAULT_TIMEOUT_OPTIONS);
+ return CollectionUtils.isNotEmpty(describeClusterResult.nodes().get());
+ } catch (Exception ex) {
+ throw new DataSourcePluginException(
+ "check kafka connectivity failed, " + ex.getMessage(), ex);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default");
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default");
+ return Collections.emptyMap();
+ }
+
+ private AdminClient createAdminClient(Map<String, String> requestParams) {
+ return AdminClient.create(
+ KafkaRequestParamsUtils.parsePropertiesFromRequestParams(requestParams));
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceFactory.java
new file mode 100644
index 00000000..bcba81c0
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.kafka;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+@AutoService(DataSourceFactory.class)
+public class KafkaDataSourceFactory implements DataSourceFactory {
+
+ public static final String KAFKA_PLUGIN_NAME = "Kafka";
+ public static final String KAFKA_PLUGIN_ICON = "kafka";
+ public static final String KAFKA_PLUGIN_VERSION = "1.0.0";
+
+ @Override
+ public String factoryIdentifier() {
+ return KAFKA_PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Sets.newHashSet(
+ DataSourcePluginInfo.builder()
+ .name(KAFKA_PLUGIN_NAME)
+ .icon(KAFKA_PLUGIN_ICON)
+ .version(KAFKA_PLUGIN_VERSION)
+ .supportVirtualTables(true)
+ .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode())
+ .build());
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new KafkaDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaOptionRule.java
new file mode 100644
index 00000000..b599bb88
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaOptionRule.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.kafka;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+import java.util.Map;
+
+public class KafkaOptionRule {
+
+ public static final Option<String> BOOTSTRAP_SERVERS =
+ Options.key("bootstrap.servers")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Kafka cluster address, separated by \",\".");
+ public static final Option<String> TOPIC =
+ Options.key("topic")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Kafka topic name. If there are multiple topics, use , to split, for example: \"tpc1,tpc2\".");
+
+ public static final Option<Boolean> PATTERN =
+ Options.key("pattern")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "If pattern is set to true,the regular expression for a pattern of topic names to read from."
+ + " All topics in clients with names that match the specified regular expression will be subscribed by the consumer.");
+
+ public static final Option<Map<String, String>> KAFKA_CONFIG =
+ Options.key("kafka.config")
+ .mapType()
+ .noDefaultValue()
+ .withDescription(
+ "{\n"
+ + "client.id=client_1\n"
+ + "max.poll.records=500\n"
+ + "auto.offset.reset=earliest\n"
+ + "enable.auto.commit=false\n"
+ + "}");
+
+ public static OptionRule optionRule() {
+ return OptionRule.builder().required(BOOTSTRAP_SERVERS).optional(KAFKA_CONFIG).build();
+ }
+
+ public static OptionRule metadataRule() {
+ return OptionRule.builder().required(TOPIC).optional(PATTERN).build();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtils.java
new file mode 100644
index 00000000..95988e76
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtils.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.kafka;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+
+import org.apache.kafka.clients.admin.AdminClientConfig;
+
+import java.util.Map;
+import java.util.Properties;
+
+public class KafkaRequestParamsUtils {
+
+ public static Properties parsePropertiesFromRequestParams(Map<String, String> requestParams) {
+ checkArgument(
+ requestParams.containsKey(KafkaOptionRule.BOOTSTRAP_SERVERS.key()),
+ String.format(
+ "Missing %s in requestParams", KafkaOptionRule.BOOTSTRAP_SERVERS.key()));
+ final Properties properties = new Properties();
+ properties.put(
+ AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,
+ requestParams.get(KafkaOptionRule.BOOTSTRAP_SERVERS.key()));
+ if (requestParams.containsKey(KafkaOptionRule.KAFKA_CONFIG.key())) {
+ Config configObject =
+ ConfigFactory.parseString(
+ requestParams.get(KafkaOptionRule.KAFKA_CONFIG.key()));
+ configObject
+ .entrySet()
+ .forEach(
+ entry -> {
+ properties.put(
+ entry.getKey(), entry.getValue().unwrapped().toString());
+ });
+ }
+ return properties;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java
new file mode 100644
index 00000000..e694de45
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.kafka;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import com.google.common.collect.ImmutableMap;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+// todo: use test container to test
+@Slf4j
+@Disabled
+public class KafkaDataSourceChannelTest {
+
+ private static final KafkaDataSourceChannel KAFKA_DATA_SOURCE_CHANNEL =
+ new KafkaDataSourceChannel();
+
+ private static final String KAFKA_PLUGIN_NAME = "kafka";
+ private static final String BOOTSTRAP_SERVER = "localhost:9092";
+
+ private static final Map<String, String> REQUEST_PARAMS =
+ new ImmutableMap.Builder<String, String>()
+ .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), BOOTSTRAP_SERVER)
+ .build();
+
+ @Test
+ public void getDataSourceOptions() {
+ OptionRule dataSourceMetadataFieldsByDataSourceName =
+ KAFKA_DATA_SOURCE_CHANNEL.getDataSourceOptions(KAFKA_PLUGIN_NAME);
+ Assertions.assertEquals(
+ 1, dataSourceMetadataFieldsByDataSourceName.getRequiredOptions().size());
+ }
+
+ @Test
+ public void getDatasourceMetadataFieldsByDataSourceName() {
+ OptionRule datasourceMetadataFieldsByDataSourceName =
+ KAFKA_DATA_SOURCE_CHANNEL.getDatasourceMetadataFieldsByDataSourceName(
+ KAFKA_PLUGIN_NAME);
+ Assertions.assertEquals(
+ 2, datasourceMetadataFieldsByDataSourceName.getOptionalOptions().size());
+ }
+
+ @Test
+ public void getTables() {
+ List<String> tables =
+ KAFKA_DATA_SOURCE_CHANNEL.getTables(KAFKA_PLUGIN_NAME, REQUEST_PARAMS, null);
+ log.info("{}", tables);
+ Assertions.assertNotNull(tables);
+ }
+
+ @Test
+ public void getDatabases() {
+ List<String> databases =
+ KAFKA_DATA_SOURCE_CHANNEL.getDatabases(KAFKA_PLUGIN_NAME, REQUEST_PARAMS);
+ log.info("{}", databases);
+ Assertions.assertNotNull(databases);
+ }
+
+ @Test
+ public void checkDataSourceConnectivity() {
+ boolean dataSourceConnectivity =
+ KAFKA_DATA_SOURCE_CHANNEL.checkDataSourceConnectivity(
+ KAFKA_PLUGIN_NAME, REQUEST_PARAMS);
+ Assertions.assertTrue(dataSourceConnectivity);
+ }
+
+ @Test
+ public void getTableFields() {
+ List<TableField> tableFields =
+ KAFKA_DATA_SOURCE_CHANNEL.getTableFields(KAFKA_PLUGIN_NAME, REQUEST_PARAMS, "", "");
+ log.info("{}", tableFields);
+ Assertions.assertTrue(tableFields.isEmpty());
+ }
+
+ @Test
+ public void testGetTableFields() {
+ Map<String, List<TableField>> tableFields =
+ KAFKA_DATA_SOURCE_CHANNEL.getTableFields(
+ KAFKA_PLUGIN_NAME, REQUEST_PARAMS, "", Collections.emptyList());
+ log.info("{}", tableFields);
+ Assertions.assertTrue(tableFields.isEmpty());
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtilsTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtilsTest.java
new file mode 100644
index 00000000..67d8d33f
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaRequestParamsUtilsTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.kafka;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+class KafkaRequestParamsUtilsTest {
+
+ @Test
+ void parsePropertiesFromRequestParams() {
+ Map<String, String> requestParams =
+ new ImmutableMap.Builder<String, String>()
+ .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), "localhost:9092")
+ .put(
+ KafkaOptionRule.KAFKA_CONFIG.key(),
+ "{" + "security.protocol = SASL_PLAINTEXT" + "}")
+ .build();
+ Properties properties =
+ KafkaRequestParamsUtils.parsePropertiesFromRequestParams(requestParams);
+ Assertions.assertEquals("SASL_PLAINTEXT", properties.getProperty("security.protocol"));
+ }
+
+ @Test
+ void parsePropertiesFromRequestParamsBadCase() {
+ Assertions.assertDoesNotThrow(
+ () ->
+ KafkaRequestParamsUtils.parsePropertiesFromRequestParams(
+ new ImmutableMap.Builder<String, String>()
+ .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), "localhost:9092")
+ .put(KafkaOptionRule.KAFKA_CONFIG.key(), "{}")
+ .build()));
+
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> KafkaRequestParamsUtils.parsePropertiesFromRequestParams(new HashMap<>()));
+
+ Assertions.assertDoesNotThrow(
+ () ->
+ KafkaRequestParamsUtils.parsePropertiesFromRequestParams(
+ new ImmutableMap.Builder<String, String>()
+ .put(KafkaOptionRule.BOOTSTRAP_SERVERS.key(), "localhost:9092")
+ .put(KafkaOptionRule.KAFKA_CONFIG.key(), "")
+ .build()));
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml
new file mode 100644
index 00000000..c7d565f8
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-mysql-cdc</artifactId>
+
+ <properties>
+ <mysql.version>8.0.16</mysql.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.auto.service/auto-service -->
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java
new file mode 100644
index 00000000..07a9f385
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.cdc.mysql;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import com.google.common.collect.Sets;
+import lombok.NonNull;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class MysqlCDCDataSourceChannel implements DataSourceChannel {
+
+ public static final Set<String> MYSQL_SYSTEM_DATABASES =
+ Sets.newHashSet("information_schema", "mysql", "performance_schema", "sys");
+
+ @Override
+ public boolean canAbleGetSchema() {
+ return true;
+ }
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return MysqlCDCOptionRule.optionRule();
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return MysqlCDCOptionRule.metadataRule();
+ }
+
+ @Override
+ public List<String> getTables(
+ String pluginName, Map<String, String> requestParams, String database) {
+ return this.getTableNames(requestParams, database);
+ }
+
+ @Override
+ public List<String> getDatabases(String pluginName, Map<String, String> requestParams) {
+ try {
+ return this.getDataBaseNames(requestParams);
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get databases failed", e);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ String pluginName, Map<String, String> requestParams) {
+ return this.checkJdbcConnectivity(requestParams);
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ String pluginName, Map<String, String> requestParams, String database, String table) {
+ return getTableFields(requestParams, database, table);
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ String pluginName,
+ Map<String, String> requestParams,
+ String database,
+ List<String> tables) {
+ Map<String, List<TableField>> tableFields = new HashMap<>(tables.size());
+ for (String table : tables) {
+ tableFields.put(table, getTableFields(requestParams, database, table));
+ }
+ return tableFields;
+ }
+
+ @SuppressWarnings("checkstyle:MagicNumber")
+ protected boolean checkJdbcConnectivity(Map<String, String> requestParams) {
+ try (Connection connection = init(requestParams);
+ Statement statement = connection.createStatement()) {
+
+ try (ResultSet resultSet = statement.executeQuery("SHOW MASTER STATUS");) {
+ if (resultSet.next()) {
+ String binlogFile = resultSet.getString("File");
+ if (StringUtils.isBlank(binlogFile)) {
+ throw new DataSourcePluginException("binlog must be enabled");
+ }
+ } else {
+ throw new DataSourcePluginException("binlog must be enabled");
+ }
+ }
+
+ try (ResultSet resultSet =
+ statement.executeQuery("SHOW VARIABLES LIKE 'binlog_format'")) {
+ if (resultSet.next()) {
+ String binlogFormat = resultSet.getString("Value");
+ if (!"ROW".equalsIgnoreCase(binlogFormat)) {
+ throw new DataSourcePluginException("binlog_format must be ROW");
+ }
+ } else {
+ throw new DataSourcePluginException("binlog_format must be ROW");
+ }
+ }
+
+ try (ResultSet resultSet =
+ statement.executeQuery("SHOW VARIABLES LIKE 'binlog_row_image'")) {
+ if (resultSet.next()) {
+ String binlogRowImage = resultSet.getString("Value");
+ if (!"FULL".equalsIgnoreCase(binlogRowImage)) {
+ throw new DataSourcePluginException("binlog_row_image must be FULL");
+ }
+ } else {
+ throw new DataSourcePluginException("binlog_row_image must be FULL");
+ }
+ }
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException(
+ "check jdbc connectivity failed, " + e.getMessage(), e);
+ }
+ }
+
+ protected Connection init(Map<String, String> requestParams) throws SQLException {
+ if (null == requestParams.get(MysqlCDCOptionRule.BASE_URL.key())) {
+ throw new DataSourcePluginException("Jdbc url is null");
+ }
+ String url = requestParams.get(MysqlCDCOptionRule.BASE_URL.key());
+ if (null != requestParams.get(MysqlCDCOptionRule.PASSWORD.key())
+ && null != requestParams.get(MysqlCDCOptionRule.USERNAME.key())) {
+ String username = requestParams.get(MysqlCDCOptionRule.USERNAME.key());
+ String password = requestParams.get(MysqlCDCOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+
+ protected List<String> getDataBaseNames(Map<String, String> requestParams) throws SQLException {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = init(requestParams);
+ PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;");
+ ResultSet re = statement.executeQuery()) {
+ // filter system databases
+ while (re.next()) {
+ String dbName = re.getString("database");
+ if (StringUtils.isNotBlank(dbName) && isNotSystemDatabase(dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ }
+ }
+
+ protected List<String> getTableNames(Map<String, String> requestParams, String dbName) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = init(requestParams);
+ ResultSet resultSet =
+ connection
+ .getMetaData()
+ .getTables(dbName, null, null, new String[]{"TABLE"})) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ }
+ }
+ return tableNames;
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ protected List<TableField> getTableFields(
+ Map<String, String> requestParams, String dbName, String tableName) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = init(requestParams);) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, dbName, tableName);
+ ResultSet resultSet = metaData.getColumns(dbName, null, tableName, null);
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ boolean isNullable = convertToBoolean(nullable);
+ tableField.setNullable(isNullable);
+ tableFields.add(tableField);
+ }
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private boolean isNotSystemDatabase(String dbName) {
+ return MYSQL_SYSTEM_DATABASES.stream()
+ .noneMatch(
+ systemDatabase -> StringUtils.equalsAnyIgnoreCase(systemDatabase, dbName));
+ }
+
+ private boolean convertToBoolean(Object value) {
+ if (value instanceof Boolean) {
+ return (Boolean) value;
+ }
+ if (value instanceof String) {
+ return value.equals("TRUE");
+ }
+ return false;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceConfig.java
new file mode 100644
index 00000000..b81d4359
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceConfig.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.cdc.mysql;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+public class MysqlCDCDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "MySQL-CDC";
+
+ public static final DataSourcePluginInfo MYSQL_CDC_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceFactory.java
new file mode 100644
index 00000000..ac4db000
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.cdc.mysql;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+
+import java.util.Collections;
+import java.util.Set;
+
+@AutoService(DataSourceFactory.class)
+public class MysqlCDCDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return "MySQL-CDC";
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Collections.singleton(MysqlCDCDataSourceConfig.MYSQL_CDC_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new MysqlCDCDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCOptionRule.java
new file mode 100644
index 00000000..c0104c25
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCOptionRule.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.cdc.mysql;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+public class MysqlCDCOptionRule {
+
+ public static final Option<String> BASE_URL =
+ Options.key("base-url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "URL has to be without database, like \"jdbc:mysql://localhost:5432/\" or"
+ + "\"jdbc:mysql://localhost:5432\" rather than \"jdbc:mysql://localhost:5432/db\"");
+
+ public static final Option<String> USERNAME =
+ Options.key("username")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Name of the database to use when connecting to the database server.");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Password to use when connecting to the database server.");
+
+ public static final Option<String> DATABASE_NAME =
+ Options.key("database-name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Database name of the database to monitor.");
+
+ public static final Option<String> TABLE_NAME =
+ Options.key("table-name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Table name of the database to monitor.");
+ public static final Option<String> SERVER_TIME_ZONE =
+ Options.key("server-time-zone")
+ .stringType()
+ .defaultValue("UTC")
+ .withDescription("The session time zone in database server.");
+
+ public static OptionRule optionRule() {
+ return OptionRule.builder()
+ .required(USERNAME, PASSWORD, BASE_URL)
+ .optional(SERVER_TIME_ZONE)
+ .build();
+ }
+
+ public static OptionRule metadataRule() {
+ return OptionRule.builder().required(DATABASE_NAME, TABLE_NAME).build();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/pom.xml
new file mode 100644
index 00000000..7bdd419d
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/pom.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-plugins-api</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java
new file mode 100644
index 00000000..dade4457
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.api;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import com.google.common.collect.ImmutableList;
+import lombok.NonNull;
+
+import java.util.List;
+import java.util.Map;
+
+public interface DataSourceChannel {
+
+ List<String> DEFAULT_DATABASES = ImmutableList.of("default");
+
+ /**
+ * get datasource metadata fields by datasource name
+ *
+ * @param pluginName plugin name
+ * @return datasource metadata fields
+ */
+ OptionRule getDataSourceOptions(@NonNull String pluginName);
+
+ /**
+ * get datasource metadata fields by datasource name
+ *
+ * @param pluginName plugin name
+ * @return datasource metadata fields
+ */
+ OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName);
+
+ List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database);
+
+ List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams);
+
+ boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams);
+
+ default boolean canAbleGetSchema() {
+ return false;
+ }
+
+ List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table);
+
+ Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables);
+
+ /**
+ * just check metadata field is right and used by virtual table
+ *
+ * @param requestParams request param(connector params)
+ * @return true if right
+ */
+ default Boolean checkMetadataFieldIsRight(Map<String, String> requestParams) {
+ return true;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceFactory.java
new file mode 100644
index 00000000..966bb033
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.api;
+
+import java.util.Set;
+
+public interface DataSourceFactory {
+
+ String factoryIdentifier();
+
+ Set<DataSourcePluginInfo> supportedDataSources();
+
+ DataSourceChannel createChannel();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginException.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginException.java
new file mode 100644
index 00000000..8913c4a4
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.api;
+
+public class DataSourcePluginException extends RuntimeException {
+
+ public DataSourcePluginException(String message) {
+ super(message);
+ }
+
+ public DataSourcePluginException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public DataSourcePluginException(Throwable cause) {
+ super(cause);
+ }
+
+ public DataSourcePluginException() {
+ super();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginInfo.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginInfo.java
new file mode 100644
index 00000000..d496945d
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourcePluginInfo.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.api;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class DataSourcePluginInfo {
+
+ public String name;
+
+ public String icon;
+
+ public String version;
+
+ /**
+ * @see DatasourcePluginTypeEnum
+ */
+ private Integer type;
+
+ /**
+ * whether support virtual tables, default false
+ */
+ private Boolean supportVirtualTables;
+
+ public DataSourcePluginInfo(
+ String name, String icon, String version, Integer type, Boolean supportVirtualTables) {
+ this.name = checkNotNull(name, "name can not be null");
+ this.icon = checkNotNull(icon, "icon can not be null");
+ this.version = checkNotNull(version, "version can not be null");
+ this.type = checkNotNull(type, "type can not be null");
+ this.supportVirtualTables = supportVirtualTables != null && supportVirtualTables;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DatasourcePluginTypeEnum.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DatasourcePluginTypeEnum.java
new file mode 100644
index 00000000..e5d8e004
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DatasourcePluginTypeEnum.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.api;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+@SuppressWarnings("checkstyle:RegexpSingleline")
+public enum DatasourcePluginTypeEnum {
+ DATABASE(1, "database", "传统数据库"),
+ FILE(2, "file", "文件"),
+ NO_STRUCTURED(3, "no_structured", "非结构化数据（NoSQLs）"),
+ STORAGE(4, "storage", "存储"),
+ REMOTE_CONNECTION(5, "remote_connection", "远程连接");
+
+ private final Integer code;
+
+ private final String name;
+
+ private final String chineseName;
+
+ DatasourcePluginTypeEnum(Integer code, String name, String chineseName) {
+ this.code = checkNotNull(code);
+ this.name = checkNotNull(name);
+ this.chineseName = checkNotNull(chineseName);
+ }
+
+ public Integer getCode() {
+ return code;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getChineseName() {
+ return chineseName;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtils.java
new file mode 100644
index 00000000..2a27a568
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.api.common;
+
+import org.apache.seatunnel.common.utils.JsonUtils;
+
+import java.util.Map;
+
+public class ParamtersUtils {
+ /**
+ * for some parameters, we need to convert them to {@link Map} eg: s3Options {
+ * "access.value": "org.apache.hadoop.fs.s3a.S3AFileSystem", "access.key":
+ * "AKIAIOSFODNN7EXAMPLE", "hadoop_s3_properties": " fs.s3a.impl =
+ * org.apache.hadoop.fs.s3a.S3AFileSystem fs.s3a.access.key = AKIAIOSFODNN7EXAMPLE "
+ *
+ * <p>Convert parameters to {@link Map}
+ *
+ * @param parameters parameters {@link Map}
+ * @return {@link Map}
+ */
+ public static Map<String, Object> convertParams(Map<String, String> parameters) {
+ String json = JsonUtils.toJsonString(parameters);
+ return JsonUtils.toMap(json, String.class, Object.class);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/model/TableField.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/model/TableField.java
new file mode 100644
index 00000000..fabcaf47
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/model/TableField.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.api.model;
+
+import lombok.Data;
+
+import java.util.Map;
+
+@Data
+public class TableField {
+
+ private String type;
+
+ private String name;
+
+ private String comment;
+
+ private Boolean primaryKey;
+
+ private String defaultValue;
+
+ private Boolean nullable;
+
+ private Map<String, String> properties;
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/utils/JdbcUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/utils/JdbcUtils.java
new file mode 100644
index 00000000..8e7cb3d8
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/utils/JdbcUtils.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.api.utils;
+
+public class JdbcUtils {
+
+ public static String replaceDatabase(String jdbcUrl, String databaseName) {
+ if (databaseName == null) {
+ return jdbcUrl;
+ }
+ String[] split = jdbcUrl.split("\\?");
+ if (split.length == 1) {
+ return replaceDatabaseWithoutParameter(jdbcUrl, databaseName);
+ }
+ return replaceDatabaseWithoutParameter(split[0], databaseName) + "?" + split[1];
+ }
+
+ private static String replaceDatabaseWithoutParameter(String jdbcUrl, String databaseName) {
+ int lastIndex = jdbcUrl.lastIndexOf(':');
+ char[] chars = jdbcUrl.toCharArray();
+ for (int i = lastIndex + 1; i < chars.length; i++) {
+ if (chars[i] == '/') {
+ return jdbcUrl.substring(0, i + 1) + databaseName;
+ }
+ }
+ return jdbcUrl + "/" + databaseName;
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/test/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtilsTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/test/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtilsTest.java
new file mode 100644
index 00000000..98b8d7ee
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/test/java/org/apache/seatunnel/datasource/plugin/api/common/ParamtersUtilsTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.api.common;
+
+import org.apache.seatunnel.common.utils.JsonUtils;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ParamtersUtilsTest {
+
+ @Test
+ public void testConvertParams() {
+ Map<String, Object> s3Options = new HashMap<>();
+ s3Options.put("access.key", "myaccess");
+ s3Options.put("access.value", "myvalue");
+ Map<String, String> hadoopConfig = new HashMap<>();
+ hadoopConfig.put("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
+ s3Options.put("hadoopConfig", hadoopConfig);
+ String s3OptionsJson = JsonUtils.toJsonString(s3Options);
+ Map<String, String> s3OptionsMap =
+ JsonUtils.toMap(s3OptionsJson, String.class, String.class);
+
+ Map<String, Object> s3OptionsMapConvertResult = ParamtersUtils.convertParams(s3OptionsMap);
+ Assertions.assertEquals(s3OptionsMapConvertResult.get("hadoopConfig"), hadoopConfig);
+ Assertions.assertEquals(
+ s3OptionsMapConvertResult.get("access.key"), s3Options.get("access.key"));
+ Assertions.assertEquals(
+ s3OptionsMapConvertResult.get("access.value"), s3Options.get("access.value"));
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/pom.xml
new file mode 100644
index 00000000..3f7e0e6e
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/pom.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-s3-redshift</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-hadoop3-3.1.4-uber</artifactId>
+ <version>${seatunnel-framework.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.amazon.redshift</groupId>
+ <artifactId>redshift-jdbc42</artifactId>
+ <version>${redshift.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop-aws.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-bundle</artifactId>
+ <version>${aws-java-sdk-bundle.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java
new file mode 100644
index 00000000..f3256186
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.redshift.s3;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.Arrays;
+import java.util.Map;
+
+@Slf4j
+public class HadoopS3AConfiguration {
+
+ /* S3 constants */
+ private static final String S3A_SCHEMA = "s3a";
+ private static final String HDFS_S3N_IMPL = "org.apache.hadoop.fs.s3native.NativeS3FileSystem";
+ private static final String HDFS_S3A_IMPL = "org.apache.hadoop.fs.s3a.S3AFileSystem";
+ private static final String S3A_PROTOCOL = "s3a";
+ private static final String DEFAULT_PROTOCOL = "s3n";
+ private static final String S3_FORMAT_KEY = "fs.%s.%s";
+ private static final String HDFS_IMPL_KEY = "impl";
+
+ public static Configuration getConfiguration(Map<String, String> s3Options) {
+
+ if (!s3Options.containsKey(S3RedshiftOptionRule.BUCKET.key())) {
+ throw new IllegalArgumentException(
+ "S3Redshift datasource bucket is null, please check your config");
+ }
+ if (!s3Options.containsKey(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())) {
+ throw new IllegalArgumentException(
+ "S3Redshift datasource endpoint is null, please check your config");
+ }
+ String bucket = s3Options.get(S3RedshiftOptionRule.BUCKET.key());
+
+ String protocol = DEFAULT_PROTOCOL;
+ if (bucket.startsWith(S3A_PROTOCOL)) {
+ protocol = S3A_PROTOCOL;
+ }
+ String fsImpl = protocol.equals(S3A_PROTOCOL) ? HDFS_S3A_IMPL : HDFS_S3N_IMPL;
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set(FS_DEFAULT_NAME_KEY, bucket);
+ hadoopConf.set(
+ S3RedshiftOptionRule.FS_S3A_ENDPOINT.key(),
+ s3Options.get(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key()));
+ hadoopConf.set(formatKey(protocol, HDFS_IMPL_KEY), fsImpl);
+ if (s3Options.containsKey(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key())) {
+ Arrays.stream(
+ s3Options
+ .get(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key())
+ .split("\n"))
+ .map(String::trim)
+ .filter(StringUtils::isNotBlank)
+ .forEach(
+ line -> {
+ String[] kv = line.split("=");
+ if (kv.length == 2) {
+ hadoopConf.set(kv[0].trim(), kv[1].trim());
+ }
+ });
+ }
+ if (S3RedshiftOptionRule.S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider
+ .getProvider()
+ .equals(s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) {
+ hadoopConf.set(
+ S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(),
+ s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()));
+ hadoopConf.set(
+ "fs.s3a.access.key", s3Options.get(S3RedshiftOptionRule.ACCESS_KEY.key()));
+ hadoopConf.set(
+ "fs.s3a.secret.key", s3Options.get(S3RedshiftOptionRule.SECRET_KEY.key()));
+ } else {
+ hadoopConf.set(
+ S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(),
+ s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()));
+ }
+ return hadoopConf;
+ }
+
+ private static String formatKey(String protocol, String key) {
+ return String.format(S3_FORMAT_KEY, protocol, key);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java
new file mode 100644
index 00000000..07aceb32
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.redshift.s3;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
+
+import com.google.common.collect.Sets;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Slf4j
+public class S3RedshiftDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return S3RedshiftOptionRule.optionRule();
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return S3RedshiftOptionRule.metadataRule();
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ return getTableNames(requestParams, database);
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ try {
+ return getDataBaseNames(pluginName, requestParams);
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("Query redshift databases failed", e);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ checkHdfsS3Connection(requestParams);
+ checkJdbcConnection(requestParams);
+ return true;
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ return getTableFields(requestParams, database, table);
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ // not need this method
+ return null;
+ }
+
+ private void checkJdbcConnection(Map<String, String> requestParams) {
+ String jdbcUrl = requestParams.get(S3RedshiftOptionRule.JDBC_URL.key());
+ String username = requestParams.get(S3RedshiftOptionRule.JDBC_USER.key());
+ String password = requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key());
+ if (StringUtils.isBlank(jdbcUrl)) {
+ throw new DataSourcePluginException("Redshift Jdbc url is empty");
+ }
+ if (StringUtils.isBlank(username) && StringUtils.isBlank(password)) {
+ try (Connection ignored = DriverManager.getConnection(jdbcUrl)) {
+ log.info("Redshift jdbc connection is valid");
+ return;
+ } catch (SQLException e) {
+ throw new DataSourcePluginException(
+ "Check Redshift jdbc connection failed,please check your config", e);
+ }
+ }
+ try (Connection ignored = DriverManager.getConnection(jdbcUrl, username, password)) {
+ log.info("Redshift jdbc connection is valid");
+ } catch (SQLException e) {
+ throw new DataSourcePluginException(
+ "Check Redshift jdbc connection failed,please check your config", e);
+ }
+ }
+
+ private void checkHdfsS3Connection(Map<String, String> requestParams) {
+ Configuration s3Conf = HadoopS3AConfiguration.getConfiguration(requestParams);
+ try (FileSystem fs = FileSystem.get(s3Conf)) {
+ fs.getFileStatus(new org.apache.hadoop.fs.Path("/"));
+ } catch (IOException e) {
+ throw new DataSourcePluginException(
+ "S3 configuration is invalid, please check your config", e);
+ }
+ }
+
+ protected Connection init(Map<String, String> requestParams, String databaseName)
+ throws SQLException {
+ if (null == requestParams.get(S3RedshiftOptionRule.JDBC_URL.key())) {
+ throw new DataSourcePluginException("Jdbc url is null");
+ }
+ String url =
+ JdbcUtils.replaceDatabase(
+ requestParams.get(S3RedshiftOptionRule.JDBC_URL.key()), databaseName);
+ if (null != requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key())
+ && null != requestParams.get(S3RedshiftOptionRule.JDBC_USER.key())) {
+ String username = requestParams.get(S3RedshiftOptionRule.JDBC_USER.key());
+ String password = requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+
+ protected List<String> getDataBaseNames(String pluginName, Map<String, String> requestParams)
+ throws SQLException {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = init(requestParams, null);
+ PreparedStatement statement =
+ connection.prepareStatement("select datname from pg_database;");
+ ResultSet re = statement.executeQuery()) {
+ while (re.next()) {
+ String dbName = re.getString("datname");
+ if (StringUtils.isNotBlank(dbName) && isNotSystemDatabase(dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+ return dbNames;
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get databases failed", e);
+ }
+ }
+
+ protected List<String> getTableNames(Map<String, String> requestParams, String dbName) {
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = init(requestParams, dbName);) {
+ ResultSet resultSet =
+ connection.getMetaData().getTables(dbName, null, null, new String[]{"TABLE"});
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ }
+ }
+ return tableNames;
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ protected List<TableField> getTableFields(
+ Map<String, String> requestParams, String dbName, String tableName) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = init(requestParams, dbName);) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, dbName, tableName);
+ String[] split = tableName.split("\\.");
+ if (split.length != 2) {
+ throw new DataSourcePluginException(
+ "Postgresql tableName should composed by schemaName.tableName");
+ }
+ ResultSet resultSet = metaData.getColumns(dbName, split[0], split[1], null);
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ boolean isNullable = convertToBoolean(nullable);
+ tableField.setNullable(isNullable);
+ tableFields.add(tableField);
+ }
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ @SuppressWarnings("checkstyle:MagicNumber")
+ private static boolean checkHostConnectable(String host, int port) {
+ try (Socket socket = new Socket()) {
+ socket.connect(new InetSocketAddress(host, port), 1000);
+ return true;
+ } catch (IOException e) {
+
+ throw new DataSourcePluginException("check host connectable failed", e);
+ }
+ }
+
+ private boolean isNotSystemDatabase(String dbName) {
+ return !POSTGRESQL_SYSTEM_DATABASES.contains(dbName.toLowerCase());
+ }
+
+ private boolean convertToBoolean(Object value) {
+ if (value instanceof Boolean) {
+ return (Boolean) value;
+ }
+ if (value instanceof String) {
+ return value.equals("TRUE");
+ }
+ return false;
+ }
+
+ public static final Set<String> POSTGRESQL_SYSTEM_DATABASES =
+ Sets.newHashSet(
+ "information_schema",
+ "pg_catalog",
+ "root",
+ "pg_toast",
+ "pg_temp_1",
+ "pg_toast_temp_1",
+ "postgres",
+ "template0",
+ "template1");
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java
new file mode 100644
index 00000000..3ef212ac
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.redshift.s3;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+@AutoService(DataSourceFactory.class)
+public class S3RedshiftDataSourceFactory implements DataSourceFactory {
+ @Override
+ public String factoryIdentifier() {
+ return "S3-Redshift";
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ DataSourcePluginInfo s3DatasourcePluginInfo =
+ DataSourcePluginInfo.builder()
+ .name("S3-Redshift")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .version("1.0.0")
+ .supportVirtualTables(false)
+ .icon("S3-Redshift")
+ .icon("S3-Redshift")
+ .build();
+
+ return Sets.newHashSet(s3DatasourcePluginInfo);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new S3RedshiftDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java
new file mode 100644
index 00000000..5d69ee80
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.redshift.s3;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+import java.util.Arrays;
+import java.util.Map;
+
+public class S3RedshiftOptionRule {
+
+ public static final Option<String> JDBC_URL =
+ Options.key("jdbc_url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Redshift jdbc connection url, eg: jdbc:redshift://localhost:5439/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
+
+ public static final Option<String> JDBC_USER =
+ Options.key("jdbc_user")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Redshift jdbc connection user");
+
+ public static final Option<String> JDBC_PASSWORD =
+ Options.key("jdbc_password")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Redshift jdbc connection password");
+
+ public static final Option<String> ACCESS_KEY =
+ Options.key("access_key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 access key");
+
+ public static final Option<String> SECRET_KEY =
+ Options.key("secret_key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 secret key");
+
+ public static final Option<String> BUCKET =
+ Options.key("bucket").stringType().noDefaultValue().withDescription("S3 bucket name");
+
+ public static final Option<String> FS_S3A_ENDPOINT =
+ Options.key("fs.s3a.endpoint")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("fs s3a endpoint");
+
+ public static final Option<S3aAwsCredentialsProvider> S3A_AWS_CREDENTIALS_PROVIDER =
+ Options.key("fs.s3a.aws.credentials.provider")
+ .enumType(S3aAwsCredentialsProvider.class)
+ .defaultValue(S3aAwsCredentialsProvider.InstanceProfileCredentialsProvider)
+ .withDescription("s3a aws credentials provider");
+
+ public static final Option<Map<String, String>> HADOOP_S3_PROPERTIES =
+ Options.key("hadoop_s3_properties")
+ .mapType()
+ .noDefaultValue()
+ .withDescription(
+ "{"
+ + "fs.s3a.buffer.dir = /data/st_test/s3a\n"
+ + "fs.s3a.fast.upload.buffer = disk\n"
+ + "}");
+
+ public static OptionRule optionRule() {
+ return OptionRule.builder()
+ .required(
+ JDBC_URL,
+ BUCKET,
+ FS_S3A_ENDPOINT,
+ S3A_AWS_CREDENTIALS_PROVIDER,
+ JDBC_USER,
+ JDBC_PASSWORD)
+ .optional(HADOOP_S3_PROPERTIES)
+ .conditional(
+ S3A_AWS_CREDENTIALS_PROVIDER,
+ S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider,
+ ACCESS_KEY,
+ SECRET_KEY)
+ .build();
+ }
+
+ public static final Option<String> PATH =
+ Options.key("path").stringType().noDefaultValue().withDescription("S3 write path");
+
+ public static final Option<FileFormat> TYPE =
+ Options.key("file_format_type")
+ .enumType(FileFormat.class)
+ .noDefaultValue()
+ .withDescription("S3 write type");
+
+ public static final Option<String> DELIMITER =
+ Options.key("delimiter")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 write delimiter");
+
+ public static final Option<Map<String, String>> SCHEMA =
+ Options.key("schema").mapType().noDefaultValue().withDescription("SeaTunnel Schema");
+
+ public static final Option<Boolean> PARSE_PARSE_PARTITION_FROM_PATH =
+ Options.key("parse_partition_from_path")
+ .booleanType()
+ .noDefaultValue()
+ .withDescription("S3 write parse_partition_from_path");
+
+ public static final Option<String> DATE_FORMAT =
+ Options.key("date_format")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 write date_format");
+
+ public static final Option<String> DATETIME_FORMAT =
+ Options.key("time_format")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 write time_format");
+
+ public static final Option<String> TIME_FORMAT =
+ Options.key("datetime_format")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 write datetime_format");
+
+ public static OptionRule metadataRule() {
+ return OptionRule.builder()
+ .required(PATH, TYPE)
+ .conditional(TYPE, FileFormat.TEXT, DELIMITER)
+ .conditional(TYPE, Arrays.asList(FileFormat.TEXT, FileFormat.JSON), SCHEMA)
+ .optional(PARSE_PARSE_PARTITION_FROM_PATH)
+ .optional(DATE_FORMAT)
+ .optional(DATETIME_FORMAT)
+ .optional(TIME_FORMAT)
+ .build();
+ }
+
+ public enum S3aAwsCredentialsProvider {
+ SimpleAWSCredentialsProvider("org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"),
+
+ InstanceProfileCredentialsProvider("com.amazonaws.auth.InstanceProfileCredentialsProvider");
+
+ private String provider;
+
+ S3aAwsCredentialsProvider(String provider) {
+ this.provider = provider;
+ }
+
+ public String getProvider() {
+ return provider;
+ }
+
+ @Override
+ public String toString() {
+ return provider;
+ }
+ }
+
+ public enum FileFormat {
+ CSV("csv"),
+ TEXT("txt"),
+ PARQUET("parquet"),
+ ORC("orc"),
+ JSON("json");
+
+ private final String type;
+
+ FileFormat(String type) {
+ this.type = type;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/pom.xml
new file mode 100644
index 00000000..be117f58
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/pom.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-s3</artifactId>
+
+ <properties>
+ <hadoop-uber.version>2.4.4-WS-SNAPSHOT</hadoop-uber.version>
+ <hadoop-aws.version>3.1.4</hadoop-aws.version>
+ <aws-java-sdk-bundle.version>1.11.271</aws-java-sdk-bundle.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-hadoop3-3.1.4-uber</artifactId>
+ <version>${hadoop-uber.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop-aws.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-bundle</artifactId>
+ <version>${aws-java-sdk-bundle.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/HadoopS3AConfiguration.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/HadoopS3AConfiguration.java
new file mode 100644
index 00000000..ecce9793
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/HadoopS3AConfiguration.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.s3;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.Map;
+
+@Slf4j
+public class HadoopS3AConfiguration {
+
+ /* S3 constants */
+ private static final String S3A_SCHEMA = "s3a";
+ private static final String HDFS_S3N_IMPL = "org.apache.hadoop.fs.s3native.NativeS3FileSystem";
+ private static final String HDFS_S3A_IMPL = "org.apache.hadoop.fs.s3a.S3AFileSystem";
+ private static final String S3A_PROTOCOL = "s3a";
+ private static final String DEFAULT_PROTOCOL = "s3n";
+ private static final String S3_FORMAT_KEY = "fs.%s.%s";
+ private static final String HDFS_IMPL_KEY = "impl";
+
+ public static Configuration getConfiguration(Map<String, String> s3Options) {
+
+ if (!s3Options.containsKey(S3OptionRule.BUCKET.key())) {
+ throw new IllegalArgumentException(
+ "S3 datasource bucket is null, please check your config");
+ }
+ if (!s3Options.containsKey(S3OptionRule.FS_S3A_ENDPOINT.key())) {
+ throw new IllegalArgumentException(
+ "S3 datasource endpoint is null, please check your config");
+ }
+ String bucket = s3Options.get(S3OptionRule.BUCKET.key());
+
+ String protocol = DEFAULT_PROTOCOL;
+ if (bucket.startsWith(S3A_PROTOCOL)) {
+ protocol = S3A_PROTOCOL;
+ }
+ String fsImpl = protocol.equals(S3A_PROTOCOL) ? HDFS_S3A_IMPL : HDFS_S3N_IMPL;
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set(FS_DEFAULT_NAME_KEY, bucket);
+ hadoopConf.set(
+ S3OptionRule.FS_S3A_ENDPOINT.key(),
+ s3Options.get(S3OptionRule.FS_S3A_ENDPOINT.key()));
+ hadoopConf.set(formatKey(protocol, HDFS_IMPL_KEY), fsImpl);
+ if (s3Options.containsKey(S3OptionRule.HADOOP_S3_PROPERTIES.key())) {
+ Config configObject =
+ ConfigFactory.parseString(
+ s3Options.get(S3OptionRule.HADOOP_S3_PROPERTIES.key()));
+ configObject
+ .entrySet()
+ .forEach(
+ entry -> {
+ hadoopConf.set(
+ entry.getKey(), entry.getValue().unwrapped().toString());
+ });
+ }
+ if (S3OptionRule.S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider.getProvider()
+ .equals(s3Options.get(S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) {
+ hadoopConf.set(
+ S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(),
+ s3Options.get(S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()));
+ hadoopConf.set("fs.s3a.access.key", s3Options.get(S3OptionRule.ACCESS_KEY.key()));
+ hadoopConf.set("fs.s3a.secret.key", s3Options.get(S3OptionRule.SECRET_KEY.key()));
+ } else {
+ hadoopConf.set(
+ S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(),
+ s3Options.get(S3OptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()));
+ }
+ return hadoopConf;
+ }
+
+ private static String formatKey(String protocol, String key) {
+ return String.format(S3_FORMAT_KEY, protocol, key);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DataSourceFactory.java
new file mode 100644
index 00000000..640b6f16
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DataSourceFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.s3;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+@AutoService(DataSourceFactory.class)
+public class S3DataSourceFactory implements DataSourceFactory {
+
+ private static final String PLUGIN_NAME = "S3";
+
+ @Override
+ public String factoryIdentifier() {
+ return PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ DataSourcePluginInfo s3DatasourcePluginInfo =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .type(DatasourcePluginTypeEnum.FILE.getCode())
+ .version("1.0.0")
+ .supportVirtualTables(false)
+ .icon("S3File")
+ .build();
+
+ return Sets.newHashSet(s3DatasourcePluginInfo);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new S3DatasourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java
new file mode 100644
index 00000000..e1ba8de5
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.s3;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import lombok.NonNull;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class S3DatasourceChannel implements DataSourceChannel {
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return S3OptionRule.optionRule();
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return S3OptionRule.metadataRule();
+ }
+
+ @Override
+ public List<String> getTables(
+ @NonNull String pluginName, Map<String, String> requestParams, String database) {
+ throw new UnsupportedOperationException("getTables is not supported for S3 datasource");
+ }
+
+ @Override
+ public List<String> getDatabases(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ throw new UnsupportedOperationException("getDatabases is not supported for S3 datasource");
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map<String, String> requestParams) {
+ Configuration conf = HadoopS3AConfiguration.getConfiguration(requestParams);
+ try (FileSystem fs = FileSystem.get(conf)) {
+ fs.listStatus(new Path("/"));
+ return true;
+ } catch (IOException e) {
+ throw new DataSourcePluginException(
+ String.format("check s3 connectivity failed, config is: %s", requestParams), e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ throw new UnsupportedOperationException(
+ "getTableFields is not supported for S3 datasource");
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map<String, String> requestParams,
+ @NonNull String database,
+ @NonNull List<String> tables) {
+ throw new UnsupportedOperationException(
+ "getTableFields is not supported for S3 datasource");
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3OptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3OptionRule.java
new file mode 100644
index 00000000..ba56f303
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3OptionRule.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.s3;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+import java.util.Arrays;
+import java.util.Map;
+
+public class S3OptionRule {
+
+ public static final Option<String> ACCESS_KEY =
+ Options.key("access_key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 access key");
+
+ public static final Option<String> SECRET_KEY =
+ Options.key("secret_key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 secret key");
+
+ public static final Option<String> BUCKET =
+ Options.key("bucket").stringType().noDefaultValue().withDescription("S3 bucket name");
+
+ public static final Option<String> FS_S3A_ENDPOINT =
+ Options.key("fs.s3a.endpoint")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("fs s3a endpoint");
+
+ public static final Option<S3aAwsCredentialsProvider> S3A_AWS_CREDENTIALS_PROVIDER =
+ Options.key("fs.s3a.aws.credentials.provider")
+ .enumType(S3aAwsCredentialsProvider.class)
+ .defaultValue(S3aAwsCredentialsProvider.InstanceProfileCredentialsProvider)
+ .withDescription("s3a aws credentials provider");
+
+ public static final Option<Map<String, String>> HADOOP_S3_PROPERTIES =
+ Options.key("hadoop_s3_properties")
+ .mapType()
+ .noDefaultValue()
+ .withDescription(
+ "{\n"
+ + "fs.s3a.buffer.dir=/data/st_test/s3a\n"
+ + "fs.s3a.fast.upload.buffer=disk\n"
+ + "}");
+
+ public static OptionRule optionRule() {
+ return OptionRule.builder()
+ .required(BUCKET, FS_S3A_ENDPOINT, S3A_AWS_CREDENTIALS_PROVIDER)
+ .optional(HADOOP_S3_PROPERTIES)
+ .conditional(
+ S3A_AWS_CREDENTIALS_PROVIDER,
+ S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider,
+ ACCESS_KEY,
+ SECRET_KEY)
+ .build();
+ }
+
+ public static final Option<String> PATH =
+ Options.key("path").stringType().noDefaultValue().withDescription("S3 write path");
+
+ public static final Option<FileFormat> TYPE =
+ Options.key("file_format_type")
+ .enumType(FileFormat.class)
+ .noDefaultValue()
+ .withDescription("S3 write type");
+
+ public static final Option<String> DELIMITER =
+ Options.key("delimiter")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 write delimiter");
+
+ public static final Option<Map<String, String>> SCHEMA =
+ Options.key("schema").mapType().noDefaultValue().withDescription("SeaTunnel Schema");
+
+ public static final Option<Boolean> PARSE_PARSE_PARTITION_FROM_PATH =
+ Options.key("parse_partition_from_path")
+ .booleanType()
+ .noDefaultValue()
+ .withDescription("S3 write parse_partition_from_path");
+
+ public static final Option<String> DATE_FORMAT =
+ Options.key("date_format")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 write date_format");
+
+ public static final Option<String> DATETIME_FORMAT =
+ Options.key("time_format")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 write time_format");
+
+ public static final Option<String> TIME_FORMAT =
+ Options.key("datetime_format")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("S3 write datetime_format");
+
+ public static OptionRule metadataRule() {
+ return OptionRule.builder()
+ .required(PATH, TYPE)
+ .conditional(TYPE, FileFormat.TEXT, DELIMITER)
+ .conditional(TYPE, Arrays.asList(FileFormat.TEXT, FileFormat.JSON), SCHEMA)
+ .optional(PARSE_PARSE_PARTITION_FROM_PATH)
+ .optional(DATE_FORMAT)
+ .optional(DATETIME_FORMAT)
+ .optional(TIME_FORMAT)
+ .build();
+ }
+
+ public enum S3aAwsCredentialsProvider {
+ SimpleAWSCredentialsProvider("org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"),
+
+ InstanceProfileCredentialsProvider("com.amazonaws.auth.InstanceProfileCredentialsProvider");
+
+ private String provider;
+
+ S3aAwsCredentialsProvider(String provider) {
+ this.provider = provider;
+ }
+
+ public String getProvider() {
+ return provider;
+ }
+
+ @Override
+ public String toString() {
+ return provider;
+ }
+ }
+
+ public enum FileFormat {
+ CSV("csv"),
+ TEXT("txt"),
+ PARQUET("parquet"),
+ ORC("orc"),
+ JSON("json");
+
+ private final String type;
+
+ FileFormat(String type) {
+ this.type = type;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/pom.xml
new file mode 100644
index 00000000..467b7914
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/pom.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-datasource-plugins</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>datasource-sqlserver-cdc</artifactId>
+
+ <properties>
+ <sqlserver.version>9.2.1.jre8</sqlserver.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>datasource-plugins-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.microsoft.sqlserver</groupId>
+ <artifactId>mssql-jdbc</artifactId>
+ <version>${sqlserver.version}</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java
new file mode 100644
index 00000000..4fed9948
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.cdc.sqlserver;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import com.google.common.collect.Sets;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Slf4j
+public class SqlServerCDCDataSourceChannel implements DataSourceChannel {
+
+ public static final Set<String> MYSQL_SYSTEM_DATABASES =
+ Sets.newHashSet("master", "tempdb", "model", "msdb");
+
+ @Override
+ public boolean canAbleGetSchema() {
+ return true;
+ }
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return SqlServerCDCOptionRule.optionRule();
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return SqlServerCDCOptionRule.metadataRule();
+ }
+
+ @Override
+ public List<String> getTables(
+ String pluginName, Map<String, String> requestParams, String database) {
+ return this.getTableNames(requestParams, database);
+ }
+
+ @Override
+ public List<String> getDatabases(String pluginName, Map<String, String> requestParams) {
+ try {
+ return this.getDataBaseNames(requestParams);
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get databases failed", e);
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ String pluginName, Map<String, String> requestParams) {
+ try (Connection connection = init(requestParams);
+ PreparedStatement statement = connection.prepareStatement("SELECT 1");
+ ResultSet rs = statement.executeQuery()) {
+ return rs.next();
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("connect datasource failed", e);
+ }
+ }
+
+ @Override
+ public List<TableField> getTableFields(
+ String pluginName, Map<String, String> requestParams, String database, String table) {
+ Pair<String, String> pair = parseSchemaAndTable(table);
+ return getTableFields(requestParams, database, pair.getLeft(), pair.getRight());
+ }
+
+ @Override
+ public Map<String, List<TableField>> getTableFields(
+ String pluginName,
+ Map<String, String> requestParams,
+ String database,
+ List<String> tables) {
+ Map<String, List<TableField>> tableFields = new HashMap<>(tables.size());
+ for (String table : tables) {
+ tableFields.put(table, getTableFields(pluginName, requestParams, database, table));
+ }
+ return tableFields;
+ }
+
+ private Connection init(Map<String, String> requestParams) throws SQLException {
+ if (null == requestParams.get(SqlServerCDCOptionRule.BASE_URL.key())) {
+ throw new DataSourcePluginException("Jdbc url is null");
+ }
+ String url = requestParams.get(SqlServerCDCOptionRule.BASE_URL.key());
+ if (null != requestParams.get(SqlServerCDCOptionRule.PASSWORD.key())
+ && null != requestParams.get(SqlServerCDCOptionRule.USERNAME.key())) {
+ String username = requestParams.get(SqlServerCDCOptionRule.USERNAME.key());
+ String password = requestParams.get(SqlServerCDCOptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+
+ private List<String> getDataBaseNames(Map<String, String> requestParams) throws SQLException {
+ List<String> dbNames = new ArrayList<>();
+ try (Connection connection = init(requestParams);
+ PreparedStatement statement =
+ connection.prepareStatement(
+ "SELECT NAME FROM SYS.DATABASES WHERE IS_CDC_ENABLED = 1;");
+ ResultSet re = statement.executeQuery()) {
+ // filter system databases
+ while (re.next()) {
+ String dbName = re.getString("NAME");
+ if (StringUtils.isNotBlank(dbName) && isNotSystemDatabase(dbName)) {
+ dbNames.add(dbName);
+ }
+ }
+
+ return dbNames;
+ }
+ }
+
+ private List<String> getTableNames(Map<String, String> requestParams, String dbName) {
+ final String sql =
+ String.format(
+ "SELECT SCHEMAS.NAME AS SCHEMA_NAME, TABLES.NAME AS TABLE_NAME"
+ + " FROM %s.SYS.SCHEMAS AS SCHEMAS"
+ + " JOIN %s.SYS.TABLES AS TABLES"
+ + " ON SCHEMAS.SCHEMA_ID = TABLES.SCHEMA_ID"
+ + " AND TABLES.IS_TRACKED_BY_CDC = 1",
+ dbName, dbName);
+
+ List<String> tableNames = new ArrayList<>();
+ try (Connection connection = init(requestParams);
+ Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sql)) {
+ while (resultSet.next()) {
+ String schemaName = resultSet.getString("SCHEMA_NAME");
+ String tableName = resultSet.getString("TABLE_NAME");
+ tableNames.add(schemaName + "." + tableName);
+ }
+ return tableNames;
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ private List<TableField> getTableFields(
+ Map<String, String> requestParams, String dbName, String schemaName, String tableName) {
+ List<TableField> tableFields = new ArrayList<>();
+ try (Connection connection = init(requestParams);) {
+ DatabaseMetaData metaData = connection.getMetaData();
+ String primaryKey = getPrimaryKey(metaData, dbName, schemaName, tableName);
+ ResultSet resultSet = metaData.getColumns(dbName, schemaName, tableName, null);
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+ tableField.setPrimaryKey(false);
+ if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
+ tableField.setPrimaryKey(true);
+ }
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+ Object nullable = resultSet.getObject("IS_NULLABLE");
+ boolean isNullable = convertToBoolean(nullable);
+ tableField.setNullable(isNullable);
+ tableFields.add(tableField);
+ }
+ } catch (SQLException e) {
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ return tableFields;
+ }
+
+ private String getPrimaryKey(
+ DatabaseMetaData metaData, String dbName, String schemaName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, schemaName, tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private boolean isNotSystemDatabase(String dbName) {
+ return MYSQL_SYSTEM_DATABASES.stream()
+ .noneMatch(
+ systemDatabase -> StringUtils.equalsAnyIgnoreCase(systemDatabase, dbName));
+ }
+
+ private boolean convertToBoolean(Object value) {
+ if (value instanceof Boolean) {
+ return (Boolean) value;
+ }
+ if (value instanceof String) {
+ return value.equals("TRUE");
+ }
+ return false;
+ }
+
+ private Pair<String, String> parseSchemaAndTable(String tableName) {
+ String[] schemaAndTable = tableName.split("\\.");
+ if (schemaAndTable.length != 2) {
+ throw new DataSourcePluginException("table name is invalid");
+ }
+ return Pair.of(schemaAndTable[0], schemaAndTable[1]);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceConfig.java
new file mode 100644
index 00000000..4da5b9d5
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceConfig.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.cdc.sqlserver;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+public class SqlServerCDCDataSourceConfig {
+
+ public static final String PLUGIN_NAME = "SqlServer-CDC";
+
+ public static final DataSourcePluginInfo SQLSERVER_CDC_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceFactory.java
new file mode 100644
index 00000000..321f9c8c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.cdc.sqlserver;
+
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+
+import com.google.auto.service.AutoService;
+
+import java.util.Collections;
+import java.util.Set;
+
+@AutoService(DataSourceFactory.class)
+public class SqlServerCDCDataSourceFactory implements DataSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return SqlServerCDCDataSourceConfig.PLUGIN_NAME;
+ }
+
+ @Override
+ public Set<DataSourcePluginInfo> supportedDataSources() {
+ return Collections.singleton(
+ SqlServerCDCDataSourceConfig.SQLSERVER_CDC_DATASOURCE_PLUGIN_INFO);
+ }
+
+ @Override
+ public DataSourceChannel createChannel() {
+ return new SqlServerCDCDataSourceChannel();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCOptionRule.java
new file mode 100644
index 00000000..95cf360b
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCOptionRule.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.cdc.sqlserver;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+public class SqlServerCDCOptionRule {
+
+ public static final Option<String> BASE_URL =
+ Options.key("base-url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "URL has to be without database, like \"jdbc:sqlserver://localhost:1433;databaseName=test/\"");
+
+ public static final Option<String> USERNAME =
+ Options.key("username")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Name of the database to use when connecting to the database server.");
+
+ public static final Option<String> PASSWORD =
+ Options.key("password")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Password to use when connecting to the database server.");
+
+ public static final Option<String> DATABASE_NAME =
+ Options.key("database-name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Database name of the database to monitor.");
+
+ public static final Option<String> TABLE_NAME =
+ Options.key("table-name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Table name of the database to monitor.");
+ public static final Option<String> SERVER_TIME_ZONE =
+ Options.key("server-time-zone")
+ .stringType()
+ .defaultValue("UTC")
+ .withDescription("The session time zone in database server.");
+
+ public static OptionRule optionRule() {
+ return OptionRule.builder()
+ .required(USERNAME, PASSWORD, BASE_URL)
+ .optional(SERVER_TIME_ZONE)
+ .build();
+ }
+
+ public static OptionRule metadataRule() {
+ return OptionRule.builder().required(DATABASE_NAME, TABLE_NAME).build();
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/test/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/test/TestSqlServerCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/test/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/test/TestSqlServerCDCDataSourceChannel.java
new file mode 100644
index 00000000..b815a318
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/test/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/test/TestSqlServerCDCDataSourceChannel.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.cdc.sqlserver.test;
+
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+import org.apache.seatunnel.datasource.plugin.cdc.sqlserver.SqlServerCDCDataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.cdc.sqlserver.SqlServerCDCDataSourceConfig;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+public class TestSqlServerCDCDataSourceChannel {
+
+ @Test
+ @Disabled
+ public void testConnect() {
+ SqlServerCDCDataSourceChannel channel = new SqlServerCDCDataSourceChannel();
+ Map<String, String> requestParams = new TreeMap<>();
+ requestParams.put("base-url", "jdbc:sqlserver://localhost:1433;databaseName=test");
... 2902 lines suppressed ...