You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2022/05/26 16:07:35 UTC
[incubator-inlong] branch master updated: [INLONG-4353][Sort] Optimize code structure and shading jar (#4379)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new dfd23f3e7 [INLONG-4353][Sort] Optimize code structure and shading jar (#4379)
dfd23f3e7 is described below
commit dfd23f3e7344edb243bc794f328519c4d39db207
Author: thexia <37...@users.noreply.github.com>
AuthorDate: Fri May 27 00:07:29 2022 +0800
[INLONG-4353][Sort] Optimize code structure and shading jar (#4379)
---
.github/workflows/ci_build.yml | 7 +
inlong-distribution/pom.xml | 27 +++-
.../main/assemblies/release.xml} | 18 +++
.../src/main/assemblies/sort-connectors.xml | 119 ++++++++++++++++
inlong-sort/pom.xml | 28 ++++
inlong-sort/sort-connectors/hbase/pom.xml | 152 +++++++++++++++++++++
inlong-sort/sort-connectors/hive/pom.xml | 66 +++++----
.../org/apache/inlong/sort/hive/HiveTableSink.java | 2 +-
inlong-sort/sort-connectors/iceberg/pom.xml | 104 ++++++++++++++
inlong-sort/sort-connectors/jdbc/pom.xml | 33 +++++
inlong-sort/sort-connectors/kafka/pom.xml | 53 +++++++
inlong-sort/sort-connectors/mongodb-cdc/pom.xml | 119 ++++++++++++++++
inlong-sort/sort-connectors/mysql-cdc/pom.xml | 98 ++++++++++++-
.../table/RowDataDebeziumDeserializeSchema.java | 2 -
inlong-sort/sort-connectors/oracle-cdc/pom.xml | 125 +++++++++++++++++
inlong-sort/sort-connectors/pom.xml | 126 +----------------
inlong-sort/sort-connectors/postgres-cdc/pom.xml | 117 ++++++++++++++++
inlong-sort/sort-connectors/pulsar/pom.xml | 65 ++++++++-
inlong-sort/sort-connectors/sqlserver-cdc/pom.xml | 110 +++++++++++++++
inlong-sort/sort-core/pom.xml | 143 +++++++++++--------
inlong-sort/sort-dist/pom.xml | 79 ++++++++++-
inlong-sort/sort-formats/format-base/pom.xml | 1 -
inlong-sort/sort-formats/format-csv/pom.xml | 6 +-
.../sort-formats/format-inlongmsg-base/pom.xml | 36 -----
inlong-sort/sort-formats/format-json/pom.xml | 9 +-
inlong-sort/sort-formats/pom.xml | 7 +-
pom.xml | 32 +++++
27 files changed, 1403 insertions(+), 281 deletions(-)
diff --git a/.github/workflows/ci_build.yml b/.github/workflows/ci_build.yml
index 2d1ec3de6..ee0bcdee6 100644
--- a/.github/workflows/ci_build.yml
+++ b/.github/workflows/ci_build.yml
@@ -93,3 +93,10 @@ jobs:
with:
name: apache-inlong-${{ env.VERSION }}-bin.tar.gz
path: ./inlong-distribution/target/apache-inlong-${{ env.VERSION }}-bin.tar.gz
+
+ - name: Upload sort connectors package
+ if: ${{ success() }}
+ uses: actions/upload-artifact@v3
+ with:
+ name: apache-inlong-${{ env.VERSION }}-sort-connectors.tar.gz
+ path: ./inlong-distribution/target/apache-inlong-${{ env.VERSION }}-sort-connectors.tar.gz
diff --git a/inlong-distribution/pom.xml b/inlong-distribution/pom.xml
index 7bba634b9..99da252ec 100644
--- a/inlong-distribution/pom.xml
+++ b/inlong-distribution/pom.xml
@@ -38,20 +38,35 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
+ <configuration>
+ <finalName>apache-${project.parent.artifactId}-${project.version}</finalName>
+ </configuration>
<executions>
<execution>
+ <id>release</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assemblies/release.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ <execution>
+ <id>sort-plugin</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assemblies/sort-connectors.xml</descriptor>
+ </descriptors>
+ </configuration>
</execution>
</executions>
- <configuration>
- <finalName>apache-${project.parent.artifactId}-${project.version}</finalName>
- <descriptors>
- <descriptor>distribution.xml</descriptor>
- </descriptors>
- </configuration>
</plugin>
</plugins>
</build>
diff --git a/inlong-distribution/distribution.xml b/inlong-distribution/src/main/assemblies/release.xml
similarity index 87%
rename from inlong-distribution/distribution.xml
rename to inlong-distribution/src/main/assemblies/release.xml
index c954b3f62..5e3658e86 100644
--- a/inlong-distribution/distribution.xml
+++ b/inlong-distribution/src/main/assemblies/release.xml
@@ -70,6 +70,24 @@
<include>sort-dist-${project.version}.jar</include>
</includes>
</fileSet>
+ <!--basic connector-->
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/pulsar/target</directory>
+ <outputDirectory>sort-plugin/connectors</outputDirectory>
+ <includes>
+ <include>sort-connector-pulsar-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/hive/target</directory>
+ <outputDirectory>sort-plugin/connectors</outputDirectory>
+ <includes>
+ <include>sort-connector-hive-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+
<!-- package InLong-Dashboard-->
<fileSet>
diff --git a/inlong-distribution/src/main/assemblies/sort-connectors.xml b/inlong-distribution/src/main/assemblies/sort-connectors.xml
new file mode 100644
index 000000000..2c7a63195
--- /dev/null
+++ b/inlong-distribution/src/main/assemblies/sort-connectors.xml
@@ -0,0 +1,119 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly>
+ <id>sort-connectors</id>
+
+ <formats>
+ <format>dir</format>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <fileSets>
+ <!--connector plugin-->
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/hbase/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-hbase-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/hive/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-hive-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/iceberg/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-iceberg-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/jdbc/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-jdbc-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/kafka/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-kafka-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/mongodb-cdc/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-mongodb-cdc-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/mysql-cdc/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-mysql-cdc-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/postgres-cdc/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-postgres-cdc-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/pulsar/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-pulsar-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/sqlserver-cdc/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-sqlserver-cdc-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../inlong-sort/sort-connectors/oracle-cdc/target</directory>
+ <outputDirectory>sort-plugin</outputDirectory>
+ <includes>
+ <include>sort-connector-oracle-cdc-${project.version}.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/inlong-sort/pom.xml b/inlong-sort/pom.xml
index 484808882..fe65425de 100644
--- a/inlong-sort/pom.xml
+++ b/inlong-sort/pom.xml
@@ -44,6 +44,7 @@
<properties>
<debezium.version>1.5.4.Final</debezium.version>
<kafka.clients.version>2.7.0</kafka.clients.version>
+ <rat.basedir>${basedir}</rat.basedir>
</properties>
<dependencyManagement>
<dependencies>
@@ -111,6 +112,33 @@
</dependencies>
<build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${plugin.shade.version}</version>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
+ <createSourcesJar>true</createSourcesJar>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/inlong-sort/sort-connectors/hbase/pom.xml b/inlong-sort/sort-connectors/hbase/pom.xml
new file mode 100644
index 000000000..d4517f709
--- /dev/null
+++ b/inlong-sort/sort-connectors/hbase/pom.xml
@@ -0,0 +1,152 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>sort-connectors</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <version>1.2.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>sort-connector-hbase</artifactId>
+ <name>Apache InLong - Sort-connector-hbase</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-hbase-2.2_${flink.scala.binary.version}</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <!--
+ Make the file hbase-default.xml under flink-sql-connector-hbase-2.2/src/main/resources as the
+ hbase-default.xml in the shaded target jar here, because we don't want to check the hbase
+ version at client side. Also we don't need the extra default configs keys.
+ -->
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>hbase-default.xml</resource>
+ <file>hbase-default.xml</file>
+ </transformer>
+ </transformers>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-hbase-base_${scala.binary.version}</include>
+ <include>org.apache.flink:flink-connector-hbase-2.2_${scala.binary.version}</include>
+ <include>org.apache.hbase:hbase-*</include>
+ <include>org.apache.hbase.thirdparty:hbase-shaded-*</include>
+ <include>org.apache.zookeeper:zookeeper</include>
+ <include>org.apache.htrace:htrace-core4</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>commons-codec:commons-codec</include>
+ <include>org.apache.commons:commons-crypto</include>
+ <include>org.apache.commons:commons-lang3</include>
+ <include>io.netty:netty-all</include>
+ <include>io.dropwizard.metrics:metrics-core</include>
+ </includes>
+ <excludes>
+ <exclude>org.apache.hbase:hbase-metrics*</exclude>
+ <exclude>org.apache.hbase:hbase-server*</exclude>
+ <exclude>org.apache.hbase:hbase-hadoop*-compat</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <!-- excluded all these files for a clean jar -->
+ <exclude>META-INF/services/com.fasterxml.**</exclude>
+ <exclude>META-INF/services/org.apache.hadoop.**</exclude>
+ <exclude>META-INF/services/javax.**</exclude>
+ <exclude>digesterRules.xml</exclude>
+ <exclude>properties.dtd</exclude>
+ <exclude>PropertyList-1.0.dtd</exclude>
+ <exclude>LICENSE.txt</exclude>
+ <exclude>*.proto</exclude>
+ <exclude>protobuf/*</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <!-- Force relocation of all HBase dependencies. -->
+ <relocation>
+ <pattern>org.apache.zookeeper</pattern>
+ <shadedPattern>org.apache.flink.hbase.shaded.org.apache.zookeeper</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.htrace</pattern>
+ <shadedPattern>org.apache.flink.hbase.shaded.org.apache.htrace</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>org.apache.flink.hbase.shaded.com.google</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.yammer.metrics</pattern>
+ <shadedPattern>org.apache.flink.hbase.shaded.com.yammer.metrics</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons</pattern>
+ <shadedPattern>org.apache.flink.hbase.shaded.org.apache.commons</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.jute</pattern>
+ <shadedPattern>org.apache.flink.hbase.shaded.org.apache.jute</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>org.apache.flink.hbase.shaded.io.netty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.hbase</pattern>
+ <shadedPattern>org.apache.flink.hbase.shaded.org.apache.hadoop.hbase</shadedPattern>
+ <!-- HBase client uses shaded KeyValueCodec to encode data and put the class name in
+ the header of request, the HBase region server can not load the shaded
+ KeyValueCodec class when decode the data, so we exclude them here. -->
+ <excludes>
+ <exclude>org.apache.hadoop.hbase.codec.*</exclude>
+ </excludes>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/inlong-sort/sort-connectors/hive/pom.xml b/inlong-sort/sort-connectors/hive/pom.xml
index a72269a18..588f6de7d 100644
--- a/inlong-sort/sort-connectors/hive/pom.xml
+++ b/inlong-sort/sort-connectors/hive/pom.xml
@@ -42,48 +42,44 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.calcite</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-druid</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <exclusions>
- <exclusion>
- <artifactId>parquet-hadoop-bundle</artifactId>
- <groupId>org.apache.parquet</groupId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-hive_${scala.binary.version}</include>
+ <include>org.apache.hive:hive-exec</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.parquet</pattern>
+ <shadedPattern>org.apache.hive.shaded.parquet</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/inlong-sort/sort-connectors/hive/src/main/java/org/apache/inlong/sort/hive/HiveTableSink.java b/inlong-sort/sort-connectors/hive/src/main/java/org/apache/inlong/sort/hive/HiveTableSink.java
index 8da4bffdd..3ac2aaa3e 100644
--- a/inlong-sort/sort-connectors/hive/src/main/java/org/apache/inlong/sort/hive/HiveTableSink.java
+++ b/inlong-sort/sort-connectors/hive/src/main/java/org/apache/inlong/sort/hive/HiveTableSink.java
@@ -27,7 +27,7 @@ import org.apache.flink.connectors.hive.util.JobConfUtils;
import org.apache.flink.connectors.hive.write.HiveBulkWriterFactory;
import org.apache.flink.connectors.hive.write.HiveOutputFormatFactory;
import org.apache.flink.connectors.hive.write.HiveWriterFactory;
-import org.apache.flink.formats.parquet.row.ParquetRowDataBuilder;
+import org.apache.flink.hive.shaded.formats.parquet.row.ParquetRowDataBuilder;
import org.apache.flink.orc.OrcSplitReaderUtil;
import org.apache.flink.orc.writer.ThreadLocalClassLoaderConfiguration;
import org.apache.flink.streaming.api.datastream.DataStream;
diff --git a/inlong-sort/sort-connectors/iceberg/pom.xml b/inlong-sort/sort-connectors/iceberg/pom.xml
new file mode 100644
index 000000000..02c30e1e7
--- /dev/null
+++ b/inlong-sort/sort-connectors/iceberg/pom.xml
@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>sort-connectors</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <version>1.2.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>sort-connector-iceberg</artifactId>
+ <name>Apache InLong - Sort-connector-iceberg</name>
+ <packaging>jar</packaging>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-flink-1.13</artifactId>
+ </dependency>
+ </dependencies>
+
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.avro</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.org.apache.avro</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.parquet</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.org.apache.parquet</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.com.google</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.com.fasterxml</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.github.benmanes</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.com.github.benmanes</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.checkerframework</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.org.checkerframework</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>shaded.parquet</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.org.apache.parquet.shaded</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.orc</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.org.apache.orc</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.airlift</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.io.airlift</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.threeten.extra</pattern>
+ <shadedPattern>org.apache.iceberg.shaded.org.threeten.extra</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/inlong-sort/sort-connectors/jdbc/pom.xml b/inlong-sort/sort-connectors/jdbc/pom.xml
index d88b6fbdb..a4fad240b 100644
--- a/inlong-sort/sort-connectors/jdbc/pom.xml
+++ b/inlong-sort/sort-connectors/jdbc/pom.xml
@@ -48,6 +48,39 @@
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-jdbc_${flink.scala.binary.version}</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-jdbc_${flink.scala.binary.version}</include>
+ <include>org.postgresql:postgresql</include>
+ <include>ru.yandex.clickhouse:clickhouse-jdbc</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
</project>
\ No newline at end of file
diff --git a/inlong-sort/sort-connectors/kafka/pom.xml b/inlong-sort/sort-connectors/kafka/pom.xml
index e1cf1620b..d8d997e7d 100644
--- a/inlong-sort/sort-connectors/kafka/pom.xml
+++ b/inlong-sort/sort-connectors/kafka/pom.xml
@@ -32,4 +32,57 @@
<name>Apache InLong - Sort-connector-kafka</name>
<packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-kafka_${flink.scala.binary.version}</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.kafka:*</include>
+ <include>org.apache.flink:flink-connector-kafka_${scala.binary.version}</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>org.apache.kafka:*</artifact>
+ <excludes>
+ <exclude>kafka/kafka-version.properties</exclude>
+ <exclude>LICENSE</exclude>
+ <!-- Does not contain anything relevant.
+ Cites a binary dependency on jersey, but this is neither reflected in the
+ dependency graph, nor are any jersey files bundled. -->
+ <exclude>NOTICE</exclude>
+ <exclude>common/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.kafka</pattern>
+ <shadedPattern>org.apache.flink.kafka.shaded.org.apache.kafka</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
</project>
\ No newline at end of file
diff --git a/inlong-sort/sort-connectors/mongodb-cdc/pom.xml b/inlong-sort/sort-connectors/mongodb-cdc/pom.xml
new file mode 100644
index 000000000..0101e8d9d
--- /dev/null
+++ b/inlong-sort/sort-connectors/mongodb-cdc/pom.xml
@@ -0,0 +1,119 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>sort-connectors</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <version>1.2.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>sort-connector-mongodb-cdc</artifactId>
+ <name>Apache InLong - Sort-connector-mongodb-cdc</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.ververica</groupId>
+ <artifactId>flink-connector-mongodb-cdc</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>io.debezium:debezium-api</include>
+ <include>io.debezium:debezium-embedded</include>
+ <include>io.debezium:debezium-core</include>
+ <include>com.ververica:flink-connector-debezium</include>
+ <include>com.ververica:flink-connector-mongodb-cdc</include>
+ <include>org.mongodb.kafka:mongo-kafka-connect</include>
+ <include>org.mongodb:mongodb-driver-sync</include>
+ <include>org.mongodb:mongodb-driver-core</include>
+ <include>org.mongodb:bson</include>
+ <include>org.apache.avro:avro</include>
+ <include>org.apache.kafka:*</include>
+ <include>com.fasterxml.*:*</include>
+ <include>com.google.guava:*</include>
+ <!-- Include fixed version 18.0-13.0 of flink shaded guava -->
+ <include>org.apache.flink:flink-shaded-guava</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>org.apache.kafka:*</artifact>
+ <excludes>
+ <exclude>kafka/kafka-version.properties</exclude>
+ <exclude>LICENSE</exclude>
+ <!-- Does not contain anything relevant.
+ Cites a binary dependency on jersey, but this is neither reflected in the
+ dependency graph, nor are any jersey files bundled. -->
+ <exclude>NOTICE</exclude>
+ <exclude>common/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.kafka</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.org.apache.kafka
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.avro</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.org.apache.avro
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.fasterxml
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.google
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/inlong-sort/sort-connectors/mysql-cdc/pom.xml b/inlong-sort/sort-connectors/mysql-cdc/pom.xml
index abf910fbd..5f2693e5e 100644
--- a/inlong-sort/sort-connectors/mysql-cdc/pom.xml
+++ b/inlong-sort/sort-connectors/mysql-cdc/pom.xml
@@ -33,12 +33,106 @@
<packaging>jar</packaging>
<dependencies>
-
<dependency>
<groupId>com.ververica</groupId>
<artifactId>flink-connector-mysql-cdc</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-json</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>io.debezium:debezium-api</include>
+ <include>io.debezium:debezium-embedded</include>
+ <include>io.debezium:debezium-core</include>
+ <include>io.debezium:debezium-ddl-parser</include>
+ <include>io.debezium:debezium-connector-mysql</include>
+ <include>com.ververica:flink-connector-debezium</include>
+ <include>com.ververica:flink-connector-mysql-cdc</include>
+ <include>org.antlr:antlr4-runtime</include>
+ <include>org.apache.kafka:*</include>
+ <include>mysql:mysql-connector-java</include>
+ <include>com.zendesk:mysql-binlog-connector-java</include>
+ <include>com.fasterxml.*:*</include>
+ <include>com.google.guava:*</include>
+ <include>com.esri.geometry:esri-geometry-api</include>
+ <include>com.zaxxer:HikariCP</include>
+ <!-- Include fixed version 18.0-13.0 of flink shaded guava -->
+ <include>org.apache.flink:flink-shaded-guava</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>org.apache.kafka:*</artifact>
+ <excludes>
+ <exclude>kafka/kafka-version.properties</exclude>
+ <exclude>LICENSE</exclude>
+ <!-- Does not contain anything relevant.
+ Cites a binary dependency on jersey, but this is neither reflected in the
+ dependency graph, nor are any jersey files bundled. -->
+ <exclude>NOTICE</exclude>
+ <exclude>common/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.kafka</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.org.apache.kafka
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.antlr</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.org.antlr
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.fasterxml
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.google
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.esri.geometry</pattern>
+ <shadedPattern>com.ververica.cdc.connectors.shaded.com.esri.geometry</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.zaxxer</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.zaxxer
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/debezium/table/RowDataDebeziumDeserializeSchema.java b/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/debezium/table/RowDataDebeziumDeserializeSchema.java
index fb29182e3..a6f5c8da6 100644
--- a/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/debezium/table/RowDataDebeziumDeserializeSchema.java
+++ b/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/debezium/table/RowDataDebeziumDeserializeSchema.java
@@ -51,7 +51,6 @@ import org.apache.kafka.connect.source.SourceRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.validation.constraints.NotNull;
import java.io.Serializable;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
@@ -581,7 +580,6 @@ public final class RowDataDebeziumDeserializeSchema
}
}
- @NotNull
private DeserializationRuntimeConverter getAllMigrationConverter() {
return new DeserializationRuntimeConverter() {
diff --git a/inlong-sort/sort-connectors/oracle-cdc/pom.xml b/inlong-sort/sort-connectors/oracle-cdc/pom.xml
new file mode 100644
index 000000000..d5634c1eb
--- /dev/null
+++ b/inlong-sort/sort-connectors/oracle-cdc/pom.xml
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>sort-connectors</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <version>1.2.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>sort-connector-oracle-cdc</artifactId>
+ <name>Apache InLong - Sort-connector-oracle-cdc</name>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>com.ververica</groupId>
+ <artifactId>flink-connector-oracle-cdc</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>io.debezium:debezium-api</include>
+ <include>io.debezium:debezium-embedded</include>
+ <include>io.debezium:debezium-core</include>
+ <incldue>io.debezium:debezium-ddl-parser</incldue>
+ <include>io.debezium:debezium-connector-oracle</include>
+ <include>com.ververica:flink-connector-debezium</include>
+ <include>com.ververica:flink-connector-oracle-cdc</include>
+ <include>org.antlr:antlr4-runtime</include>
+ <include>com.github.jsqlparser:jsqlparser</include>
+ <include>com.oracle.ojdbc:*</include>
+ <include>org.apache.kafka:*</include>
+ <include>com.fasterxml.*:*</include>
+ <include>com.google.guava:*</include>
+ <!-- Include fixed version 18.0-13.0 of flink shaded guava -->
+ <include>org.apache.flink:flink-shaded-guava</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>org.apache.kafka:*</artifact>
+ <excludes>
+ <exclude>kafka/kafka-version.properties</exclude>
+ <exclude>LICENSE</exclude>
+ <!-- Does not contain anything relevant.
+ Cites a binary dependency on jersey, but this is neither reflected in the
+ dependency graph, nor are any jersey files bundled. -->
+ <exclude>NOTICE</exclude>
+ <exclude>common/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.kafka</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.org.apache.kafka
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.antlr</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.org.antlr
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.google
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.fasterxml
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>net.sf.jsqlparser</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.net.sf.jsqlparser
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/inlong-sort/sort-connectors/pom.xml b/inlong-sort/sort-connectors/pom.xml
index 64103e7bb..70726f09d 100644
--- a/inlong-sort/sort-connectors/pom.xml
+++ b/inlong-sort/sort-connectors/pom.xml
@@ -39,15 +39,15 @@
<module>kafka</module>
<module>jdbc</module>
<module>pulsar</module>
+ <module>iceberg</module>
+ <module>hbase</module>
+ <module>postgres-cdc</module>
+ <module>mongodb-cdc</module>
+ <module>sqlserver-cdc</module>
+ <module>oracle-cdc</module>
</modules>
<dependencies>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <!--for flink core-->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
@@ -73,120 +73,6 @@
<artifactId>flink-table-runtime-blink_${flink.scala.binary.version}</artifactId>
<scope>provided</scope>
</dependency>
- <!--for hadoop common-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!--for iceberg-->
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-flink-1.13</artifactId>
- </dependency>
- <!--for kafka-->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-kafka_${flink.scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- </dependency>
- <!--for hbase-->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-hbase-2.2_${flink.scala.binary.version}</artifactId>
- </dependency>
- <!--for postgres cdc -->
- <dependency>
- <groupId>com.ververica</groupId>
- <artifactId>flink-connector-postgres-cdc</artifactId>
- </dependency>
- <!--for mongodb-->
- <dependency>
- <groupId>com.ververica</groupId>
- <artifactId>flink-connector-mongodb-cdc</artifactId>
- </dependency>
- <!--for oracle-->
- <dependency>
- <groupId>com.ververica</groupId>
- <artifactId>flink-connector-oracle-cdc</artifactId>
- </dependency>
- <!--for debezium-->
- <dependency>
- <groupId>io.debezium</groupId>
- <artifactId>debezium-embedded</artifactId>
- </dependency>
- <dependency>
- <artifactId>debezium-core</artifactId>
- <groupId>io.debezium</groupId>
- </dependency>
- <!--for sort format-->
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-format-base</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-format-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-format-csv</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-format-kv</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-format-inlongmsg-base</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-format-inlongmsg-csv</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-format-json</artifactId>
- <version>${project.version}</version>
- </dependency>
- <!-- flink json -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-json</artifactId>
- </dependency>
- <!--for csv-->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-csv</artifactId>
- </dependency>
- <!--for avro-->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-avro</artifactId>
- </dependency>
- <!--for orc-->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-orc_${scala.binary.version}</artifactId>
- </dependency>
- <!--for parquet-->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-parquet_${flink.scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>com.ververica</groupId>
- <artifactId>flink-connector-sqlserver-cdc</artifactId>
- </dependency>
</dependencies>
</project>
diff --git a/inlong-sort/sort-connectors/postgres-cdc/pom.xml b/inlong-sort/sort-connectors/postgres-cdc/pom.xml
new file mode 100644
index 000000000..0bc10c852
--- /dev/null
+++ b/inlong-sort/sort-connectors/postgres-cdc/pom.xml
@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>sort-connectors</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <version>1.2.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>sort-connector-postgres-cdc</artifactId>
+ <name>Apache InLong - Sort-connector-postgres-cdc</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.ververica</groupId>
+ <artifactId>flink-connector-postgres-cdc</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>io.debezium:debezium-api</include>
+ <include>io.debezium:debezium-embedded</include>
+ <include>io.debezium:debezium-core</include>
+ <include>io.debezium:debezium-connector-postgres</include>
+ <include>com.ververica:flink-connector-debezium</include>
+ <include>com.ververica:flink-connector-postgres-cdc</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>com.google.guava:*</include>
+ <include>org.apache.kafka:*</include>
+ <include>org.postgresql:postgresql</include>
+ <include>com.fasterxml.*:*</include>
+ <!-- Include fixed version 18.0-13.0 of flink shaded guava -->
+ <include>org.apache.flink:flink-shaded-guava</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>org.apache.kafka:*</artifact>
+ <excludes>
+ <exclude>kafka/kafka-version.properties</exclude>
+ <exclude>LICENSE</exclude>
+ <!-- Does not contain anything relevant.
+ Cites a binary dependency on jersey, but this is neither reflected in the
+ dependency graph, nor are any jersey files bundled. -->
+ <exclude>NOTICE</exclude>
+ <exclude>common/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.kafka</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.org.apache.kafka
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.google
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.fasterxml
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/inlong-sort/sort-connectors/pulsar/pom.xml b/inlong-sort/sort-connectors/pulsar/pom.xml
index 508650b94..49cf200a6 100644
--- a/inlong-sort/sort-connectors/pulsar/pom.xml
+++ b/inlong-sort/sort-connectors/pulsar/pom.xml
@@ -33,17 +33,76 @@
<packaging>jar</packaging>
<dependencies>
+ <dependency>
+ <groupId>io.streamnative.connectors</groupId>
+ <artifactId>pulsar-flink-connector-origin</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client-all</artifactId>
</dependency>
-
<dependency>
<groupId>io.streamnative.connectors</groupId>
- <artifactId>pulsar-flink-connector_${scala.binary.version}</artifactId>
+ <artifactId>flink-protobuf</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-json</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-avro</artifactId>
</dependency>
-
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <!-- Shade all the dependencies to avoid conflicts -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <artifactSet>
+ <includes>
+ <include>io.streamnative.connectors:pulsar-flink-connector-origin*</include>
+ <include>io.streamnative.connectors:flink-protobuf</include>
+ <include>org.apache.pulsar:*</include>
+ <include>org.bouncycastle*:*</include>
+ <include>org.bouncycastle*:*</include>
+ <include>javax.*:*</include>
+ <include>org.lz4*:*</include>
+ <include>org.slf4j:jul-to-slf4j</include>
+ <include>io.airlift:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>log4j.properties</exclude>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.PluginXmlResourceTransformer"/>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
\ No newline at end of file
diff --git a/inlong-sort/sort-connectors/sqlserver-cdc/pom.xml b/inlong-sort/sort-connectors/sqlserver-cdc/pom.xml
new file mode 100644
index 000000000..f250d9a7f
--- /dev/null
+++ b/inlong-sort/sort-connectors/sqlserver-cdc/pom.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>sort-connectors</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <version>1.2.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>sort-connector-sqlserver-cdc</artifactId>
+ <name>Apache InLong - Sort-connector-sqlserver-cdc</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.ververica</groupId>
+ <artifactId>flink-connector-sqlserver-cdc</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>io.debezium:debezium-api</include>
+ <include>io.debezium:debezium-embedded</include>
+ <include>io.debezium:debezium-core</include>
+ <include>io.debezium:debezium-connector-sqlserver</include>
+ <include>com.ververica:flink-connector-debezium</include>
+ <include>com.ververica:flink-connector-sqlserver-cdc</include>
+ <include>com.microsoft.sqlserver:*</include>
+ <include>org.apache.kafka:*</include>
+ <include>com.fasterxml.*:*</include>
+ <include>com.google.guava:*</include>
+ <!-- Include fixed version 18.0-13.0 of flink shaded guava -->
+ <include>org.apache.flink:flink-shaded-guava</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>org.apache.kafka:*</artifact>
+ <excludes>
+ <exclude>kafka/kafka-version.properties</exclude>
+ <exclude>LICENSE</exclude>
+ <!-- Does not contain anything relevant.
+ Cites a binary dependency on jersey, but this is neither reflected in the
+ dependency graph, nor are any jersey files bundled. -->
+ <exclude>NOTICE</exclude>
+ <exclude>common/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.kafka</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.org.apache.kafka
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.google
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>
+ com.ververica.cdc.connectors.shaded.com.fasterxml
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/inlong-sort/sort-core/pom.xml b/inlong-sort/sort-core/pom.xml
index 75a1cc213..8e3b630a4 100644
--- a/inlong-sort/sort-core/pom.xml
+++ b/inlong-sort/sort-core/pom.xml
@@ -47,33 +47,7 @@
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-hive</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-pulsar</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-jdbc</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-mysql-cdc</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-kafka</artifactId>
- <version>${project.version}</version>
- </dependency>
- <!--audit sdk-->
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>audit-sdk</artifactId>
+ <artifactId>sort-format-base</artifactId>
<version>${project.version}</version>
</dependency>
<!--flink core-->
@@ -102,62 +76,111 @@
<artifactId>flink-table-planner-blink_${flink.scala.binary.version}</artifactId>
<scope>provided</scope>
</dependency>
- <!-- flink connector -->
+
+ <!-- for test -->
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-hive_${flink.scala.binary.version}</artifactId>
+ <artifactId>flink-test-utils_${flink.scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
- <!-- flink chdfs -->
<dependency>
- <groupId>com.qcloud</groupId>
- <artifactId>chdfs_hadoop_plugin_network</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-hive</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
</dependency>
- <!--for cos-->
<dependency>
- <groupId>com.qcloud.cos</groupId>
- <artifactId>flink-cos-fs-hadoop</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-hbase</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
</dependency>
- <!--for jackson-->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-iceberg</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
</dependency>
- <!--for xml-->
<dependency>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- <version>1.4.01</version>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
- <artifactId>tubemq-client</artifactId>
+ <artifactId>sort-connector-kafka</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
- <!-- for test -->
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_${flink.scala.binary.version}</artifactId>
- <version>${flink.version}</version>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-mongodb-cdc</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-mysql-cdc</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>pulsar</artifactId>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-postgres-cdc</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-pulsar</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-sqlserver-cdc</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-connector-oracle-cdc</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-inlongmsg-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-json</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-csv</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
+ <artifactId>hadoop-common</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/inlong-sort/sort-dist/pom.xml b/inlong-sort/sort-dist/pom.xml
index 00eeb1676..44adc9ec7 100644
--- a/inlong-sort/sort-dist/pom.xml
+++ b/inlong-sort/sort-dist/pom.xml
@@ -47,6 +47,62 @@
<artifactId>sort-api</artifactId>
<version>${project.version}</version>
</dependency>
+ <!--flink format-->
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-csv</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-inlongmsg-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-inlongmsg-csv</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-json</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>sort-format-kv</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-csv</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-json</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-sql-avro</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-sql-parquet_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-sql-orc_${scala.binary.version}</artifactId>
+ </dependency>
</dependencies>
<build>
@@ -54,17 +110,36 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.2.4</version>
<executions>
<execution>
+ <id>shade-flink</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
<shadedArtifactAttached>false</shadedArtifactAttached>
<finalName>${project.artifactId}-${project.version}</finalName>
+ <artifactSet>
+ <includes>
+ <include>org.apache.inlong:sort-common</include>
+ <include>org.apache.inlong:sort-core</include>
+ <include>org.apache.inlong:sort-api</include>
+ <!--User do not need to care which format include, sort include all basic format-->
+ <include>org.apache.inlong:sort-format-common</include>
+ <include>org.apache.inlong:sort-format-base</include>
+ <include>org.apache.inlong:sort-format-csv</include>
+ <include>org.apache.inlong:sort-format-inlongmsg-base</include>
+ <include>org.apache.inlong:sort-format-inlongmsg-csv</include>
+ <include>org.apache.inlong:sort-format-json</include>
+ <include>org.apache.inlong:sort-format-kv</include>
+ <include>org.apache.flink:flink-csv</include>
+ <include>org.apache.flink:flink-json</include>
+ <include>org.apache.flink:flink-sql-avro</include>
+ <include>org.apache.flink:flink-sql-parquet_${scala.binary.version}</include>
+ <include>org.apache.flink:flink-sql-orc_${scala.binary.version}</include>
+ </includes>
+ </artifactSet>
<filters>
<!-- Globally exclude log4j.properties from our JAR files. -->
<filter>
diff --git a/inlong-sort/sort-formats/format-base/pom.xml b/inlong-sort/sort-formats/format-base/pom.xml
index 007770606..ed6c64167 100644
--- a/inlong-sort/sort-formats/format-base/pom.xml
+++ b/inlong-sort/sort-formats/format-base/pom.xml
@@ -36,7 +36,6 @@
<packaging>jar</packaging>
<dependencies>
-
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-common</artifactId>
diff --git a/inlong-sort/sort-formats/format-csv/pom.xml b/inlong-sort/sort-formats/format-csv/pom.xml
index fe1a7cea1..e22d29ada 100644
--- a/inlong-sort/sort-formats/format-csv/pom.xml
+++ b/inlong-sort/sort-formats/format-csv/pom.xml
@@ -36,9 +36,12 @@
<packaging>jar</packaging>
<dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-csv</artifactId>
+ </dependency>
<!-- core dependencies -->
-
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-format-common</artifactId>
@@ -85,7 +88,6 @@
</dependency>
</dependencies>
-
<profiles>
<!-- Create SQL Client uber jars by default -->
<profile>
diff --git a/inlong-sort/sort-formats/format-inlongmsg-base/pom.xml b/inlong-sort/sort-formats/format-inlongmsg-base/pom.xml
index 49edb5358..823908e06 100644
--- a/inlong-sort/sort-formats/format-inlongmsg-base/pom.xml
+++ b/inlong-sort/sort-formats/format-inlongmsg-base/pom.xml
@@ -100,40 +100,4 @@
</dependency>
</dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.2.4</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>
- org.apache.flink.shaded.guava18.com.google.common
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.thirdparty</pattern>
- <shadedPattern>
- org.apache.flink.shaded.guava18.com.google.thirdparty
- </shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
</project>
diff --git a/inlong-sort/sort-formats/format-json/pom.xml b/inlong-sort/sort-formats/format-json/pom.xml
index 9a0846c96..c4f953fa9 100644
--- a/inlong-sort/sort-formats/format-json/pom.xml
+++ b/inlong-sort/sort-formats/format-json/pom.xml
@@ -36,6 +36,10 @@
<packaging>jar</packaging>
<dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-json</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
@@ -50,11 +54,6 @@
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-json</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
diff --git a/inlong-sort/sort-formats/pom.xml b/inlong-sort/sort-formats/pom.xml
index d95ff8439..b09690d04 100644
--- a/inlong-sort/sort-formats/pom.xml
+++ b/inlong-sort/sort-formats/pom.xml
@@ -135,6 +135,7 @@
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-common</artifactId>
<version>${flink.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
@@ -175,12 +176,6 @@
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.0.0</version>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
diff --git a/pom.xml b/pom.xml
index 437024258..18f5ee3ef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,6 +112,7 @@
<plugin.assembly.version>3.2.0</plugin.assembly.version>
<plugin.surefire.version>2.22.2</plugin.surefire.version>
<plugin.failsafe.version>2.22.2</plugin.failsafe.version>
+ <plugin.shade.version>3.2.4</plugin.shade.version>
<plugin.maven.source>3.0.1</plugin.maven.source>
<exec.maven.version>1.6.0</exec.maven.version>
<build.helper.maven.version>3.0.0</build.helper.maven.version>
@@ -201,6 +202,7 @@
<flink.connector.postgres.cdc.version>2.2.1</flink.connector.postgres.cdc.version>
<flink.connector.sqlserver.cdc.version>2.2.1</flink.connector.sqlserver.cdc.version>
<flink.pulsar.version>1.13.6.1-rc9</flink.pulsar.version>
+ <flink.protobuf.version>2.7.6</flink.protobuf.version>
<flink.connector.mongodb.cdc.version>2.2.1</flink.connector.mongodb.cdc.version>
<flink.connector.oracle.cdc.version>2.2.1</flink.connector.oracle.cdc.version>
<qcloud.flink.cos.fs.hadoop.version>1.10.0-0.1.10</qcloud.flink.cos.fs.hadoop.version>
@@ -939,6 +941,21 @@
<artifactId>flink-orc_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-sql-avro</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-sql-parquet_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-sql-orc_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded-jackson</artifactId>
@@ -1020,6 +1037,16 @@
<artifactId>pulsar-flink-connector_${scala.binary.version}</artifactId>
<version>${flink.pulsar.version}</version>
</dependency>
+ <dependency>
+ <groupId>io.streamnative.connectors</groupId>
+ <artifactId>pulsar-flink-connector-origin</artifactId>
+ <version>${flink.pulsar.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.streamnative.connectors</groupId>
+ <artifactId>flink-protobuf</artifactId>
+ <version>${flink.protobuf.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
@@ -1450,6 +1477,11 @@
<target>${maven.compiler.target}</target>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>${plugin.maven.source}</version>
+ </plugin>
</plugins>
</pluginManagement>
<plugins>