You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ki...@apache.org on 2022/09/03 07:14:10 UTC
[incubator-seatunnel] branch dev updated: [#2604]Dependency management split (#2623)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 5f4d2890c [#2604]Dependency management split (#2623)
5f4d2890c is described below
commit 5f4d2890c51fcd974436a79d4ee27adca303d9f8
Author: Kirs <ki...@apache.org>
AuthorDate: Sat Sep 3 15:14:04 2022 +0800
[#2604]Dependency management split (#2623)
* [#2604]Dependency management split
#2604 subtask
Split Spark connector
* remove unless dependency
---
pom.xml | 147 +--------------------
seatunnel-apis/seatunnel-api-spark/pom.xml | 6 +
.../seatunnel-connectors-spark/pom.xml | 40 ++++++
.../seatunnel-connector-spark-clickhouse/pom.xml | 26 +---
.../seatunnel-connector-spark-console/pom.xml | 19 ---
.../seatunnel-connector-spark-doris/pom.xml | 28 ++--
.../pom.xml | 21 +--
.../seatunnel-connector-spark-email/pom.xml | 22 +--
.../seatunnel-connector-spark-fake/pom.xml | 17 ---
.../seatunnel-connector-spark-feishu/pom.xml | 22 +--
.../seatunnel-connector-spark-file/pom.xml | 19 ---
.../seatunnel-connector-spark-hbase/pom.xml | 24 +---
.../seatunnel-connector-spark-hive/pom.xml | 19 ---
.../seatunnel-connector-spark-http/pom.xml | 20 ---
.../seatunnel-connector-spark-hudi/pom.xml | 20 +--
.../seatunnel-connector-spark-iceberg/pom.xml | 21 +--
.../seatunnel-connector-spark-jdbc/pom.xml | 31 ++---
.../seatunnel-connector-spark-kafka/pom.xml | 17 +--
.../seatunnel-connector-spark-kudu/pom.xml | 21 +--
.../seatunnel-connector-spark-mongodb/pom.xml | 27 +---
.../seatunnel-connector-spark-neo4j/pom.xml | 20 +--
.../seatunnel-connector-spark-phoenix/pom.xml | 28 ++--
.../seatunnel-connector-spark-redis/pom.xml | 27 +---
.../seatunnel-connector-spark-socket/pom.xml | 17 ---
.../seatunnel-connector-spark-tidb/pom.xml | 22 +--
.../seatunnel-connector-spark-webhook/pom.xml | 18 ---
seatunnel-core/seatunnel-core-spark/pom.xml | 10 +-
seatunnel-core/seatunnel-spark-starter/pom.xml | 6 +
.../seatunnel-spark-connector-v2-example/pom.xml | 14 +-
.../seatunnel-spark-examples/pom.xml | 8 +-
.../seatunnel-transforms-spark/pom.xml | 40 ++++++
.../seatunnel-transform-spark-json/pom.xml | 21 ---
.../seatunnel-transform-spark-null-rate/pom.xml | 19 ---
.../seatunnel-transform-spark-nulltf/pom.xml | 20 ---
.../seatunnel-transform-spark-replace/pom.xml | 19 ---
.../seatunnel-transform-spark-split/pom.xml | 20 ---
.../seatunnel-transform-spark-sql/pom.xml | 21 ---
.../seatunnel-transform-spark-uuid/pom.xml | 19 ---
.../seatunnel-translation-spark/pom.xml | 26 ++++
.../seatunnel-translation-spark-2.4/pom.xml | 12 --
.../seatunnel-translation-spark-common/pom.xml | 6 -
41 files changed, 237 insertions(+), 723 deletions(-)
diff --git a/pom.xml b/pom.xml
index 112dd81df..14e6091ba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -117,11 +117,10 @@
<scala.binary.version>2.11</scala.binary.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
- <spark.version>2.4.0</spark.version>
- <spark.binary.version>2.4</spark.binary.version>
- <neo4j.connector.spark.version>4.1.0</neo4j.connector.spark.version>
<iceberg.version>0.13.1</iceberg.version>
<flink.1.13.6.version>1.13.6</flink.1.13.6.version>
+ <spark.2.4.0.version>2.4.0</spark.2.4.0.version>
+ <spark.binary.2.4.version>2.4</spark.binary.2.4.version>
<hudi.version>0.11.1</hudi.version>
<orc.version>1.5.6</orc.version>
<avro.version>1.8.2</avro.version>
@@ -157,18 +156,12 @@
<maven-pmd-plugin.version>3.8</maven-pmd-plugin.version>
<spoiwo.version>1.8.0</spoiwo.version>
<play-mailer.version>7.0.2</play-mailer.version>
- <phoenix-spark.version>5.0.0-HBase-2.0</phoenix-spark.version>
<zkclient.version>0.3</zkclient.version>
<elasticsearch6.client.version>6.3.1</elasticsearch6.client.version>
<elasticsearch7.client.version>7.5.1</elasticsearch7.client.version>
<flink-shaded-hadoop-2.version>2.7.5-7.0</flink-shaded-hadoop-2.version>
<parquet-avro.version>1.10.0</parquet-avro.version>
- <elasticsearch-spark.version>6.8.3</elasticsearch-spark.version>
<clickhouse-jdbc.version>0.2</clickhouse-jdbc.version>
- <hbase-spark.version>1.0.0</hbase-spark.version>
- <kudu-spark.version>1.7.0</kudu-spark.version>
- <mongo-spark.version>2.2.0</mongo-spark.version>
- <spark-redis.version>2.6.0</spark-redis.version>
<commons-lang3.version>3.4</commons-lang3.version>
<commons-net.version>3.6</commons-net.version>
<kudu.version>1.11.1</kudu.version>
@@ -188,7 +181,6 @@
<docker.tag>${project.version}</docker.tag>
<jcommander.version>1.81</jcommander.version>
<junit.version>5.9.0</junit.version>
- <tispark.version>2.4.1</tispark.version>
<druid.version>0.22.1</druid.version>
<sshd.version>2.7.0</sshd.version>
<calcite-druid.version>1.29.0</calcite-druid.version>
@@ -243,34 +235,6 @@
<artifactId>seatunnel-config-shade</artifactId>
<version>${seatunnel.config.shade.version}</version>
</dependency>
- <!--spark-->
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
- </dependency>
<dependency>
<groupId>net.jpountz.lz4</groupId>
<artifactId>lz4</artifactId>
@@ -321,24 +285,6 @@
<version>${play-mailer.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-spark</artifactId>
- <version>${phoenix-spark.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.glassfish.web</groupId>
- <artifactId>javax.servlet.jsp</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.mongodb.spark</groupId>
- <artifactId>mongo-spark-connector_${scala.binary.version}</artifactId>
- <version>${mongo-spark.version}</version>
- </dependency>
-
<dependency>
<groupId>com.101tec</groupId>
<artifactId>zkclient</artifactId>
@@ -363,52 +309,12 @@
<version>${parquet-avro.version}</version>
</dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch-spark-20_${scala.binary.version}</artifactId>
- <version>${elasticsearch-spark.version}</version>
- </dependency>
-
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<version>${clickhouse-jdbc.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hbase.connectors.spark</groupId>
- <artifactId>hbase-spark</artifactId>
- <version>${hbase-spark.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.glassfish.web</groupId>
- <artifactId>javax.servlet.jsp</artifactId>
- </exclusion>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kudu</groupId>
- <artifactId>kudu-spark2_${scala.binary.version}</artifactId>
- <version>${kudu-spark.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.redislabs</groupId>
- <artifactId>spark-redis_${scala.binary.version}</artifactId>
- <version>${spark-redis.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-kafka-0-10_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
-
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
@@ -439,12 +345,6 @@
<version>${hudi.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-bundle_${scala.binary.version}</artifactId>
- <version>${hudi.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
@@ -455,19 +355,6 @@
<artifactId>httpcore</artifactId>
<version>${httpcore.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore-nio</artifactId>
- <version>${httpcore-nio.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpasyncclient</artifactId>
- <version>${httpasyncclient.version}</version>
- </dependency>
-
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
@@ -540,12 +427,6 @@
</exclusion>
</exclusions>
</dependency>
-
- <dependency>
- <groupId>com.pingcap.tispark</groupId>
- <artifactId>tispark-assembly</artifactId>
- <version>${tispark.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
@@ -567,30 +448,6 @@
<version>${config.version}</version>
</dependency>
- <dependency>
- <groupId>org.neo4j</groupId>
- <artifactId>neo4j-connector-apache-spark_${scala.binary.version}</artifactId>
- <version>${neo4j.connector.spark.version}_for_spark_${spark.binary.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.influxdb</groupId>
- <artifactId>influxdb-java</artifactId>
- <version>${influxdb-java.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-core</artifactId>
- <version>${iceberg.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-spark-runtime</artifactId>
- <version>${iceberg.version}</version>
- <scope>provided</scope>
- </dependency>
-
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
diff --git a/seatunnel-apis/seatunnel-api-spark/pom.xml b/seatunnel-apis/seatunnel-api-spark/pom.xml
index 8f4e7ca0a..68be601da 100644
--- a/seatunnel-apis/seatunnel-api-spark/pom.xml
+++ b/seatunnel-apis/seatunnel-api-spark/pom.xml
@@ -45,14 +45,20 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>net.jpountz.lz4</groupId>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/pom.xml
index 553a54dad..ccdafa83e 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/pom.xml
@@ -55,5 +55,45 @@
<module>seatunnel-connector-spark-http</module>
<module>seatunnel-connector-spark-webhook</module>
</modules>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+ </dependencies>
+
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api-spark</artifactId>
+ <scope>provided</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ </dependency>
+ </dependencies>
</project>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/pom.xml
index f9560d02a..c74b9ece4 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/pom.xml
@@ -28,41 +28,29 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-clickhouse</artifactId>
+
+ <properties>
+ <clickhouse.jdbc.version>0.2</clickhouse.jdbc.version>
+ <sshd.scp.version>2.7.0</sshd.scp.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <scope>provided</scope>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
+ <version>${clickhouse.jdbc.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-scp</artifactId>
+ <version>${sshd.scp.version}</version>
</dependency>
-
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>3.2.3</version>
<scope>test</scope>
</dependency>
-
</dependencies>
</project>
\ No newline at end of file
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-console/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-console/pom.xml
index a54b05357..7e6a4a8a9 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-console/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-console/pom.xml
@@ -29,23 +29,4 @@
<artifactId>seatunnel-connector-spark-console</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
-
</project>
\ No newline at end of file
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-doris/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-doris/pom.xml
index 4c701f2e8..84588b8ee 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-doris/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-doris/pom.xml
@@ -28,43 +28,37 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-doris</artifactId>
+
+ <properties>
+ <httpclient.version>4.5.13</httpclient.version>
+ <httpcore.version>4.4.4</httpcore.version>
+ <httpnio.version>4.4.4</httpnio.version>
+ <httpasyncclient.version>4.1.4</httpasyncclient.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
+ <version>${httpcore.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore-nio</artifactId>
+ <version>${httpnio.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpasyncclient</artifactId>
+ <version>${httpasyncclient.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-elasticsearch/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-elasticsearch/pom.xml
index 792a3e8ed..22fc9ed6c 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-elasticsearch/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-elasticsearch/pom.xml
@@ -28,27 +28,16 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-elasticsearch</artifactId>
+
+ <properties>
+ <elasticsearch.version>6.8.3</elasticsearch.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-spark-20_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${elasticsearch.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-email/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-email/pom.xml
index 650b8c605..99e51f8be 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-email/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-email/pom.xml
@@ -28,31 +28,21 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-email</artifactId>
+ <properties>
+ <spoiwo.version>1.8.0</spoiwo.version>
+ <play-mailer.version>7.0.2</play-mailer.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>com.norbitltd</groupId>
<artifactId>spoiwo_${scala.binary.version}</artifactId>
+ <version>${spoiwo.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-mailer_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${play-mailer.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-fake/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-fake/pom.xml
index 8e1223270..5c2483508 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-fake/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-fake/pom.xml
@@ -30,23 +30,6 @@
<artifactId>seatunnel-connector-spark-fake</artifactId>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-feishu/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-feishu/pom.xml
index 039d8da22..5ac937dbe 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-feishu/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-feishu/pom.xml
@@ -28,28 +28,16 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-feishu</artifactId>
+
+ <properties>
+ <httpclient.version>4.5.13</httpclient.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
</dependency>
</dependencies>
</project>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-file/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-file/pom.xml
index 85c3fbd8f..4c35a3417 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-file/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-file/pom.xml
@@ -29,23 +29,4 @@
<artifactId>seatunnel-connector-spark-file</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
-
</project>
\ No newline at end of file
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hbase/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hbase/pom.xml
index 8fe340c44..0ae94a479 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hbase/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hbase/pom.xml
@@ -28,38 +28,24 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-hbase</artifactId>
+ <properties>
+ <hbase.spark.version>1.0.0</hbase.spark.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
</dependency>
-
<dependency>
<groupId>org.apache.hbase.connectors.spark</groupId>
<artifactId>hbase-spark</artifactId>
+ <version>${hbase.spark.version}</version>
<exclusions>
<exclusion>
<groupId>org.codehaus.janino</groupId>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hive/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hive/pom.xml
index 37fda3923..e94bf7f00 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hive/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hive/pom.xml
@@ -29,23 +29,4 @@
<artifactId>seatunnel-connector-spark-hive</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
-
</project>
\ No newline at end of file
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-http/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-http/pom.xml
index 758a7fca6..5998c3a91 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-http/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-http/pom.xml
@@ -29,24 +29,4 @@
<artifactId>seatunnel-connector-spark-http</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
- </dependencies>
-
</project>
\ No newline at end of file
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hudi/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hudi/pom.xml
index 6c3038067..018c06806 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hudi/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-hudi/pom.xml
@@ -28,28 +28,16 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-hudi</artifactId>
+ <properties>
+ <hudi-spark-bundle.version>0.11.1</hudi-spark-bundle.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-spark-bundle_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${hudi-spark-bundle.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-iceberg/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-iceberg/pom.xml
index 678f6d1fd..6b71ff108 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-iceberg/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-iceberg/pom.xml
@@ -28,32 +28,21 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-iceberg</artifactId>
+ <properties>
+ <iceberg.version>0.13.1</iceberg.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-core</artifactId>
+ <version>${iceberg.version}</version>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-spark-runtime</artifactId>
+ <version>${iceberg.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/pom.xml
index 485381fb8..cac85b1f7 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/pom.xml
@@ -28,37 +28,28 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-jdbc</artifactId>
+
+ <properties>
+ <pg.version>42.3.3</pg.version>
+ <mysql.version>8.0.16</mysql.version>
+ </properties>
<dependencies>
<dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_${scala.binary.version}</artifactId>
</dependency>
+
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.version}</version>
+ <scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
+ <version>${pg.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-kafka/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-kafka/pom.xml
index fa39bf9ac..2e24a9eeb 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-kafka/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-kafka/pom.xml
@@ -30,15 +30,10 @@
<artifactId>seatunnel-connector-spark-kafka</artifactId>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-10_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
</dependency>
<dependency>
@@ -46,16 +41,6 @@
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
</dependencies>
</project>
\ No newline at end of file
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-kudu/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-kudu/pom.xml
index 750b015f9..73ecdbb04 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-kudu/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-kudu/pom.xml
@@ -28,25 +28,11 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-kudu</artifactId>
+ <properties>
+ <kudu.version>1.7.0</kudu.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
@@ -55,6 +41,7 @@
<dependency>
<groupId>org.apache.kudu</groupId>
<artifactId>kudu-spark2_${scala.binary.version}</artifactId>
+ <version>${kudu.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-mongodb/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-mongodb/pom.xml
index da7c82ad2..39ab76e25 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-mongodb/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-mongodb/pom.xml
@@ -24,33 +24,16 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-mongodb</artifactId>
+
+ <properties>
+ <mongodb.version>2.2.0</mongodb.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-common</artifactId>
- <version>${project.version}</version>
- </dependency>
-
<dependency>
<groupId>org.mongodb.spark</groupId>
<artifactId>mongo-spark-connector_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${mongodb.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-neo4j/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-neo4j/pom.xml
index 42ce83a6e..355026b6b 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-neo4j/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-neo4j/pom.xml
@@ -28,24 +28,11 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-neo4j</artifactId>
+ <properties>
+ <neo4j.version>4.1.0</neo4j.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -56,6 +43,7 @@
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-connector-apache-spark_${scala.binary.version}</artifactId>
+ <version>${neo4j.version}_for_spark_${spark.binary.2.4.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-phoenix/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-phoenix/pom.xml
index 013bced01..59dc253fc 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-phoenix/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-phoenix/pom.xml
@@ -28,39 +28,33 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-phoenix</artifactId>
+
+ <properties>
+ <phoenix.version>5.0.0-HBase-2.0</phoenix.version>
+ <zkclient.version>0.3</zkclient.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-spark</artifactId>
+ <version>${phoenix.version}</version>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.glassfish.web</groupId>
+ <artifactId>javax.servlet.jsp</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.101tec</groupId>
<artifactId>zkclient</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${zkclient.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-redis/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-redis/pom.xml
index b70120dbd..a08575b92 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-redis/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-redis/pom.xml
@@ -28,33 +28,18 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-redis</artifactId>
-
+ <properties>
+ <redis.version>2.6.0</redis.version>
+ </properties>
<dependencies>
<dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_${scala.binary.version}</artifactId>
</dependency>
-
<dependency>
<groupId>com.redislabs</groupId>
<artifactId>spark-redis_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ <version>${redis.version}</version>
</dependency>
</dependencies>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-socket/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-socket/pom.xml
index 87a3f84c0..8e951ed3b 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-socket/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-socket/pom.xml
@@ -30,23 +30,6 @@
<artifactId>seatunnel-connector-spark-socket</artifactId>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-tidb/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-tidb/pom.xml
index 9955b057a..c9b8c7d7b 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-tidb/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-tidb/pom.xml
@@ -28,25 +28,12 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-connector-spark-tidb</artifactId>
+
+ <properties>
+ <tis.version>2.4.1</tis.version>
+ </properties>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
@@ -55,6 +42,7 @@
<dependency>
<groupId>com.pingcap.tispark</groupId>
<artifactId>tispark-assembly</artifactId>
+ <version>${tis.version}</version>
<exclusions>
<exclusion>
<groupId>mysql</groupId>
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-webhook/pom.xml b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-webhook/pom.xml
index 0a7a9246c..104b1c6da 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-webhook/pom.xml
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-webhook/pom.xml
@@ -30,28 +30,10 @@
<artifactId>seatunnel-connector-spark-webhook</artifactId>
<dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
</dependency>
-
</dependencies>
</project>
\ No newline at end of file
diff --git a/seatunnel-core/seatunnel-core-spark/pom.xml b/seatunnel-core/seatunnel-core-spark/pom.xml
index c8b6037a6..fa92953dc 100644
--- a/seatunnel-core/seatunnel-core-spark/pom.xml
+++ b/seatunnel-core/seatunnel-core-spark/pom.xml
@@ -56,14 +56,20 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
</dependency>
<dependency>
@@ -151,7 +157,7 @@
<argument>build</argument>
<argument>--no-cache</argument>
<argument>--build-arg</argument>
- <argument>SPARK_VERSION=${spark.version}</argument>
+ <argument>SPARK_VERSION=${spark.2.4.0.version}</argument>
<argument>--build-arg</argument>
<argument>HADOOP_VERSION=${hadoop.binary.version}</argument>
<argument>-t</argument>
@@ -180,7 +186,7 @@
<argument>build</argument>
<argument>--no-cache</argument>
<argument>--build-arg</argument>
- <argument>SPARK_VERSION=${spark.version}</argument>
+ <argument>SPARK_VERSION=${spark.2.4.0.version}</argument>
<argument>--build-arg</argument>
<argument>HADOOP_VERSION=${hadoop.binary.version}</argument>
<argument>--push</argument>
diff --git a/seatunnel-core/seatunnel-spark-starter/pom.xml b/seatunnel-core/seatunnel-spark-starter/pom.xml
index 9fa398bb8..e894a6a15 100644
--- a/seatunnel-core/seatunnel-spark-starter/pom.xml
+++ b/seatunnel-core/seatunnel-spark-starter/pom.xml
@@ -56,14 +56,20 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
</dependency>
<dependency>
diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml b/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml
index b0d88aba4..90dbe5c17 100644
--- a/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml
+++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml
@@ -61,29 +61,35 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
+ <version>${spark.2.4.0.version}</version>
<scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
+ <version>${spark.2.4.0.version}</version>
<scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
+ <version>${spark.2.4.0.version}</version>
<scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
+ <version>${spark.2.4.0.version}</version>
<scope>${spark.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.glassfish.web</groupId>
+ <artifactId>javax.servlet.jsp</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
diff --git a/seatunnel-examples/seatunnel-spark-examples/pom.xml b/seatunnel-examples/seatunnel-spark-examples/pom.xml
index 49c634f0b..0254e3e92 100644
--- a/seatunnel-examples/seatunnel-spark-examples/pom.xml
+++ b/seatunnel-examples/seatunnel-spark-examples/pom.xml
@@ -65,28 +65,28 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
+ <version>${spark.2.4.0.version}</version>
<scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
+ <version>${spark.2.4.0.version}</version>
<scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
+ <version>${spark.2.4.0.version}</version>
<scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
+ <version>${spark.2.4.0.version}</version>
<scope>${spark.scope}</scope>
</dependency>
<dependency>
diff --git a/seatunnel-transforms/seatunnel-transforms-spark/pom.xml b/seatunnel-transforms/seatunnel-transforms-spark/pom.xml
index 8367d8bdc..daa8cd9c0 100644
--- a/seatunnel-transforms/seatunnel-transforms-spark/pom.xml
+++ b/seatunnel-transforms/seatunnel-transforms-spark/pom.xml
@@ -39,5 +39,45 @@
<module>seatunnel-transform-spark-nulltf</module>
<module>seatunnel-transform-spark-null-rate</module>
</modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api-spark</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api-spark</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ </dependency>
+ </dependencies>
</project>
diff --git a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-json/pom.xml b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-json/pom.xml
index 0a8cba314..04e89ee98 100644
--- a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-json/pom.xml
+++ b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-json/pom.xml
@@ -28,25 +28,4 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-transform-spark-json</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
-
-
</project>
diff --git a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-null-rate/pom.xml b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-null-rate/pom.xml
index 95179c970..dada4d4d3 100644
--- a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-null-rate/pom.xml
+++ b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-null-rate/pom.xml
@@ -28,23 +28,4 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-transform-spark-null-rate</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
</project>
diff --git a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-nulltf/pom.xml b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-nulltf/pom.xml
index 6d4d7412a..50fcd2756 100644
--- a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-nulltf/pom.xml
+++ b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-nulltf/pom.xml
@@ -28,24 +28,4 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-transform-spark-nulltf</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
-
</project>
diff --git a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-replace/pom.xml b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-replace/pom.xml
index d5a2fb454..949225c2a 100644
--- a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-replace/pom.xml
+++ b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-replace/pom.xml
@@ -28,23 +28,4 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-transform-spark-replace</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
</project>
diff --git a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-split/pom.xml b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-split/pom.xml
index d9feed469..6ff13f468 100644
--- a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-split/pom.xml
+++ b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-split/pom.xml
@@ -28,24 +28,4 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-transform-spark-split</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
-
</project>
diff --git a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-sql/pom.xml b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-sql/pom.xml
index d00b8943a..d7e218335 100644
--- a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-sql/pom.xml
+++ b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-sql/pom.xml
@@ -28,25 +28,4 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-transform-spark-sql</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
-
-
</project>
diff --git a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-uuid/pom.xml b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-uuid/pom.xml
index 82f1b6d28..b8c65fe02 100644
--- a/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-uuid/pom.xml
+++ b/seatunnel-transforms/seatunnel-transforms-spark/seatunnel-transform-spark-uuid/pom.xml
@@ -28,23 +28,4 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>seatunnel-transform-spark-uuid</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-spark</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
</project>
diff --git a/seatunnel-translation/seatunnel-translation-spark/pom.xml b/seatunnel-translation/seatunnel-translation-spark/pom.xml
index a6c777881..f2fe73e2b 100644
--- a/seatunnel-translation/seatunnel-translation-spark/pom.xml
+++ b/seatunnel-translation/seatunnel-translation-spark/pom.xml
@@ -29,4 +29,30 @@
<module>seatunnel-translation-spark-2.4</module>
<module>seatunnel-translation-spark-common</module>
</modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <!--spark-->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.2.4.0.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
</project>
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/pom.xml b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/pom.xml
index 850812a1a..6eb4029b5 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/pom.xml
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/pom.xml
@@ -25,12 +25,6 @@
<artifactId>seatunnel-translation-spark-2.4</artifactId>
- <properties>
- <spark.version>2.4.0</spark.version>
- <scala.binary.version>2.11</scala.binary.version>
- <spark.scope>provided</spark.scope>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.seatunnel</groupId>
@@ -42,22 +36,16 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
</dependency>
</dependencies>
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/pom.xml b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/pom.xml
index c05c33d78..3388ca15c 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/pom.xml
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/pom.xml
@@ -36,22 +36,16 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.scope}</scope>
</dependency>
</dependencies>
</project>
\ No newline at end of file