You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by di...@apache.org on 2022/09/22 03:28:31 UTC
[incubator-seatunnel] branch dev updated: [Binary-Package]Add script to automatically download plugins (#2831)
This is an automated email from the ASF dual-hosted git repository.
dijiekstra pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 853d91492 [Binary-Package]Add script to automatically download plugins (#2831)
853d91492 is described below
commit 853d914922ee335515bd562dd605f13bb407234d
Author: Kirs <ki...@apache.org>
AuthorDate: Thu Sep 22 11:28:25 2022 +0800
[Binary-Package]Add script to automatically download plugins (#2831)
* [Binary-Package]Add script to automatically download plugins
Binary package releases no longer package connector plugins. The declaration -Prelease needs to be displayed, otherwise it is still included.
Provides a script to automatically download plugins (we use maven-wrapper)
* fix maven-wrapper version
* modify plugin_config name
---
bin/install-plugin.sh | 83 ++++++++++++++++++++
config/plugin_config | 92 +++++++++++++++++++++++
docs/en/start/local.mdx | 17 ++++-
seatunnel-dist/release-docs/LICENSE | 1 +
seatunnel-dist/release-docs/NOTICE | 12 ++-
seatunnel-dist/src/main/assembly/assembly-bin.xml | 54 +++++--------
6 files changed, 220 insertions(+), 39 deletions(-)
diff --git a/bin/install-plugin.sh b/bin/install-plugin.sh
new file mode 100644
index 000000000..c4afa87e4
--- /dev/null
+++ b/bin/install-plugin.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#This script is used to download the connector plug-ins required during the running process.
+#All are downloaded by default. You can also choose what you need.
+#You only need to configure the plug-in name in config/plugin_config.
+
+# connector default version is 2.2.0, you can also choose a custom version. eg: 2.1.2: sh install-plugin.sh 2.1.2
+version=2.2.0
+
+if [ -n "$1" ]; then
+ version="$1"
+fi
+
+echo "Install SeaTunnel connectors plugins, usage version is $version $1"
+
+if [ ! -d connectors ];
+ then
+ mkdir connectors
+ echo "create connectors directory"
+fi
+if [ ! -d connectors/flink-sql ];
+ then
+ mkdir connectors/flink-sql
+ echo "create flink-sql connectors directory"
+fi
+if [ ! -d connectors/flink ];
+ then
+ mkdir connectors/flink
+ echo "create flink connectors directory"
+fi
+if [ ! -d connectors/spark ];
+ then
+ mkdir connectors/spark
+ echo "create spark connectors directory"
+fi
+if [ ! -d connectors/seatunnel ];
+ then
+ mkdir connectors/seatunnel
+ echo "create seatunnel connectors directory"
+fi
+
+path=flink-sql
+
+while read line; do
+ if [ ${line:0:1} != "-" ] && [ ${line:0:1} != "#" ]
+ then
+ echo "install connector : " $line
+ ./mvnw dependency:get -DgroupId=org.apache.seatunnel -DartifactId=${line} -Dversion=${version} -Ddest=connectors/${path}
+ fi
+ if [ "$line" = "--flink-sql-connectors--" ]
+ then
+ path=flink-sql
+ fi
+ if [ "$line" = "--flink-connectors--" ]
+ then
+ path=flink
+ fi
+ if [ "$line" = "--spark-connectors--" ]
+ then
+ path=spark
+ fi
+ if [ "$line" = "--connectors-v2--" ]
+ then
+ path=seatunnel
+ fi
+
+done < config/plugin_config
\ No newline at end of file
diff --git a/config/plugin_config b/config/plugin_config
new file mode 100644
index 000000000..dec806026
--- /dev/null
+++ b/config/plugin_config
@@ -0,0 +1,92 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# This mapping is used to resolve the Jar package name without version (or call artifactId)
+#
+# corresponding to the module in the user Config, helping SeaTunnel to load the correct Jar package.
+# Don't modify the delimiter " -- ", just select the plugin you need
+--flink-sql-connectors--
+flink-sql-connector-elasticsearch-6
+flink-sql-connector-elasticsearch-7
+flink-sql-connector-jdbc
+flink-sql-connector-kafka
+--flink-connectors--
+seatunnel-connector-flink-assert
+seatunnel-connector-flink-druid
+seatunnel-connector-flink-fake
+seatunnel-connector-flink-file
+seatunnel-connector-flink-influxdb
+seatunnel-connector-flink-jdbc
+seatunnel-connector-flink-kafka
+seatunnel-connector-flink-socket
+seatunnel-connector-flink-http
+seatunnel-connector-flink-clickhouse
+seatunnel-connector-flink-console
+seatunnel-connector-flink-doris
+seatunnel-connector-flink-elasticsearch7
+--spark-connectors--
+seatunnel-connector-spark-elasticsearch
+seatunnel-connector-spark-fake
+seatunnel-connector-spark-feishu
+seatunnel-connector-spark-file
+seatunnel-connector-spark-hbase
+seatunnel-connector-spark-hive
+seatunnel-connector-spark-http
+seatunnel-connector-spark-hudi
+seatunnel-connector-spark-iceberg
+seatunnel-connector-spark-jdbc
+seatunnel-connector-spark-kafka
+seatunnel-connector-spark-kudu
+seatunnel-connector-spark-mongodb
+seatunnel-connector-spark-neo4j
+seatunnel-connector-spark-phoenix
+seatunnel-connector-spark-redis
+seatunnel-connector-spark-socket
+seatunnel-connector-spark-tidb
+seatunnel-connector-spark-clickhouse
+seatunnel-connector-spark-console
+seatunnel-connector-spark-doris
+seatunnel-connector-spark-email
+--connectors-v2--
+connector-fake
+connector-console
+onnector-assert
+connector-kafka
+connector-http-base
+connector-http-feishu
+connector-hive
+connector-clickhouse
+connector-jdbc
+connector-kudu
+connector-email
+connector-file-hadoop
+connector-file-local
+connector-file-oss
+connector-pulsar
+connector-hudi
+connector-dingtalk
+connector-elasticsearch
+connector-iotdb
+connector-neo4j
+connector-file-ftp
+connector-socket
+connector-redis
+connector-datahub
+connector-sentry
+connector-mongodb
+connector-iceberg
+--end--
\ No newline at end of file
diff --git a/docs/en/start/local.mdx b/docs/en/start/local.mdx
index 8f4499390..52be5a0c0 100644
--- a/docs/en/start/local.mdx
+++ b/docs/en/start/local.mdx
@@ -29,9 +29,24 @@ export version="2.1.0"
wget "https://archive.apache.org/dist/incubator/seatunnel/${version}/apache-seatunnel-incubating-${version}-bin.tar.gz"
tar -xzvf "apache-seatunnel-incubating-${version}-bin.tar.gz"
```
-
<!-- TODO: We should add example module as quick start which is no need for install Spark or Flink -->
+## Install connectors plugin
+Since 2.2.0, the binary package does not provide connector dependencies by default, so when using it for the first time, we need to execute the following command to install the connector: (Of course, you can also manually download the connector from [Apache Maven Repository](https://repo. maven.apache.org/maven2/org/apache/seatunnel/ to download, then manually move to the connectors directory).
+```bash
+sh bin/install_plugin.sh
+````
+If you need to specify the version of the connector, take 2.2.0 as an example, we need to execute
+```bash
+sh bin/install_plugin.sh 2.2.0
+````
+Usually we don't need all the connector plugins, so you can specify the plugins you need by configuring `config/plugin_config`, for example, you only need the `flink-assert` plugin, then you can modify plugin.properties as
+````plugin_config
+--flink-connectors--
+seatunnel-connector-flink-assert
+--end--
+```bash
+
## Run SeaTunnel Application
**Configure SeaTunnel**: Change the setting in `config/seatunnel-env.sh`, it is base on the path your engine install at [prepare step two](#prepare).
diff --git a/seatunnel-dist/release-docs/LICENSE b/seatunnel-dist/release-docs/LICENSE
index 90f13fdf2..3a7532f41 100644
--- a/seatunnel-dist/release-docs/LICENSE
+++ b/seatunnel-dist/release-docs/LICENSE
@@ -300,6 +300,7 @@ The text of each license is the standard Apache 2.0 license.
(The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.7.1 - https://github.com/xerial/snappy-java)
(Apache-2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.8.3 - https://github.com/xerial/snappy-java)
(Apache-2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.1.3 - https://github.com/xerial/snappy-java)
+ (Apache-2.0) maven-wrapper (org.apache.maven:maven-wrapper:3.8.4 https://maven.apache.org/wrapper/)
========================================================================
MOZILLA PUBLIC LICENSE License
diff --git a/seatunnel-dist/release-docs/NOTICE b/seatunnel-dist/release-docs/NOTICE
index e93707e4c..c79a9b11b 100644
--- a/seatunnel-dist/release-docs/NOTICE
+++ b/seatunnel-dist/release-docs/NOTICE
@@ -615,13 +615,19 @@ http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
========================================================================
- NOTICE
+Apache Maven Wrapper NOTICE
========================================================================
+This product contains code form the Apache Maven Wrapper Project:
+Apache Maven Wrapper
+Copyright 2013-2022 The Apache Software Foundation
-========================================================================
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
- NOTICE
+The original idea and initial implementation of the maven-wrapper module is derived
+from the Gradle Wrapper which was written originally by Hans Dockter and Adam Murdoch.
+Copyright 2007 the original author or authors.
========================================================================
\ No newline at end of file
diff --git a/seatunnel-dist/src/main/assembly/assembly-bin.xml b/seatunnel-dist/src/main/assembly/assembly-bin.xml
index 125e1a176..2aafdd6ae 100644
--- a/seatunnel-dist/src/main/assembly/assembly-bin.xml
+++ b/seatunnel-dist/src/main/assembly/assembly-bin.xml
@@ -34,6 +34,7 @@
<exclude>**/.project</exclude>
<exclude>**/.settings/**</exclude>
<exclude>lib/**</exclude>
+ <exclude>**/.DS_Store</exclude>
</excludes>
<includes>
@@ -81,6 +82,24 @@
</includes>
<outputDirectory>.</outputDirectory>
</fileSet>
+ <!-- maven wrapper tools -->
+ <fileSet>
+ <directory>${basedir}/.././</directory>
+ <includes>
+ <include>mvnw</include>
+ <include>mvnw.cmd</include>
+ </includes>
+ <fileMode>0755</fileMode>
+ <outputDirectory>.</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/.././</directory>
+ <includes>
+ <include>.mvn/wrapper/maven-wrapper.properties</include>
+ </includes>
+ <fileMode>0755</fileMode>
+ <outputDirectory>.</outputDirectory>
+ </fileSet>
</fileSets>
<files>
@@ -108,40 +127,5 @@
<outputDirectory>/lib</outputDirectory>
<scope>provided</scope>
</dependencySet>
-
- <!-- ============ Connectors Jars ============ -->
- <!-- Flink connectors -->
- <dependencySet>
- <useProjectArtifact>false</useProjectArtifact>
- <useTransitiveDependencies>true</useTransitiveDependencies>
- <unpack>false</unpack>
- <includes>
- <include>org.apache.seatunnel:seatunnel-connector-flink-*:jar</include>
- </includes>
- <outputDirectory>/connectors/flink</outputDirectory>
- <scope>provided</scope>
- </dependencySet>
- <!-- Flink SQL connectors -->
- <dependencySet>
- <useProjectArtifact>false</useProjectArtifact>
- <useTransitiveDependencies>true</useTransitiveDependencies>
- <unpack>false</unpack>
- <includes>
- <include>org.apache.seatunnel:flink-sql-connector-*:jar</include>
- </includes>
- <outputDirectory>/connectors/flink-sql</outputDirectory>
- <scope>provided</scope>
- </dependencySet>
- <!-- Spark connectors -->
- <dependencySet>
- <useProjectArtifact>false</useProjectArtifact>
- <useTransitiveDependencies>true</useTransitiveDependencies>
- <unpack>false</unpack>
- <includes>
- <include>org.apache.seatunnel:seatunnel-connector-spark-*:jar</include>
- </includes>
- <outputDirectory>/connectors/spark</outputDirectory>
- <scope>provided</scope>
- </dependencySet>
</dependencySets>
</assembly>