You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ga...@apache.org on 2022/10/09 14:30:56 UTC
[incubator-seatunnel] branch dev updated: [Engine] [ConfigFile] Add config file into binary package. (#3033)
This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 5b9033ecd [Engine] [ConfigFile] Add config file into binary package. (#3033)
5b9033ecd is described below
commit 5b9033ecdadf8e6e2d680c9d45069def8baa3dc0
Author: Hisoka <fa...@qq.com>
AuthorDate: Sun Oct 9 22:30:51 2022 +0800
[Engine] [ConfigFile] Add config file into binary package. (#3033)
* [Engine] [ConfigFile] Add config file into binary package.
---
.../seatunnel-starter/src/main/bin/seatunnel-cluster.sh | 12 +++++++++++-
.../seatunnel-starter/src/main/bin/seatunnel.sh | 16 +++++++++++++++-
seatunnel-dist/src/main/assembly/assembly-bin-ci.xml | 9 +++++++++
seatunnel-dist/src/main/assembly/assembly-bin.xml | 9 +++++++++
.../common/container/seatunnel/SeaTunnelContainer.java | 1 +
.../apache/seatunnel/engine/e2e/SeaTunnelContainer.java | 5 +++++
.../org/apache/seatunnel/engine/common/Constant.java | 4 ++--
.../seatunnel/engine/common/config/ConfigProvider.java | 4 ++--
.../resources/{seatunnel-default.yaml => seatunnel.yaml} | 0
9 files changed, 54 insertions(+), 6 deletions(-)
diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
index 228d07d01..ebef64f22 100755
--- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
+++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
@@ -50,4 +50,14 @@ else
args=$@
fi
-java -cp ${APP_JAR} ${APP_MAIN} ${args}
+set +u
+# SeaTunnel Engine Config
+if [ -z $HAZELCAST_CONFIG ]; then
+ HAZELCAST_CONFIG=${CONF_DIR}/hazelcast.yaml
+fi
+
+if [ -z $SEATUNNEL_CONFIG ]; then
+ SEATUNNEL_CONFIG=${CONF_DIR}/seatunnel.yaml
+fi
+
+java -Dseatunnel.config=${SEATUNNEL_CONFIG} -Dhazelcast.config=${HAZELCAST_CONFIG} -cp ${APP_JAR} ${APP_MAIN} ${args}
diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
index 601bfd4e7..89f5b4674 100755
--- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
+++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
@@ -50,4 +50,18 @@ else
args=$@
fi
-java -cp ${APP_JAR} ${APP_MAIN} ${args}
+set +u
+# SeaTunnel Engine Config
+if [ -z $HAZELCAST_CLIENT_CONFIG ]; then
+ HAZELCAST_CLIENT_CONFIG=${CONF_DIR}/hazelcast-client.yaml
+fi
+
+if [ -z $HAZELCAST_CONFIG ]; then
+ HAZELCAST_CONFIG=${CONF_DIR}/hazelcast.yaml
+fi
+
+if [ -z $SEATUNNEL_CONFIG ]; then
+ SEATUNNEL_CONFIG=${CONF_DIR}/seatunnel.yaml
+fi
+
+java -Dhazelcast.client.config=${HAZELCAST_CLIENT_CONFIG} -Dseatunnel.config=${SEATUNNEL_CONFIG} -Dhazelcast.config=${HAZELCAST_CONFIG} -cp ${APP_JAR} ${APP_MAIN} ${args}
diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
index c5d3d8c98..f2f949471 100644
--- a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
+++ b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
@@ -84,6 +84,15 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
+ <!--Engine Config File-->
+ <fileSet>
+ <directory>../seatunnel-engine/seatunnel-engine-common/src/main/resources</directory>
+ <includes>
+ <include>*.yaml</include>
+ </includes>
+ <outputDirectory>/config</outputDirectory>
+ <fileMode>0644</fileMode>
+ </fileSet>
<!--Licenses And NOTICE-->
<fileSet>
<directory>release-docs</directory>
diff --git a/seatunnel-dist/src/main/assembly/assembly-bin.xml b/seatunnel-dist/src/main/assembly/assembly-bin.xml
index e9cdc6551..1f8af0454 100644
--- a/seatunnel-dist/src/main/assembly/assembly-bin.xml
+++ b/seatunnel-dist/src/main/assembly/assembly-bin.xml
@@ -85,6 +85,15 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
+ <!--Engine Config File-->
+ <fileSet>
+ <directory>../seatunnel-engine/seatunnel-engine-common/src/main/resources</directory>
+ <includes>
+ <include>*.yaml</include>
+ </includes>
+ <outputDirectory>/config</outputDirectory>
+ <fileMode>0644</fileMode>
+ </fileSet>
<!--Licenses And NOTICE-->
<fileSet>
<directory>release-docs</directory>
diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
index 1ce6fb22d..e53942e43 100644
--- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
+++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
@@ -61,6 +61,7 @@ public class SeaTunnelContainer extends AbstractTestContainer {
server.start();
copySeaTunnelStarter(server);
// execute extra commands
+ // TODO copy config file
executeExtraCommands(server);
}
diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java
index 4c3938a40..7c5bafab5 100644
--- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java
+++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java
@@ -52,6 +52,8 @@ public abstract class SeaTunnelContainer {
private static final String PLUGIN_MAPPING_FILE = "plugin-mapping.properties";
private static final String SEATUNNEL_BIN = Paths.get(SEATUNNEL_HOME, "bin").toString();
private static final String SEATUNNEL_LIB = Paths.get(SEATUNNEL_HOME, "lib").toString();
+ private static final String SEATUNNEL_CONFIG = Paths.get(SEATUNNEL_HOME, "config").toString();
+
private static final String SEATUNNEL_CONNECTORS = Paths.get(SEATUNNEL_HOME, "connectors").toString();
private static final String CLIENT_SHELL = "seatunnel.sh";
private static final String SERVER_SHELL = "seatunnel-cluster.sh";
@@ -78,6 +80,9 @@ public abstract class SeaTunnelContainer {
mountMapping.put(PROJECT_ROOT_PATH + "/seatunnel-core/seatunnel-starter/target/seatunnel-starter.jar",
Paths.get(SEATUNNEL_LIB, "seatunnel-starter.jar").toString());
+ mountMapping.put(PROJECT_ROOT_PATH + "/seatunnel-engine/seatunnel-engine-common/src/main/resources/",
+ SEATUNNEL_CONFIG);
+
// copy bin
mountMapping.put(PROJECT_ROOT_PATH + "/seatunnel-core/seatunnel-starter/src/main/bin/",
Paths.get(SEATUNNEL_BIN).toString());
diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/Constant.java b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/Constant.java
index ba02b7f4f..37040a6ef 100644
--- a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/Constant.java
+++ b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/Constant.java
@@ -30,11 +30,11 @@ public class Constant {
*/
public static final int DEFAULT_SEATUNNEL_MULTICAST_PORT = 53326;
- public static final String SYSPROP_SEATUNNEL_CONFIG = "hazelcast.seatunnel.config";
+ public static final String SYSPROP_SEATUNNEL_CONFIG = "seatunnel.config";
public static final String HAZELCAST_SEATUNNEL_CONF_FILE_PREFIX = "seatunnel";
- public static final String HAZELCAST_SEATUNNEL_DEFAULT_YAML = "seatunnel-default.yaml";
+ public static final String HAZELCAST_SEATUNNEL_DEFAULT_YAML = "seatunnel.yaml";
public static final int OPERATION_RETRY_TIME = 5;
diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/ConfigProvider.java b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/ConfigProvider.java
index 591afc0ec..bb99048ce 100644
--- a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/ConfigProvider.java
+++ b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/ConfigProvider.java
@@ -74,7 +74,7 @@ public final class ConfigProvider {
YamlClientConfigLocator yamlConfigLocator = new YamlClientConfigLocator();
if (yamlConfigLocator.locateFromSystemProperty()) {
- // 1. Try loading config if provided in system property and it is an YAML file
+ // 1. Try loading config if provided in system property, and it is an YAML file
config = new YamlClientConfigBuilder(yamlConfigLocator.getIn()).build();
} else if (yamlConfigLocator.locateInWorkDirOrOnClasspath()) {
// 2. Try loading YAML config from the working directory or from the classpath
@@ -95,7 +95,7 @@ public final class ConfigProvider {
YamlConfigLocator yamlConfigLocator = new YamlConfigLocator();
if (yamlConfigLocator.locateFromSystemProperty()) {
- // 1. Try loading config if provided in system property and it is an YAML file
+ // 1. Try loading config if provided in system property, and it is an YAML file
config = new YamlConfigBuilder(yamlConfigLocator.getIn()).setProperties(properties).build();
} else if (yamlConfigLocator.locateInWorkDirOrOnClasspath()) {
// 2. Try loading YAML config from the working directory or from the classpath
diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel-default.yaml b/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml
similarity index 100%
rename from seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel-default.yaml
rename to seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml