You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ga...@apache.org on 2023/03/16 05:44:54 UTC
[incubator-seatunnel] branch dev updated: [Bug] JVM parameters distinguish between client and server (#4297)
This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new de9856787 [Bug] JVM parameters distinguish between client and server (#4297)
de9856787 is described below
commit de98567877f04fbb2e6f72eae1970fc45de8e164
Author: monster <60...@users.noreply.github.com>
AuthorDate: Thu Mar 16 13:44:48 2023 +0800
[Bug] JVM parameters distinguish between client and server (#4297)
* [bug] JVM parameters distinguish between client and server
---
config/jvm_client_options | 108 +++++++++++++++++++++
.../src/main/bin/seatunnel-cluster.sh | 2 +-
.../seatunnel-starter/src/main/bin/seatunnel.sh | 2 +-
.../starter/seatunnel/jvm/JvmOptionsParser.java | 44 ++++-----
.../src/test/resources/jvm_client_options | 108 +++++++++++++++++++++
5 files changed, 237 insertions(+), 27 deletions(-)
diff --git a/config/jvm_client_options b/config/jvm_client_options
new file mode 100644
index 000000000..2c7c97bb9
--- /dev/null
+++ b/config/jvm_client_options
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms128m
+-Xmx512m
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+#8-13:-XX:+UseConcMarkSweepGC
+#8-13:-XX:CMSInitiatingOccupancyFraction=75
+#8-13:-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1 GC is only supported on JDK version 10 or later
+# to use G1GC, uncomment the next two lines and update the version on the
+# following three lines to your version of the JDK
+# 10-13:-XX:-UseConcMarkSweepGC
+# 10-13:-XX:-UseCMSInitiatingOccupancyOnly
+#14-:-XX:+UseG1GC
+#14-:-XX:G1ReservePercent=25
+#14-:-XX:InitiatingHeapOccupancyPercent=30
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+# -XX:+AlwaysPreTouch
+
+## basic
+
+# explicitly set the stack size
+# -Xss1m
+
+# turn off a JDK optimization that throws away stack traces for common
+# exceptions because stack traces are important for debugging
+# -XX:-OmitStackTraceInFastThrow
+
+# enable helpful NullPointerExceptions (https://openjdk.java.net/jeps/358), if
+# they are supported
+# 14-:-XX:+ShowCodeDetailsInExceptionMessages
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+# -XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+# ${heap.dump.path}
+
+# specify an alternative path for JVM fatal error logs
+# ${error.file}
+
+## JDK 8 GC logging
+
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:${loggc}
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=/data/seatunnel/
\ No newline at end of file
diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
index f8945677f..3ae22cb24 100755
--- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
+++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
@@ -88,7 +88,7 @@ CLASS_PATH=${APP_DIR}/lib/*:${APP_JAR}
ST_TMPDIR=`java -cp ${CLASS_PATH} org.apache.seatunnel.core.starter.seatunnel.jvm.TempDirectory`
# The JVM options parser produces the final JVM options to start seatunnel-engine.
-JVM_OPTIONS=`java -cp ${CLASS_PATH} org.apache.seatunnel.core.starter.seatunnel.jvm.JvmOptionsParser ${CONF_DIR}`
+JVM_OPTIONS=`java -cp ${CLASS_PATH} org.apache.seatunnel.core.starter.seatunnel.jvm.JvmOptionsParser ${CONF_DIR}/jvm_options`
JAVA_OPTS="${JAVA_OPTS} ${JVM_OPTIONS//\$\{loggc\}/${ST_TMPDIR}}"
echo "JAVA_OPTS:" ${JAVA_OPTS}
diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
index d8b5a9a1f..f4a062696 100755
--- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
+++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
@@ -96,7 +96,7 @@ CLASS_PATH=${APP_DIR}/lib/*:${APP_JAR}
ST_TMPDIR=`java -cp ${CLASS_PATH} org.apache.seatunnel.core.starter.seatunnel.jvm.TempDirectory`
# The JVM options parser produces the final JVM options to start seatunnel-engine.
-JVM_OPTIONS=`java -cp ${CLASS_PATH} org.apache.seatunnel.core.starter.seatunnel.jvm.JvmOptionsParser ${CONF_DIR}`
+JVM_OPTIONS=`java -cp ${CLASS_PATH} org.apache.seatunnel.core.starter.seatunnel.jvm.JvmOptionsParser ${CONF_DIR}/jvm_client_options`
JAVA_OPTS="${JAVA_OPTS} ${JVM_OPTIONS//\$\{loggc\}/${ST_TMPDIR}}"
echo "JAVA_OPTS:" ${JAVA_OPTS}
diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/jvm/JvmOptionsParser.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/jvm/JvmOptionsParser.java
index 6543f61ac..627316dae 100644
--- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/jvm/JvmOptionsParser.java
+++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/jvm/JvmOptionsParser.java
@@ -65,32 +65,26 @@ final class JvmOptionsParser {
@SneakyThrows
List<String> readJvmOptionsFiles(final Path config) {
- final ArrayList<Path> jvmOptionsFiles = new ArrayList<>();
- jvmOptionsFiles.add(config.resolve("jvm_options"));
-
final List<String> jvmOptions = new ArrayList<>();
-
- for (final Path jvmOptionsFile : jvmOptionsFiles) {
- final SortedMap<Integer, String> invalidLines = new TreeMap<>();
- try (InputStream is = Files.newInputStream(jvmOptionsFile);
- Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
- BufferedReader br = new BufferedReader(reader)) {
- parse(
- JavaVersion.majorVersion(JavaVersion.CURRENT),
- br,
- jvmOptions::add,
- invalidLines::put);
- }
- if (!invalidLines.isEmpty()) {
- throw new JvmOptionsParserException(
- CommonErrorCode.IMPROPERLY_FORMATTED_JVM_OPTION,
- String.format(
- Locale.ROOT,
- "encountered [%d] error%s parsing [%s]",
- invalidLines.size(),
- invalidLines.size() == 1 ? "" : "s",
- jvmOptionsFile));
- }
+ final SortedMap<Integer, String> invalidLines = new TreeMap<>();
+ try (InputStream is = Files.newInputStream(config);
+ Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
+ BufferedReader br = new BufferedReader(reader)) {
+ parse(
+ JavaVersion.majorVersion(JavaVersion.CURRENT),
+ br,
+ jvmOptions::add,
+ invalidLines::put);
+ }
+ if (!invalidLines.isEmpty()) {
+ throw new JvmOptionsParserException(
+ CommonErrorCode.IMPROPERLY_FORMATTED_JVM_OPTION,
+ String.format(
+ Locale.ROOT,
+ "encountered [%d] error%s parsing [%s]",
+ invalidLines.size(),
+ invalidLines.size() == 1 ? "" : "s",
+ config));
}
return jvmOptions;
}
diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/jvm_client_options b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/jvm_client_options
new file mode 100644
index 000000000..507f9b8f6
--- /dev/null
+++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/jvm_client_options
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+#-Xms4g
+#-Xmx4g
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+#8-13:-XX:+UseConcMarkSweepGC
+#8-13:-XX:CMSInitiatingOccupancyFraction=75
+#8-13:-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1 GC is only supported on JDK version 10 or later
+# to use G1GC, uncomment the next two lines and update the version on the
+# following three lines to your version of the JDK
+# 10-13:-XX:-UseConcMarkSweepGC
+# 10-13:-XX:-UseCMSInitiatingOccupancyOnly
+#14-:-XX:+UseG1GC
+#14-:-XX:G1ReservePercent=25
+#14-:-XX:InitiatingHeapOccupancyPercent=30
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+# -XX:+AlwaysPreTouch
+
+## basic
+
+# explicitly set the stack size
+# -Xss1m
+
+# turn off a JDK optimization that throws away stack traces for common
+# exceptions because stack traces are important for debugging
+# -XX:-OmitStackTraceInFastThrow
+
+# enable helpful NullPointerExceptions (https://openjdk.java.net/jeps/358), if
+# they are supported
+# 14-:-XX:+ShowCodeDetailsInExceptionMessages
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+# -XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+# ${heap.dump.path}
+
+# specify an alternative path for JVM fatal error logs
+# ${error.file}
+
+## JDK 8 GC logging
+
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:${loggc}
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=/data/seatunnel/
\ No newline at end of file