You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ki...@apache.org on 2021/07/29 10:32:06 UTC
[dolphinscheduler] branch 1.3.7-prepare updated:
[1.3.7-prepare][Improvement] Impove text description,
OSUtilsTest and reduce heap memory for create/upgrade-dolphinscheduler.sh
(#5890)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch 1.3.7-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/1.3.7-prepare by this push:
new cb01b0d [1.3.7-prepare][Improvement] Impove text description, OSUtilsTest and reduce heap memory for create/upgrade-dolphinscheduler.sh (#5890)
cb01b0d is described below
commit cb01b0deb1c5966e3c7d3395f6acd70a784f6b92
Author: Shiwen Cheng <ch...@gmail.com>
AuthorDate: Thu Jul 29 18:31:57 2021 +0800
[1.3.7-prepare][Improvement] Impove text description, OSUtilsTest and reduce heap memory for create/upgrade-dolphinscheduler.sh (#5890)
* Rename NORAML_NODE_STATUS to NORMAL_NODE_STATUS
* Fix OSUtilsTest
* Optimize comments in properties
* Reduce heap size to 64m in create-dolphinscheduler.sh and upgrade-dolphinscheduler.sh
* [Improvement][License] Update oshi version in release-docs/LICENSE
* Add test prefix for test method
---
.../DOLPHIN/1.3.6/configuration/dolphin-master.xml | 2 +-
.../DOLPHIN/1.3.6/configuration/dolphin-worker.xml | 2 +-
.../conf/dolphinscheduler/master.properties.tpl | 2 +-
.../conf/dolphinscheduler/worker.properties.tpl | 2 +-
.../apache/dolphinscheduler/common/Constants.java | 2 +-
.../dolphinscheduler/common/utils/OSUtils.java | 6 +--
.../src/main/resources/common.properties | 2 +-
.../dolphinscheduler/common/os/OSUtilsTest.java | 49 +++++++++-------------
.../dolphinscheduler/common/utils/OSUtilsTest.java | 4 +-
dolphinscheduler-dist/release-docs/LICENSE | 2 +-
.../server/registry/HeartBeatTask.java | 2 +-
.../src/main/resources/master.properties | 2 +-
.../src/main/resources/worker.properties | 2 +-
script/create-dolphinscheduler.sh | 2 +-
script/upgrade-dolphinscheduler.sh | 2 +-
15 files changed, 36 insertions(+), 47 deletions(-)
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-master.xml b/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-master.xml
index e7b2df2..989fdea 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-master.xml
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-master.xml
@@ -30,7 +30,7 @@
<value-attributes>
<type>int</type>
</value-attributes>
- <description>master execute thread number to limit process instances</description>
+ <description>master execute thread number to limit process instances in parallel</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-worker.xml
index cd16f83..f10f5c2 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-worker.xml
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-worker.xml
@@ -30,7 +30,7 @@
<value-attributes>
<type>int</type>
</value-attributes>
- <description>worker execute thread number to limit task instances</description>
+ <description>worker execute thread number to limit task instances in parallel</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
diff --git a/docker/build/conf/dolphinscheduler/master.properties.tpl b/docker/build/conf/dolphinscheduler/master.properties.tpl
index 3c79c01..046d5c1 100644
--- a/docker/build/conf/dolphinscheduler/master.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/master.properties.tpl
@@ -18,7 +18,7 @@
# master listen port
#master.listen.port=5678
-# master execute thread number to limit process instances
+# master execute thread number to limit process instances in parallel
master.exec.threads=${MASTER_EXEC_THREADS}
# master execute task number in parallel per process instance
diff --git a/docker/build/conf/dolphinscheduler/worker.properties.tpl b/docker/build/conf/dolphinscheduler/worker.properties.tpl
index 7d266d7..eb28ca8 100644
--- a/docker/build/conf/dolphinscheduler/worker.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/worker.properties.tpl
@@ -18,7 +18,7 @@
# worker listener port
#worker.listen.port=1234
-# worker execute thread number to limit task instances
+# worker execute thread number to limit task instances in parallel
worker.exec.threads=${WORKER_EXEC_THREADS}
# worker heartbeat interval, the unit is second
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index e0a96d4..864af98 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -995,7 +995,7 @@ public final class Constants {
*/
public static final String PLUGIN_JAR_SUFFIX = ".jar";
- public static final int NORAML_NODE_STATUS = 0;
+ public static final int NORMAL_NODE_STATUS = 0;
public static final int ABNORMAL_NODE_STATUS = 1;
public static final String START_TIME = "start time";
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
index e5612c3..a3868ac 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
@@ -88,7 +88,7 @@ public class OSUtils {
*/
public static double availablePhysicalMemorySize() {
GlobalMemory memory = hal.getMemory();
- double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024;
+ double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
@@ -101,9 +101,9 @@ public class OSUtils {
* Keep 2 decimal
* @return available Physical Memory Size, unit: G
*/
- public static double totalMemorySize() {
+ public static double totalPhysicalMemorySize() {
GlobalMemory memory = hal.getMemory();
- double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
+ double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties
index 5abb907..3149847 100644
--- a/dolphinscheduler-common/src/main/resources/common.properties
+++ b/dolphinscheduler-common/src/main/resources/common.properties
@@ -39,7 +39,7 @@ login.user.keytab.path=/opt/hdfs.headless.keytab
# kerberos expire time, the unit is hour
kerberos.expire.time=2
-# resource view suffixss
+# resource view suffixs
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java
index bca3972..e86fbaa 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java
@@ -19,15 +19,11 @@ package org.apache.dolphinscheduler.common.os;
import org.apache.dolphinscheduler.common.utils.OSUtils;
-import java.math.RoundingMode;
-import java.text.DecimalFormat;
-
+import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import oshi.hardware.GlobalMemory;
-
/**
* OSUtilsTest
*/
@@ -36,40 +32,33 @@ public class OSUtilsTest {
private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class);
@Test
- public void memoryUsage() {
- logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239
+ public void testMemoryUsage() {
+ double memoryUsage = OSUtils.memoryUsage();
+ logger.info("memoryUsage : {}", memoryUsage);
+ Assert.assertTrue(memoryUsage >= 0.0);
}
@Test
- public void availablePhysicalMemorySize() {
- logger.info("availablePhysicalMemorySize : {}", OSUtils.availablePhysicalMemorySize());
- logger.info("availablePhysicalMemorySize : {}", OSUtils.totalMemorySize() / 10);
+ public void testPhysicalMemorySize() {
+ double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
+ double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
+ logger.info("availablePhysicalMemorySize : {}", availablePhysicalMemorySize);
+ logger.info("totalPhysicalMemorySize : {}", totalPhysicalMemorySize);
+ Assert.assertTrue(availablePhysicalMemorySize >= 0.0);
+ Assert.assertTrue(totalPhysicalMemorySize >= 0.0);
}
-
@Test
- public void loadAverage() {
- logger.info("memoryUsage : {}", OSUtils.loadAverage());
- }
-
-
- private void printMemory(GlobalMemory memory) {
- logger.info("memoryUsage : {} %" , (memory.getTotal() - memory.getAvailable()) * 100 / memory.getTotal() );
+ public void testLoadAverage() {
+ double loadAverage = OSUtils.loadAverage();
+ logger.info("loadAverage : {}", loadAverage);
+ Assert.assertTrue(loadAverage >= 0.0);
}
-
@Test
- public void cpuUsage() throws Exception {
- logger.info("cpuUsage : {}", OSUtils.cpuUsage());
- Thread.sleep(1000L);
- logger.info("cpuUsage : {}", OSUtils.cpuUsage());
-
+ public void testCpuUsage() {
double cpuUsage = OSUtils.cpuUsage();
-
- DecimalFormat df = new DecimalFormat("0.00");
-
- df.setRoundingMode(RoundingMode.HALF_UP);
-
- logger.info("cpuUsage1 : {}", df.format(cpuUsage));
+ logger.info("cpuUsage : {}", cpuUsage);
+ Assert.assertTrue(cpuUsage >= 0.0);
}
}
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
index 28d589e..6138206 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
@@ -38,8 +38,8 @@ public class OSUtilsTest {
public void testOSMetric(){
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
Assert.assertTrue(availablePhysicalMemorySize > 0.0f);
- double totalMemorySize = OSUtils.totalMemorySize();
- Assert.assertTrue(totalMemorySize > 0.0f);
+ double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
+ Assert.assertTrue(totalPhysicalMemorySize > 0.0f);
double loadAverage = OSUtils.loadAverage();
logger.info("loadAverage {}", loadAverage);
double memoryUsage = OSUtils.memoryUsage();
diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE
index 8157349..49091b3 100644
--- a/dolphinscheduler-dist/release-docs/LICENSE
+++ b/dolphinscheduler-dist/release-docs/LICENSE
@@ -443,7 +443,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
core-3.1.1 https://mvnrepository.com/artifact/org.eclipse.jdt/core/3.1.1 Eclipse Public License v1.0
logback-classic 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.2.3, EPL 1.0 and LGPL 2.1
logback-core 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-core/1.2.3, EPL 1.0 and LGPL 2.1
- oshi-core 3.5.0: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.5.0, EPL 1.0
+ oshi-core 3.9.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.9.1, EPL 1.0
junit 4.12: https://mvnrepository.com/artifact/junit/junit/4.12, EPL 1.0
h2-1.4.200 https://github.com/h2database/h2database/blob/master/LICENSE.txt, MPL 2.0 or EPL 1.0
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
index c7e1edd..ccdd445 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
@@ -75,7 +75,7 @@ public class HeartBeatTask implements Runnable {
double loadAverage = OSUtils.loadAverage();
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
- int status = Constants.NORAML_NODE_STATUS;
+ int status = Constants.NORMAL_NODE_STATUS;
if (loadAverage > maxCpuloadAvg || availablePhysicalMemorySize < reservedMemory) {
logger.warn("current cpu load average {} is too high or available memory {}G is too low, under max.cpuload.avg={} and reserved.memory={}G",
loadAverage, availablePhysicalMemorySize, maxCpuloadAvg, reservedMemory);
diff --git a/dolphinscheduler-server/src/main/resources/master.properties b/dolphinscheduler-server/src/main/resources/master.properties
index 3363d1f..cc45622 100644
--- a/dolphinscheduler-server/src/main/resources/master.properties
+++ b/dolphinscheduler-server/src/main/resources/master.properties
@@ -18,7 +18,7 @@
# master listen port
#master.listen.port=5678
-# master execute thread number to limit process instances
+# master execute thread number to limit process instances in parallel
#master.exec.threads=100
# master execute task number in parallel per process instance
diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties
index c19f439..bb822a5 100644
--- a/dolphinscheduler-server/src/main/resources/worker.properties
+++ b/dolphinscheduler-server/src/main/resources/worker.properties
@@ -18,7 +18,7 @@
# worker listener port
#worker.listen.port=1234
-# worker execute thread number to limit task instances
+# worker execute thread number to limit task instances in parallel
#worker.exec.threads=100
# worker heartbeat interval, the unit is second
diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh
index 7ab8cbc..9ca9499 100755
--- a/script/create-dolphinscheduler.sh
+++ b/script/create-dolphinscheduler.sh
@@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
-export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export STOP_TIMEOUT=5
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler
diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh
index 6944e3a..42ba839 100755
--- a/script/upgrade-dolphinscheduler.sh
+++ b/script/upgrade-dolphinscheduler.sh
@@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
-export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export STOP_TIMEOUT=5
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler