You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluo.apache.org by mw...@apache.org on 2018/01/08 20:17:03 UTC
[fluo-yarn] branch master updated: fixes apache/fluo#671 made
reserved mem configurable (#15)
This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-yarn.git
The following commit(s) were added to refs/heads/master by this push:
new 2d1643a fixes apache/fluo#671 made reserved mem configurable (#15)
2d1643a is described below
commit 2d1643aef0e1390dbc26da1388cdbc7d69d20c6a
Author: Keith Turner <ke...@deenlo.com>
AuthorDate: Mon Jan 8 15:17:01 2018 -0500
fixes apache/fluo#671 made reserved mem configurable (#15)
---
.../java/org/apache/fluo/yarn/core/FluoYarnEnv.java | 20 ++++++++++++++++++++
.../org/apache/fluo/yarn/core/FluoYarnLauncher.java | 12 ++++++++++++
.../apache/fluo/yarn/core/FluoYarnProperties.java | 4 ++++
distribution/conf/fluo-yarn.properties | 15 ++++++++++-----
distribution/lib/fetch.sh | 20 ++++++++++----------
pom.xml | 2 +-
6 files changed, 57 insertions(+), 16 deletions(-)
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
index c5a8629..15e77b0 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
@@ -116,6 +116,16 @@ public class FluoYarnEnv {
FluoYarnProperties.WORKER_MAX_MEMORY_MB_DEFAULT));
}
+ public String getWorkerReservedMemory() {
+ String rmem = props.getProperty(FluoYarnProperties.WORKER_RESERVED_MEMORY_MB_PROP, null);
+ if (rmem == null) {
+ return rmem;
+ }
+
+ Preconditions.checkArgument(Integer.parseInt(rmem) > 0);
+ return rmem;
+ }
+
public int getOracleCores() {
return Integer.valueOf(props.getProperty(FluoYarnProperties.ORACLE_NUM_CORES_PROP,
FluoYarnProperties.ORACLE_NUM_CORES_DEFAULT));
@@ -131,6 +141,16 @@ public class FluoYarnEnv {
FluoYarnProperties.ORACLE_MAX_MEMORY_MB_DEFAULT));
}
+ public String getOracleReservedMemory() {
+ String rmem = props.getProperty(FluoYarnProperties.ORACLE_RESERVED_MEMORY_MB_PROP, null);
+ if (rmem == null) {
+ return rmem;
+ }
+
+ Preconditions.checkArgument(Integer.parseInt(rmem) > 0);
+ return rmem;
+ }
+
public String getYarnQueue() {
return props.getProperty(FluoYarnProperties.QUEUE_PROP, null);
}
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
index e79f9dc..056604f 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
@@ -17,7 +17,9 @@ package org.apache.fluo.yarn.core;
import java.io.File;
import java.util.Collection;
+import java.util.Collections;
+import org.apache.twill.api.Configs;
import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.ResourceSpecification;
import org.apache.twill.api.TwillApplication;
@@ -129,6 +131,16 @@ public class FluoYarnLauncher {
preparer.setSchedulerQueue(env.getYarnQueue());
}
+ if (env.getWorkerReservedMemory() != null) {
+ preparer.withConfiguration(WORKER_ID, Collections
+ .singletonMap(Configs.Keys.JAVA_RESERVED_MEMORY_MB, env.getWorkerReservedMemory()));
+ }
+
+ if (env.getOracleReservedMemory() != null) {
+ preparer.withConfiguration(ORACLE_ID, Collections
+ .singletonMap(Configs.Keys.JAVA_RESERVED_MEMORY_MB, env.getOracleReservedMemory()));
+ }
+
TwillController controller = preparer.start();
ResourceReport report = controller.getResourceReport();
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
index 990afa7..c707928 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
@@ -35,6 +35,8 @@ public class FluoYarnProperties {
// Worker properties
public static final String WORKER_INSTANCES_PROP = YARN_PREFIX + ".worker.instances";
public static final String WORKER_MAX_MEMORY_MB_PROP = YARN_PREFIX + ".worker.max.memory.mb";
+ public static final String WORKER_RESERVED_MEMORY_MB_PROP =
+ YARN_PREFIX + ".worker.reserved.memory.mb";
public static final String WORKER_NUM_CORES_PROP = YARN_PREFIX + ".worker.num.cores";
public static final String WORKER_INSTANCES_DEFAULT = "1";
public static final String WORKER_MAX_MEMORY_MB_DEFAULT = "1024";
@@ -43,6 +45,8 @@ public class FluoYarnProperties {
// Oracle properties
public static final String ORACLE_INSTANCES_PROP = YARN_PREFIX + ".oracle.instances";
public static final String ORACLE_MAX_MEMORY_MB_PROP = YARN_PREFIX + ".oracle.max.memory.mb";
+ public static final String ORACLE_RESERVED_MEMORY_MB_PROP =
+ YARN_PREFIX + ".oracle.reserved.memory.mb";
public static final String ORACLE_NUM_CORES_PROP = YARN_PREFIX + ".oracle.num.cores";
public static final String ORACLE_INSTANCES_DEFAULT = "1";
public static final String ORACLE_MAX_MEMORY_MB_DEFAULT = "512";
diff --git a/distribution/conf/fluo-yarn.properties b/distribution/conf/fluo-yarn.properties
index c2f3546..f2b1705 100644
--- a/distribution/conf/fluo-yarn.properties
+++ b/distribution/conf/fluo-yarn.properties
@@ -43,6 +43,9 @@
## Max memory of Oracle yarn containers (in MB)
#fluo.yarn.oracle.max.memory.mb=512
+## Reserved memory. Behaves same as worker property, so see its docs.
+#fluo.yarn.oracle.reserved.memory.mb=200
+
## Number of Oracle virtual cores
#fluo.yarn.oracle.num.cores=1
@@ -51,12 +54,14 @@
## Number of Worker instances
#fluo.yarn.worker.instances=1
-## Max memory of worker YARN containers (in MB). If YARN is killing worker processes consider
-## increasing twill.java.reserved.memory.mb (which defaults to 200 and is set in yarn-site.xml).
-## The twill.java.reserved.memory.mb config determines the gap between the YARN memory limit set
-## below and the java -Xmx setting. For example, if max memory is 1024 and twill reserved memory
-## is 200, the java -Xmx setting will be 1024-200 = 824 MB.
+## Max memory of worker YARN containers (in MB).
#fluo.yarn.worker.max.memory.mb=1024
+## This property is used to compute the Java heap size by subtracting it from the max memory
+## property. For example, if max memory is 1024 and reserved memory is 200, then java -Xmx setting
+## will be 1024-200 = 824 MB. If YARN is killing workers because too much memory is used, then
+## consider increasing this property.
+#fluo.yarn.worker.reserved.memory.mb=200
+
## Number of worker virtual cores
#fluo.yarn.worker.num.cores=1
diff --git a/distribution/lib/fetch.sh b/distribution/lib/fetch.sh
index 0881187..2c351f0 100755
--- a/distribution/lib/fetch.sh
+++ b/distribution/lib/fetch.sh
@@ -17,7 +17,7 @@ lib_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
maven_prefix=https://repo1.maven.org/maven2
function download {
- IFS=':' read -ra DEP <<< "$1"
+ IFS=':' read -ra DEP <<< "$1"
dir=$lib_dir/
if [ -n "$2" ]; then
dir=$lib_dir/$2
@@ -35,7 +35,7 @@ function download {
if [ -f $dir/$fn ]; then
echo "SUCCESS: Dependency exists - $dir/$fn"
- else
+ else
wget -q $download_url -P $dir
if [ $? == 0 ]; then
echo "SUCCESS: Dependency downloaded from $download_url"
@@ -57,14 +57,14 @@ download com.yammer.metrics:metrics-annotation:jar:2.2.0
download com.yammer.metrics:metrics-core:jar:2.2.0
download net.sf.jopt-simple:jopt-simple:jar:3.2
download org.apache.kafka:kafka_2.10:jar:0.8.0
-download org.apache.twill:twill-api:jar:0.11.0
-download org.apache.twill:twill-common:jar:0.11.0
-download org.apache.twill:twill-core:jar:0.11.0
-download org.apache.twill:twill-discovery-api:jar:0.11.0
-download org.apache.twill:twill-discovery-core:jar:0.11.0
-download org.apache.twill:twill-ext:jar:0.11.0
-download org.apache.twill:twill-yarn:jar:0.11.0
-download org.apache.twill:twill-zookeeper:jar:0.11.0
+download org.apache.twill:twill-api:jar:0.12.0
+download org.apache.twill:twill-common:jar:0.12.0
+download org.apache.twill:twill-core:jar:0.12.0
+download org.apache.twill:twill-discovery-api:jar:0.12.0
+download org.apache.twill:twill-discovery-core:jar:0.12.0
+download org.apache.twill:twill-ext:jar:0.12.0
+download org.apache.twill:twill-yarn:jar:0.12.0
+download org.apache.twill:twill-zookeeper:jar:0.12.0
download org.ow2.asm:asm-all:jar:5.0.2
download org.scala-lang:scala-compiler:jar:2.10.1
download org.scala-lang:scala-library:jar:2.10.1
diff --git a/pom.xml b/pom.xml
index 00cda0d..ec732cb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,7 +42,7 @@
<hadoop.version>2.6.3</hadoop.version>
<logback.version>1.1.3</logback.version>
<slf4j.version>1.7.12</slf4j.version>
- <twill.version>0.11.0</twill.version>
+ <twill.version>0.12.0</twill.version>
</properties>
<dependencyManagement>
<dependencies>
--
To stop receiving notification emails like this one, please contact
['"commits@fluo.apache.org" <co...@fluo.apache.org>'].