You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2017/04/08 18:08:26 UTC
[03/24] twill git commit: (TWILL-90) Add new configuration options to
set AM memory size
(TWILL-90) Add new configuration options to set AM memory size
This closes #36 on Github.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/4fb75281
Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/4fb75281
Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/4fb75281
Branch: refs/heads/site
Commit: 4fb7528110e5c242760c931923e485d530bf6c9b
Parents: 0cc3159
Author: Terence Yim <ch...@apache.org>
Authored: Thu Mar 16 14:27:25 2017 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Thu Mar 16 16:13:24 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/twill/api/Configs.java | 22 ++++++++++++++++++++
.../org/apache/twill/internal/Constants.java | 7 -------
.../internal/yarn/Hadoop20YarnAppClient.java | 8 +++----
.../internal/yarn/Hadoop21YarnAppClient.java | 10 ++++-----
.../apache/twill/yarn/YarnTwillPreparer.java | 6 ++++--
5 files changed, 35 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/twill/blob/4fb75281/twill-api/src/main/java/org/apache/twill/api/Configs.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/Configs.java b/twill-api/src/main/java/org/apache/twill/api/Configs.java
index ecc26d2..570eafe 100644
--- a/twill-api/src/main/java/org/apache/twill/api/Configs.java
+++ b/twill-api/src/main/java/org/apache/twill/api/Configs.java
@@ -67,6 +67,17 @@ public final class Configs {
*/
public static final String LOCATION_CACHE_ANTIQUE_EXPIRY_MS = "twill.location.cache.antique.expiry.ms";
+ /**
+ * Size in MB for the memory size of the YARN application master container.
+ */
+ public static final String YARN_AM_MEMORY_MB = "twill.yarn.am.memory.mb";
+
+ /**
+ * Size in MB for the reserved non-heap memory size for the YARN application Java process.
+ * The actual reserved memory size is limited by the {@link #HEAP_RESERVED_MIN_RATIO}.
+ */
+ public static final String YARN_AM_RESERVED_MEMORY_MB = "twill.yarn.am.reserved.memory.mb";
+
private Keys() {
}
}
@@ -96,6 +107,17 @@ public final class Configs {
*/
public static final long LOCATION_CACHE_ANTIQUE_EXPIRY_MS = TimeUnit.MINUTES.toMillis(5);
+ /**
+ * Default AM container memory size.
+ */
+ public static final int YARN_AM_MEMORY_MB = 512;
+
+ /**
+ * Default AM JVM reserved memory.
+ */
+ public static final int YARN_AM_RESERVED_MEMORY_MB = 150;
+
+
private Defaults() {
}
}
http://git-wip-us.apache.org/repos/asf/twill/blob/4fb75281/twill-common/src/main/java/org/apache/twill/internal/Constants.java
----------------------------------------------------------------------
diff --git a/twill-common/src/main/java/org/apache/twill/internal/Constants.java b/twill-common/src/main/java/org/apache/twill/internal/Constants.java
index 569b396..6e799d5 100644
--- a/twill-common/src/main/java/org/apache/twill/internal/Constants.java
+++ b/twill-common/src/main/java/org/apache/twill/internal/Constants.java
@@ -37,13 +37,6 @@ public final class Constants {
*/
public static final int CONSTRAINED_PROVISION_REQUEST_TIMEOUT = 5000;
- public static final double HEAP_MIN_RATIO = 0.7d;
-
- /** Memory size of AM. */
- public static final int APP_MASTER_MEMORY_MB = 512;
-
- public static final int APP_MASTER_RESERVED_MEMORY_MB = 150;
-
/** Command names for the restart runnable instances. */
public static final String RESTART_ALL_RUNNABLE_INSTANCES = "restartAllRunnableInstances";
public static final String RESTART_RUNNABLES_INSTANCES = "restartRunnablesInstances";
http://git-wip-us.apache.org/repos/asf/twill/blob/4fb75281/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
index bb494d4..0ea58c0 100644
--- a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
+++ b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
@@ -18,7 +18,6 @@
package org.apache.twill.internal.yarn;
import com.google.common.base.Throwables;
-import com.google.common.util.concurrent.AbstractIdleService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
@@ -37,8 +36,8 @@ import org.apache.hadoop.yarn.client.YarnClient;
import org.apache.hadoop.yarn.client.YarnClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.twill.api.Configs;
import org.apache.twill.api.TwillSpecification;
-import org.apache.twill.internal.Constants;
import org.apache.twill.internal.ProcessController;
import org.apache.twill.internal.ProcessLauncher;
import org.apache.twill.internal.appmaster.ApplicationMasterInfo;
@@ -96,10 +95,11 @@ public final class Hadoop20YarnAppClient implements YarnAppClient {
appSubmissionContext.setQueue(schedulerQueue);
}
- // TODO: Make it adjustable through TwillSpec (TWILL-90)
+
+ int memoryMB = configuration.getInt(Configs.Keys.YARN_AM_MEMORY_MB, Configs.Defaults.YARN_AM_MEMORY_MB);
// Set the resource requirement for AM
Resource amResource = Records.newRecord(Resource.class);
- amResource.setMemory(Constants.APP_MASTER_MEMORY_MB);
+ amResource.setMemory(memoryMB);
final Resource capability = adjustMemory(response, amResource);
ApplicationMasterInfo appMasterInfo = new ApplicationMasterInfo(appId, capability.getMemory(), 1);
http://git-wip-us.apache.org/repos/asf/twill/blob/4fb75281/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
index f24630f..0a64178 100644
--- a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
+++ b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
@@ -18,7 +18,6 @@
package org.apache.twill.internal.yarn;
import com.google.common.base.Throwables;
-import com.google.common.util.concurrent.AbstractIdleService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
@@ -34,8 +33,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.twill.api.Configs;
import org.apache.twill.api.TwillSpecification;
-import org.apache.twill.internal.Constants;
import org.apache.twill.internal.ProcessController;
import org.apache.twill.internal.ProcessLauncher;
import org.apache.twill.internal.appmaster.ApplicationMasterInfo;
@@ -55,6 +54,7 @@ import javax.annotation.Nullable;
* Apache Hadoop 2.1 and beyond.
* </p>
*/
+@SuppressWarnings("unused")
public final class Hadoop21YarnAppClient implements YarnAppClient {
private static final Logger LOG = LoggerFactory.getLogger(Hadoop21YarnAppClient.class);
@@ -90,9 +90,9 @@ public final class Hadoop21YarnAppClient implements YarnAppClient {
appSubmissionContext.setQueue(schedulerQueue);
}
- // TODO: Make it adjustable through TwillSpec (TWILL-90)
// Set the resource requirement for AM
- final Resource capability = adjustMemory(response, Resource.newInstance(Constants.APP_MASTER_MEMORY_MB, 1));
+ int memoryMB = configuration.getInt(Configs.Keys.YARN_AM_MEMORY_MB, Configs.Defaults.YARN_AM_MEMORY_MB);
+ final Resource capability = adjustMemory(response, Resource.newInstance(memoryMB, 1));
ApplicationMasterInfo appMasterInfo = new ApplicationMasterInfo(appId, capability.getMemory(),
capability.getVirtualCores());
@@ -181,7 +181,7 @@ public final class Hadoop21YarnAppClient implements YarnAppClient {
private final YarnClient yarnClient;
private final ApplicationId appId;
- public ProcessControllerImpl(YarnClient yarnClient, ApplicationId appId) {
+ ProcessControllerImpl(YarnClient yarnClient, ApplicationId appId) {
this.yarnClient = yarnClient;
this.appId = appId;
}
http://git-wip-us.apache.org/repos/asf/twill/blob/4fb75281/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index 6fbbf93..d9e70fd 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -404,9 +404,11 @@ final class YarnTwillPreparer implements TwillPreparer {
// org.apache.twill.internal.appmaster.ApplicationMasterMain
// false
+ int reservedMemoryMB = yarnConfig.getInt(Configs.Keys.YARN_AM_RESERVED_MEMORY_MB,
+ Configs.Defaults.YARN_AM_RESERVED_MEMORY_MB);
int memory = Resources.computeMaxHeapSize(appMasterInfo.getMemoryMB(),
- Constants.APP_MASTER_RESERVED_MEMORY_MB,
- Constants.HEAP_MIN_RATIO);
+ reservedMemoryMB,
+ minHeapRatio);
return launcher.prepareLaunch(ImmutableMap.<String, String>of(), localFiles.values(), credentials)
.addCommand(
"$JAVA_HOME/bin/java",