You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/05/12 00:55:01 UTC
[27/50] [abbrv] hadoop git commit: YARN-1912. ResourceLocalizer
started without any jvm memory control. Contributed by Masatake Iwasaki
YARN-1912. ResourceLocalizer started without any jvm memory control.
Contributed by Masatake Iwasaki
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6471d18b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6471d18b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6471d18b
Branch: refs/heads/HDFS-7240
Commit: 6471d18bc72bc6c83ce31a03b5c5f5737847bb6d
Parents: 333f9a8
Author: Xuan <xg...@apache.org>
Authored: Fri May 8 20:01:21 2015 -0700
Committer: Xuan <xg...@apache.org>
Committed: Fri May 8 20:01:21 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 7 +++++++
.../server/nodemanager/LinuxContainerExecutor.java | 1 +
.../nodemanager/WindowsSecureContainerExecutor.java | 3 ++-
.../localizer/ContainerLocalizer.java | 13 ++++++++++++-
.../TestLinuxContainerExecutorWithMocks.java | 15 ++++++++-------
6 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 05ae144..8f07b79 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -215,6 +215,9 @@ Release 2.8.0 - UNRELEASED
YARN-3271. FairScheduler: Move tests related to max-runnable-apps from
TestFairScheduler to TestAppRunnability. (nijel via kasha)
+ YARN-1912. ResourceLocalizer started without any jvm memory control.
+ (Masatake Iwasaki via xgong)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 0851f3c..48a75c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1004,6 +1004,13 @@ public class YarnConfiguration extends Configuration {
public static final String NM_HEALTH_CHECK_SCRIPT_OPTS =
NM_PREFIX + "health-checker.script.opts";
+ /** The JVM options used on forking ContainerLocalizer process
+ by container executor. */
+ public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY =
+ NM_PREFIX + "container-localizer.java.opts";
+ public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
+ "-Xmx256m";
+
/** The Docker image name(For DockerContainerExecutor).*/
public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME =
NM_PREFIX + "docker-container-executor.image-name";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index f8da958..16068d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -244,6 +244,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
if (javaLibPath != null) {
command.add("-Djava.library.path=" + javaLibPath);
}
+ command.addAll(ContainerLocalizer.getJavaOpts(getConf()));
buildMainArgs(command, user, appId, locId, nmAddr, localDirs);
String[] commandArray = command.toArray(new String[command.size()]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
index b7bec5f..619b845 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -697,7 +697,8 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
if (javaLibPath != null) {
command.add("-Djava.library.path=" + javaLibPath);
}
-
+ command.addAll(ContainerLocalizer.getJavaOpts(getConf()));
+
ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
localDirs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 32e3553..f82f894 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.SerializedException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -312,7 +313,17 @@ public class ContainerLocalizer {
status.addAllResources(currentResources);
return status;
}
-
+
+ /**
+ * Returns the JVM options to to launch the resource localizer.
+ * @param conf the configuration properties to launch the resource localizer.
+ */
+ public static List<String> getJavaOpts(Configuration conf) {
+ String opts = conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
+ YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT);
+ return Arrays.asList(opts.split(" "));
+ }
+
/**
* Adds the ContainerLocalizer arguments for a @{link ShellCommandExecutor},
* as expected by ContainerLocalizer.main
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6471d18b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 98ab8e0..dce2cd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -187,19 +187,20 @@ public class TestLinuxContainerExecutorWithMocks {
try {
mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler);
List<String> result=readMockParams();
- Assert.assertEquals(result.size(), 17);
+ Assert.assertEquals(result.size(), 18);
Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
Assert.assertEquals(result.get(1), "test");
Assert.assertEquals(result.get(2), "0" );
Assert.assertEquals(result.get(3),"application_0" );
Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath");
Assert.assertEquals(result.get(8), "-classpath" );
- Assert.assertEquals(result.get(11),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
- Assert.assertEquals(result.get(12), "test");
- Assert.assertEquals(result.get(13), "application_0");
- Assert.assertEquals(result.get(14),"12345" );
- Assert.assertEquals(result.get(15),"localhost" );
- Assert.assertEquals(result.get(16),"8040" );
+ Assert.assertEquals(result.get(11), "-Xmx256m" );
+ Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
+ Assert.assertEquals(result.get(13), "test");
+ Assert.assertEquals(result.get(14), "application_0");
+ Assert.assertEquals(result.get(15),"12345" );
+ Assert.assertEquals(result.get(16),"localhost" );
+ Assert.assertEquals(result.get(17),"8040" );
} catch (InterruptedException e) {
LOG.error("Error:"+e.getMessage(),e);