You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2017/09/15 16:56:14 UTC
[04/50] [abbrv] hadoop git commit: Revert 'YARN-4727. Unable to
override the $HADOOP_CONF_DIR env variable for container. Contributed by
Jason Lowe.'
Revert 'YARN-4727. Unable to override the $HADOOP_CONF_DIR env variable for container. Contributed by Jason Lowe.'
This reverts commit 729d05f5293acf63e7e4aa3bfbf29b999c9a2906.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3c44195
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3c44195
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3c44195
Branch: refs/heads/YARN-1011
Commit: a3c44195bed724c02bb76859fe2690d6a9e8f2e9
Parents: 729d05f
Author: Eric Payne <ep...@apache.org>
Authored: Wed Sep 13 14:38:58 2017 -0500
Committer: Eric Payne <ep...@apache.org>
Committed: Wed Sep 13 14:38:58 2017 -0500
----------------------------------------------------------------------
.../launcher/ContainerLaunch.java | 5 ++-
.../launcher/TestContainerLaunch.java | 32 ++++++--------------
2 files changed, 13 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c44195/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index d0ce787..50443f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1148,7 +1148,10 @@ public class ContainerLaunch implements Callable<Integer> {
environment.put(Environment.PWD.name(), pwd.toString());
- putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name());
+ putEnvIfNotNull(environment,
+ Environment.HADOOP_CONF_DIR.name(),
+ System.getenv(Environment.HADOOP_CONF_DIR.name())
+ );
if (!Shell.WINDOWS) {
environment.put("JVM_PID", "$$");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c44195/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 7176942..085b60f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -46,7 +46,6 @@ import java.util.StringTokenizer;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
-import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
@@ -55,7 +54,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.StringUtils;
@@ -726,15 +724,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
- final String userConfDir = "user_set_HADOOP_CONF_DIR";
- userSetEnv.put(Environment.HADOOP_CONF_DIR.name(), userConfDir);
containerLaunchContext.setEnvironment(userSetEnv);
File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
File processStartFile =
- new File(tmpDir, "env_vars.tmp").getAbsoluteFile();
- final File processFinalFile =
new File(tmpDir, "env_vars.txt").getAbsoluteFile();
if (Shell.WINDOWS) {
fileWriter.println("@echo " + Environment.CONTAINER_ID.$() + "> "
@@ -755,8 +749,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
+ processStartFile);
fileWriter.println("@echo " + Environment.HOME.$() + ">> "
+ processStartFile);
- fileWriter.println("@echo " + Environment.HADOOP_CONF_DIR.$() + ">> "
- + processStartFile);
for (String serviceName : containerManager.getAuxServiceMetaData()
.keySet()) {
fileWriter.println("@echo %" + AuxiliaryServiceHelper.NM_AUX_SERVICE
@@ -764,8 +756,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
+ processStartFile);
}
fileWriter.println("@echo " + cId + ">> " + processStartFile);
- fileWriter.println("@move /Y " + processStartFile + " "
- + processFinalFile);
fileWriter.println("@ping -n 100 127.0.0.1 >nul");
} else {
fileWriter.write("\numask 0"); // So that start file is readable by the test
@@ -787,8 +777,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
+ processStartFile);
fileWriter.write("\necho $" + Environment.HOME.name() + " >> "
+ processStartFile);
- fileWriter.write("\necho $" + Environment.HADOOP_CONF_DIR.name() + " >> "
- + processStartFile);
for (String serviceName : containerManager.getAuxServiceMetaData()
.keySet()) {
fileWriter.write("\necho $" + AuxiliaryServiceHelper.NM_AUX_SERVICE
@@ -796,7 +784,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
+ processStartFile);
}
fileWriter.write("\necho $$ >> " + processStartFile);
- fileWriter.write("\nmv " + processStartFile + " " + processFinalFile);
fileWriter.write("\nexec sleep 100");
}
fileWriter.close();
@@ -830,12 +817,13 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
StartContainersRequest.newInstance(list);
containerManager.startContainers(allRequests);
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- return processFinalFile.exists();
- }
- }, 10, 20000);
+ int timeoutSecs = 0;
+ while (!processStartFile.exists() && timeoutSecs++ < 20) {
+ Thread.sleep(1000);
+ LOG.info("Waiting for process start-file to be created");
+ }
+ Assert.assertTrue("ProcessStartFile doesn't exist!",
+ processStartFile.exists());
// Now verify the contents of the file
List<String> localDirs = dirsHandler.getLocalDirs();
@@ -855,7 +843,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
}
BufferedReader reader =
- new BufferedReader(new FileReader(processFinalFile));
+ new BufferedReader(new FileReader(processStartFile));
Assert.assertEquals(cId.toString(), reader.readLine());
Assert.assertEquals(context.getNodeId().getHost(), reader.readLine());
Assert.assertEquals(String.valueOf(context.getNodeId().getPort()),
@@ -878,7 +866,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
YarnConfiguration.NM_USER_HOME_DIR,
YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
reader.readLine());
- Assert.assertEquals(userConfDir, reader.readLine());
+
for (String serviceName : containerManager.getAuxServiceMetaData().keySet()) {
Assert.assertEquals(
containerManager.getAuxServiceMetaData().get(serviceName),
@@ -917,8 +905,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
containerLaunchContext.getEnvironment()
.get(Environment.HOME.name()));
- Assert.assertEquals(userConfDir, containerLaunchContext.getEnvironment()
- .get(Environment.HADOOP_CONF_DIR.name()));
// Get the pid of the process
String pid = reader.readLine().trim();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org