You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/11/15 11:30:27 UTC
svn commit: r1202117 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodeman...
Author: vinodkv
Date: Tue Nov 15 10:30:26 2011
New Revision: 1202117
URL: http://svn.apache.org/viewvc?rev=1202117&view=rev
Log:
MAPREDUCE-3102. Changed NodeManager to fail fast when LinuxContainerExecutor has wrong configuration or permissions. Contributed by Hitesh Shah.
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1202117&r1=1202116&r2=1202117&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Nov 15 10:30:26 2011
@@ -91,6 +91,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3331. Improvement to single node cluster setup documentation for
0.23 (Anupam Seth via mahadev)
+ MAPREDUCE-3102. Changed NodeManager to fail fast when LinuxContainerExecutor
+ has wrong configuration or permissions. (Hitesh Shah via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java?rev=1202117&r1=1202116&r2=1202117&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java Tue Nov 15 10:30:26 2011
@@ -63,6 +63,13 @@ public abstract class ContainerExecutor
}
/**
+ * Run the executor initialization steps.
+ * Verify that the necessary configs, permissions are in place.
+ * @throws IOException
+ */
+ public abstract void init() throws IOException;
+
+ /**
* Prepare the environment for containers in this application to execute.
* For $x in local.dirs
* create $x/$user/$appId
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1202117&r1=1202116&r2=1202117&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Tue Nov 15 10:30:26 2011
@@ -70,6 +70,11 @@ public class DefaultContainerExecutor ex
}
@Override
+ public void init() throws IOException {
+ // nothing to do or verify here
+ }
+
+ @Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
List<Path> localDirs) throws IOException, InterruptedException {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1202117&r1=1202116&r2=1202117&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Tue Nov 15 10:30:26 2011
@@ -100,6 +100,29 @@ public class LinuxContainerExecutor exte
: conf.get(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, defaultPath);
}
+ @Override
+ public void init() throws IOException {
+ // Send command to executor which will just start up,
+ // verify configuration/permissions and exit
+ List<String> command = new ArrayList<String>(
+ Arrays.asList(containerExecutorExe,
+ "--checksetup"));
+ String[] commandArray = command.toArray(new String[command.size()]);
+ ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("checkLinuxExecutorSetup: " + Arrays.toString(commandArray));
+ }
+ try {
+ shExec.execute();
+ } catch (ExitCodeException e) {
+ int exitCode = shExec.getExitCode();
+ LOG.warn("Exit code from container is : " + exitCode);
+ logOutput(shExec.getOutput());
+ throw new IOException("Linux container executor not configured properly"
+ + " (error=" + exitCode + ")", e);
+ }
+ }
+
@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1202117&r1=1202116&r2=1202117&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Tue Nov 15 10:30:26 2011
@@ -110,6 +110,11 @@ public class NodeManager extends Composi
ContainerExecutor exec = ReflectionUtils.newInstance(
conf.getClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
DefaultContainerExecutor.class, ContainerExecutor.class), conf);
+ try {
+ exec.init();
+ } catch (IOException e) {
+ throw new YarnException("Failed to initialize container executor", e);
+ }
DeletionService del = new DeletionService(exec);
addService(del);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1202117&r1=1202116&r2=1202117&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Tue Nov 15 10:30:26 2011
@@ -302,10 +302,12 @@ public class ContainerLaunch implements
// by this time, it will never be launched
exec.deactivateContainer(containerId);
- LOG.debug("Getting pid for container " + containerIdStr + " to kill"
- + " from pid file "
- + (pidFilePath != null ? pidFilePath.toString() : "null"));
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Getting pid for container " + containerIdStr + " to kill"
+ + " from pid file "
+ + (pidFilePath != null ? pidFilePath.toString() : "null"));
+ }
+
// however the container process may have already started
try {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c?rev=1202117&r1=1202116&r2=1202117&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c Tue Nov 15 10:30:26 2011
@@ -39,11 +39,14 @@
void display_usage(FILE *stream) {
fprintf(stream,
+ "Usage: container-executor --checksetup\n");
+ fprintf(stream,
"Usage: container-executor user command command-args\n");
fprintf(stream, "Commands:\n");
fprintf(stream, " initialize container: %2d appid tokens cmd app...\n",
INITIALIZE_CONTAINER);
- fprintf(stream, " launch container: %2d appid containerid workdir container-script tokens\n",
+ fprintf(stream,
+ " launch container: %2d appid containerid workdir container-script tokens pidfile\n",
LAUNCH_CONTAINER);
fprintf(stream, " signal container: %2d container-pid signal\n",
SIGNAL_CONTAINER);
@@ -52,14 +55,31 @@ void display_usage(FILE *stream) {
}
int main(int argc, char **argv) {
- //Minimum number of arguments required to run the container-executor
+ int invalid_args = 0;
+ int do_check_setup = 0;
+
+ LOGFILE = stdout;
+ ERRORFILE = stderr;
+
+ // Minimum number of arguments required to run
+ // the std. container-executor commands is 4
+ // 4 args not needed for checksetup option
if (argc < 4) {
+ invalid_args = 1;
+ if (argc == 2) {
+ const char *arg1 = argv[1];
+ if (strcmp("--checksetup", arg1) == 0) {
+ invalid_args = 0;
+ do_check_setup = 1;
+ }
+ }
+ }
+
+ if (invalid_args != 0) {
display_usage(stdout);
return INVALID_ARGUMENT_NUMBER;
}
- LOGFILE = stdout;
- ERRORFILE = stderr;
int command;
const char * app_id = NULL;
const char * container_id = NULL;
@@ -111,11 +131,19 @@ int main(int argc, char **argv) {
return INVALID_CONTAINER_EXEC_PERMISSIONS;
}
+ if (do_check_setup != 0) {
+ // basic setup checks done
+ // verified configs available and valid
+ // verified executor permissions
+ return 0;
+ }
+
//checks done for user name
if (argv[optind] == NULL) {
fprintf(ERRORFILE, "Invalid user name.\n");
return INVALID_USER_NAME;
}
+
int ret = set_user(argv[optind]);
if (ret != 0) {
return ret;
@@ -143,7 +171,7 @@ int main(int argc, char **argv) {
break;
case LAUNCH_CONTAINER:
if (argc < 9) {
- fprintf(ERRORFILE, "Too few arguments (%d vs 8) for launch container\n",
+ fprintf(ERRORFILE, "Too few arguments (%d vs 9) for launch container\n",
argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java?rev=1202117&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java Tue Nov 15 10:30:26 2011
@@ -0,0 +1,55 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+
+public class TestNodeManager {
+
+ public static final class InvalidContainerExecutor extends
+ DefaultContainerExecutor {
+ @Override
+ public void init() throws IOException {
+ throw new IOException("dummy executor init called");
+ }
+ }
+
+ @Test
+ public void testContainerExecutorInitCall() {
+ NodeManager nm = new NodeManager();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
+ InvalidContainerExecutor.class,
+ ContainerExecutor.class);
+ try {
+ nm.init(conf);
+ fail("Init should fail");
+ } catch (YarnException e) {
+ //PASS
+ assert(e.getCause().getMessage().contains("dummy executor init called"));
+ }
+ }
+
+}