You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2011/11/30 00:28:20 UTC
svn commit: r1208135 [3/3] - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobcli...
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1208135&r1=1208134&r2=1208135&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Tue Nov 29 23:28:16 2011
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.Writer;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -37,6 +38,8 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
@@ -47,6 +50,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -54,10 +58,19 @@ public class TestNMWebServer {
private static final File testRootDir = new File("target",
TestNMWebServer.class.getSimpleName());
+ private static File testLogDir = new File("target",
+ TestNMWebServer.class.getSimpleName() + "LogDir");
@Before
public void setup() {
testRootDir.mkdirs();
+ testLogDir.mkdir();
+ }
+
+ @After
+ public void tearDown() {
+ FileUtil.fullyDelete(testRootDir);
+ FileUtil.fullyDelete(testLogDir);
}
@Test
@@ -74,9 +87,14 @@ public class TestNMWebServer {
}
};
Configuration conf = new Configuration();
- WebServer server = new WebServer(nmContext, resourceView,
- new ApplicationACLsManager(conf));
conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
+ NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
+ healthChecker.init(conf);
+ LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
+
+ WebServer server = new WebServer(nmContext, resourceView,
+ new ApplicationACLsManager(conf), dirsHandler);
server.init(conf);
server.start();
@@ -119,20 +137,20 @@ public class TestNMWebServer {
containerId.getApplicationAttemptId().getApplicationId();
nmContext.getApplications().get(applicationId).getContainers()
.put(containerId, container);
- writeContainerLogs(conf, nmContext, containerId);
+ writeContainerLogs(nmContext, containerId, dirsHandler);
}
// TODO: Pull logs and test contents.
// Thread.sleep(1000000);
}
- private void writeContainerLogs(Configuration conf, Context nmContext,
- ContainerId containerId)
+ private void writeContainerLogs(Context nmContext,
+ ContainerId containerId, LocalDirsHandlerService dirsHandler)
throws IOException {
// ContainerLogDir should be created
File containerLogDir =
- ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs(conf,
- containerId).get(0);
+ ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs(containerId,
+ dirsHandler).get(0);
containerLogDir.mkdirs();
for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
Writer writer = new FileWriter(new File(containerLogDir, fileType));
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1208135&r1=1208134&r2=1208135&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Tue Nov 29 23:28:16 2011
@@ -23,7 +23,6 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.NodeHealthCheckerService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -41,6 +40,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
@@ -51,7 +51,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.service.CompositeService;
-import org.apache.hadoop.yarn.service.Service.STATE;
public class MiniYARNCluster extends CompositeService {
@@ -69,13 +68,23 @@ public class MiniYARNCluster extends Com
private File testWorkDir;
- public MiniYARNCluster(String testName) {
- //default number of nodeManagers = 1
- this(testName, 1);
- }
+ // Number of nm-local-dirs per nodemanager
+ private int numLocalDirs;
+ // Number of nm-log-dirs per nodemanager
+ private int numLogDirs;
+
+ /**
+ * @param testName name of the test
+ * @param noOfNodeManagers the number of node managers in the cluster
+ * @param numLocalDirs the number of nm-local-dirs per nodemanager
+ * @param numLogDirs the number of nm-log-dirs per nodemanager
+ */
+ public MiniYARNCluster(String testName, int noOfNodeManagers,
+ int numLocalDirs, int numLogDirs) {
- public MiniYARNCluster(String testName, int noOfNodeManagers) {
super(testName);
+ this.numLocalDirs = numLocalDirs;
+ this.numLogDirs = numLogDirs;
this.testWorkDir = new File("target", testName);
try {
FileContext.getLocalFSFileContext().delete(
@@ -166,25 +175,39 @@ public class MiniYARNCluster extends Com
super.init(config);
}
+ /**
+ * Create local/log directories
+ * @param dirType type of directories i.e. local dirs or log dirs
+ * @param numDirs number of directories
+ * @return the created directories as a comma delimited String
+ */
+ private String prepareDirs(String dirType, int numDirs) {
+ File []dirs = new File[numDirs];
+ String dirsString = "";
+ for (int i = 0; i < numDirs; i++) {
+ dirs[i]= new File(testWorkDir, MiniYARNCluster.this.getName()
+ + "-" + dirType + "Dir-nm-" + index + "_" + i);
+ dirs[i].mkdir();
+ LOG.info("Created " + dirType + "Dir in " + dirs[i].getAbsolutePath());
+ String delimiter = (i > 0) ? "," : "";
+ dirsString = dirsString.concat(delimiter + dirs[i].getAbsolutePath());
+ }
+ return dirsString;
+ }
+
public synchronized void start() {
try {
- File localDir = new File(testWorkDir, MiniYARNCluster.this.getName()
- + "-localDir-nm-" + index);
- localDir.mkdir();
- LOG.info("Created localDir in " + localDir.getAbsolutePath());
- getConfig().set(YarnConfiguration.NM_LOCAL_DIRS,
- localDir.getAbsolutePath());
- File logDir =
- new File(testWorkDir, MiniYARNCluster.this.getName()
- + "-logDir-nm-" + index);
+ // create nm-local-dirs and configure them for the nodemanager
+ String localDirsString = prepareDirs("local", numLocalDirs);
+ getConfig().set(YarnConfiguration.NM_LOCAL_DIRS, localDirsString);
+ // create nm-log-dirs and configure them for the nodemanager
+ String logDirsString = prepareDirs("log", numLogDirs);
+ getConfig().set(YarnConfiguration.NM_LOG_DIRS, logDirsString);
+
File remoteLogDir =
new File(testWorkDir, MiniYARNCluster.this.getName()
+ "-remoteLogDir-nm-" + index);
- logDir.mkdir();
remoteLogDir.mkdir();
- LOG.info("Created logDir in " + logDir.getAbsolutePath());
- getConfig().set(YarnConfiguration.NM_LOG_DIRS,
- logDir.getAbsolutePath());
getConfig().set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
remoteLogDir.getAbsolutePath());
// By default AM + 2 containers
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1208135&r1=1208134&r2=1208135&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Tue Nov 29 23:28:16 2011
@@ -117,7 +117,7 @@ public class TestContainerManagerSecurit
conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L);
UserGroupInformation.setConfiguration(conf);
yarnCluster = new MiniYARNCluster(TestContainerManagerSecurity.class
- .getName());
+ .getName(), 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm?rev=1208135&r1=1208134&r2=1208135&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm Tue Nov 29 23:28:16 2011
@@ -398,6 +398,15 @@ Hadoop MapReduce Next Generation - Clust
| | | Timeout for health script execution. |
*-------------------------+-------------------------+------------------------+
+ The health checker script is not supposed to give ERROR if only some of the
+ local disks become bad. NodeManager has the ability to periodically check
+ the health of the local disks (specifically checks nodemanager-local-dirs
+ and nodemanager-log-dirs) and after reaching the threshold of number of
+ bad directories based on the value set for the config property
+ yarn.nodemanager.disk-health-checker.min-healthy-disks. The boot disk is
+ either raided or a failure in the boot disk is identified by the health
+ checker script.
+
* {Slaves file}
Typically you choose one machine in the cluster to act as the NameNode and
@@ -874,13 +883,6 @@ KVNO Timestamp Principal
*-------------------------+-------------------------+------------------------+
|| Parameter || Value || Notes |
*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.local-dirs>>> | |
-| | Comma-separated list of NodeManager local directories. | |
-| | | Paths to NodeManager local directories. Should be same as the value |
-| | | which was provided to key in <<<conf/yarn-site.xml>>>. This is |
-| | | required to validate paths passed to the setuid executable in order |
-| | to prevent arbitrary paths being passed to it. |
-*-------------------------+-------------------------+------------------------+
| <<<yarn.nodemanager.linux-container-executor.group>>> | <hadoop> | |
| | | Unix group of the NodeManager. The group owner of the |
| | |<container-executor> binary should be this group. Should be same as the |
@@ -888,14 +890,6 @@ KVNO Timestamp Principal
| | | required for validating the secure access of the <container-executor> |
| | | binary. |
*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.log-dirs>>> | |
-| | Comma-separated list of NodeManager log directories. | |
-| | | Paths to NodeManager log directories. Should be same as the value |
-| | | which was provided to key in <<<conf/yarn-site.xml>>>. This is |
-| | | required to set proper permissions on the log files so that they can |
-| | | be written to by the user's containers and read by the NodeManager for |
-| | | <log aggregation>. |
-*-------------------------+-------------------------+------------------------+
| <<<banned.users>>> | hfds,yarn,mapred,bin | Banned users. |
*-------------------------+-------------------------+------------------------+
| <<<min.user.id>>> | 1000 | Prevent other super-users. |