You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by bo...@apache.org on 2012/10/01 16:50:10 UTC
svn commit: r1392366 - in
/hadoop/common/branches/branch-0.23/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-y...
Author: bobby
Date: Mon Oct 1 14:50:10 2012
New Revision: 1392366
URL: http://svn.apache.org/viewvc?rev=1392366&view=rev
Log:
svn merge -c 1391912 FIXES: YARN-116. Add the ability to change the RM include/exclude file without a restart. (Contributed by xieguiming and Harsh J)
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1392366&r1=1392365&r2=1392366&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Mon Oct 1 14:50:10 2012
@@ -14,6 +14,9 @@ Release 0.23.5 - UNRELEASED
YARN-137. Change the default YARN scheduler to be the CapacityScheduler.
(sseth via acmurthy)
+ YARN-116. Add the ability to change the RM include/exclude file without
+ a restart. (xieguiming and Harsh J via sseth)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1392366&r1=1392365&r2=1392366&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Mon Oct 1 14:50:10 2012
@@ -189,15 +189,14 @@ public class AdminService extends Abstra
throws YarnRemoteException {
UserGroupInformation user = checkAcls("refreshNodes");
try {
- this.nodesListManager.refreshNodes();
+ this.nodesListManager.refreshNodes(new YarnConfiguration());
RMAuditLogger.logSuccess(user.getShortUserName(), "refreshNodes",
"AdminService");
return recordFactory.newRecordInstance(RefreshNodesResponse.class);
} catch (IOException ioe) {
LOG.info("Exception refreshing nodes ", ioe);
RMAuditLogger.logFailure(user.getShortUserName(), "refreshNodes",
- adminAcl.toString(), "AdminService",
- "Exception refreshing nodes");
+ adminAcl.toString(), "AdminService", "Exception refreshing nodes");
throw RPCUtil.getRemoteException(ioe);
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java?rev=1392366&r1=1392365&r2=1392366&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java Mon Oct 1 14:50:10 2012
@@ -87,8 +87,16 @@ public class NodesListManager extends Ab
}
}
- public void refreshNodes() throws IOException {
+ public void refreshNodes(Configuration yarnConf) throws IOException {
synchronized (hostsReader) {
+ if (null == yarnConf) {
+ yarnConf = new YarnConfiguration();
+ }
+ hostsReader.updateFileNames(yarnConf.get(
+ YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
+ YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), yarnConf.get(
+ YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
+ YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH));
hostsReader.refresh();
printConfiguredHosts();
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java?rev=1392366&r1=1392365&r2=1392366&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java Mon Oct 1 14:50:10 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -56,7 +57,7 @@ public class TestResourceTrackerService
.getRecordFactory(null);
/**
- * decommissioning using a include hosts file
+ * Decommissioning using a pre-configured include hosts file
*/
@Test
public void testDecommissionWithIncludeHosts() throws Exception {
@@ -83,7 +84,7 @@ public class TestResourceTrackerService
writeToHostsFile("host1");
- rm.getNodesListManager().refreshNodes();
+ rm.getNodesListManager().refreshNodes(conf);
nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
@@ -98,7 +99,7 @@ public class TestResourceTrackerService
}
/**
- * decommissioning using a exclude hosts file
+ * Decommissioning using a pre-configured exclude hosts file
*/
@Test
public void testDecommissionWithExcludeHosts() throws Exception {
@@ -123,7 +124,7 @@ public class TestResourceTrackerService
writeToHostsFile("host2");
- rm.getNodesListManager().refreshNodes();
+ rm.getNodesListManager().refreshNodes(conf);
nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
@@ -132,7 +133,81 @@ public class TestResourceTrackerService
NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));
checkDecommissionedNMCount(rm, ++initialMetricCount);
}
+
+ /**
+ * Decommissioning using a post-configured include hosts file
+ */
+ @Test
+ public void testAddNewIncludePathToConfiguration() throws Exception {
+ Configuration conf = new Configuration();
+ rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ ClusterMetrics metrics = ClusterMetrics.getMetrics();
+ assert(metrics != null);
+ int initialMetricCount = metrics.getNumDecommisionedNMs();
+ HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ writeToHostsFile("host1");
+ conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+ rm.getNodesListManager().refreshNodes(conf);
+ nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ "Node should not have been decomissioned.",
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals("Node should have been decomissioned but is in state" +
+ nodeHeartbeat.getNodeAction(),
+ NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
+ checkDecommissionedNMCount(rm, ++initialMetricCount);
+ }
+ /**
+ * Decommissioning using a post-configured exclude hosts file
+ */
+ @Test
+ public void testAddNewExcludePathToConfiguration() throws Exception {
+ Configuration conf = new Configuration();
+ rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ ClusterMetrics metrics = ClusterMetrics.getMetrics();
+ assert(metrics != null);
+ int initialMetricCount = metrics.getNumDecommisionedNMs();
+ HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ writeToHostsFile("host2");
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+ rm.getNodesListManager().refreshNodes(conf);
+ nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ "Node should not have been decomissioned.",
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals("Node should have been decomissioned but is in state" +
+ nodeHeartbeat.getNodeAction(),
+ NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
+ checkDecommissionedNMCount(rm, ++initialMetricCount);
+ }
+
@Test
public void testNodeRegistrationFailure() throws Exception {
writeToHostsFile("host1");