You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ss...@apache.org on 2012/09/30 00:39:03 UTC
svn commit: r1391912 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-reso...
Author: sseth
Date: Sat Sep 29 22:39:03 2012
New Revision: 1391912
URL: http://svn.apache.org/viewvc?rev=1391912&view=rev
Log:
YARN-116. Add the ability to change the RM include/exclude file without a restart. (Contributed by xieguiming and Harsh J)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1391912&r1=1391911&r2=1391912&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Sat Sep 29 22:39:03 2012
@@ -40,6 +40,9 @@ Release 2.0.3-alpha - Unreleased
YARN-53. Added the missing getGroups API to ResourceManager. (Bo Wang via
vinodkv)
+ YARN-116. Add the ability to change the RM include/exclude file without
+ a restart. (xieguiming and Harsh J via sseth)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1391912&r1=1391911&r2=1391912&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Sat Sep 29 22:39:03 2012
@@ -189,15 +189,14 @@ public class AdminService extends Abstra
throws YarnRemoteException {
UserGroupInformation user = checkAcls("refreshNodes");
try {
- this.nodesListManager.refreshNodes();
+ this.nodesListManager.refreshNodes(new YarnConfiguration());
RMAuditLogger.logSuccess(user.getShortUserName(), "refreshNodes",
"AdminService");
return recordFactory.newRecordInstance(RefreshNodesResponse.class);
} catch (IOException ioe) {
LOG.info("Exception refreshing nodes ", ioe);
RMAuditLogger.logFailure(user.getShortUserName(), "refreshNodes",
- adminAcl.toString(), "AdminService",
- "Exception refreshing nodes");
+ adminAcl.toString(), "AdminService", "Exception refreshing nodes");
throw RPCUtil.getRemoteException(ioe);
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java?rev=1391912&r1=1391911&r2=1391912&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java Sat Sep 29 22:39:03 2012
@@ -103,8 +103,16 @@ public class NodesListManager extends Ab
}
}
- public void refreshNodes() throws IOException {
+ public void refreshNodes(Configuration yarnConf) throws IOException {
synchronized (hostsReader) {
+ if (null == yarnConf) {
+ yarnConf = new YarnConfiguration();
+ }
+ hostsReader.updateFileNames(yarnConf.get(
+ YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
+ YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH), yarnConf.get(
+ YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
+ YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH));
hostsReader.refresh();
printConfiguredHosts();
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java?rev=1391912&r1=1391911&r2=1391912&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java Sat Sep 29 22:39:03 2012
@@ -54,7 +54,7 @@ public class TestResourceTrackerService
private MockRM rm;
/**
- * decommissioning using a include hosts file
+ * Decommissioning using a pre-configured include hosts file
*/
@Test
public void testDecommissionWithIncludeHosts() throws Exception {
@@ -86,7 +86,7 @@ public class TestResourceTrackerService
String ip = NetUtils.normalizeHostName("localhost");
writeToHostsFile("host1", ip);
- rm.getNodesListManager().refreshNodes();
+ rm.getNodesListManager().refreshNodes(conf);
nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
@@ -106,7 +106,7 @@ public class TestResourceTrackerService
}
/**
- * decommissioning using a exclude hosts file
+ * Decommissioning using a pre-configured exclude hosts file
*/
@Test
public void testDecommissionWithExcludeHosts() throws Exception {
@@ -133,7 +133,7 @@ public class TestResourceTrackerService
String ip = NetUtils.normalizeHostName("localhost");
writeToHostsFile("host2", ip);
- rm.getNodesListManager().refreshNodes();
+ rm.getNodesListManager().refreshNodes(conf);
nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
@@ -147,7 +147,81 @@ public class TestResourceTrackerService
NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));
checkDecommissionedNMCount(rm, ++metricCount);
}
+
+ /**
+ * Decommissioning using a post-configured include hosts file
+ */
+ @Test
+ public void testAddNewIncludePathToConfiguration() throws Exception {
+ Configuration conf = new Configuration();
+ rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ ClusterMetrics metrics = ClusterMetrics.getMetrics();
+ assert(metrics != null);
+ int initialMetricCount = metrics.getNumDecommisionedNMs();
+ HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ writeToHostsFile("host1");
+ conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+ rm.getNodesListManager().refreshNodes(conf);
+ nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ "Node should not have been decomissioned.",
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals("Node should have been decomissioned but is in state" +
+ nodeHeartbeat.getNodeAction(),
+ NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
+ checkDecommissionedNMCount(rm, ++initialMetricCount);
+ }
+ /**
+ * Decommissioning using a post-configured exclude hosts file
+ */
+ @Test
+ public void testAddNewExcludePathToConfiguration() throws Exception {
+ Configuration conf = new Configuration();
+ rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ ClusterMetrics metrics = ClusterMetrics.getMetrics();
+ assert(metrics != null);
+ int initialMetricCount = metrics.getNumDecommisionedNMs();
+ HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ writeToHostsFile("host2");
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+ rm.getNodesListManager().refreshNodes(conf);
+ nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ "Node should not have been decomissioned.",
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals("Node should have been decomissioned but is in state" +
+ nodeHeartbeat.getNodeAction(),
+ NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
+ checkDecommissionedNMCount(rm, ++initialMetricCount);
+ }
+
@Test
public void testNodeRegistrationFailure() throws Exception {
writeToHostsFile("host1");