You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2016/04/15 22:37:42 UTC
hadoop git commit: YARN-4940. yarn node -list -all failed if RM start
with decommissioned node. Contributed by sandflee
Repository: hadoop
Updated Branches:
refs/heads/trunk fdbafbc9e -> 69f3d428d
YARN-4940. yarn node -list -all failed if RM start with decommissioned node. Contributed by sandflee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69f3d428
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69f3d428
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69f3d428
Branch: refs/heads/trunk
Commit: 69f3d428d5c3ab0c79cacffc22b1f59408622ae7
Parents: fdbafbc
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Apr 15 20:36:45 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Apr 15 20:36:45 2016 +0000
----------------------------------------------------------------------
.../resourcemanager/NodesListManager.java | 36 ++------------
.../resourcemanager/rmnode/RMNodeImpl.java | 4 +-
.../resourcemanager/TestClientRMService.java | 49 +++++++++++++++++++-
.../resourcemanager/TestRMNodeTransitions.java | 4 +-
4 files changed, 55 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69f3d428/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index ec2708e..121c418 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -163,7 +163,7 @@ public class NodesListManager extends CompositeService implements
private void setDecomissionedNMs() {
Set<String> excludeList = hostsReader.getExcludedHosts();
for (final String host : excludeList) {
- UnknownNodeId nodeId = new UnknownNodeId(host);
+ NodeId nodeId = createUnknownNodeId(host);
RMNodeImpl rmNode = new RMNodeImpl(nodeId,
rmContext, host, -1, -1, new UnknownNode(host), null, null);
rmContext.getInactiveRMNodes().put(nodeId, rmNode);
@@ -430,38 +430,8 @@ public class NodesListManager extends CompositeService implements
* A NodeId instance needed upon startup for populating inactive nodes Map.
* It only knows the hostname/ip and marks the port to -1 or invalid.
*/
- public static class UnknownNodeId extends NodeId {
-
- private String host;
-
- public UnknownNodeId(String host) {
- this.host = host;
- }
-
- @Override
- public String getHost() {
- return this.host;
- }
-
- @Override
- protected void setHost(String hst) {
-
- }
-
- @Override
- public int getPort() {
- return -1;
- }
-
- @Override
- protected void setPort(int port) {
-
- }
-
- @Override
- protected void build() {
-
- }
+ public static NodeId createUnknownNodeId(String host) {
+ return NodeId.newInstance(host, -1);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69f3d428/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 5f8317e..9b80716 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -786,8 +786,8 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
if (previousRMNode != null) {
rmNode.updateMetricsForRejoinedNode(previousRMNode.getState());
} else {
- NodesListManager.UnknownNodeId unknownNodeId =
- new NodesListManager.UnknownNodeId(nodeId.getHost());
+ NodeId unknownNodeId =
+ NodesListManager.createUnknownNodeId(nodeId.getHost());
previousRMNode =
rmNode.context.getInactiveRMNodes().remove(unknownNodeId);
if (previousRMNode != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69f3d428/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 331f3ac..bb31f6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -28,6 +28,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
@@ -249,7 +251,7 @@ public class TestClientRMService {
Assert.assertTrue(report.getNodeLabels() != null
&& report.getNodeLabels().isEmpty());
}
-
+
rpc.stopProxy(client, conf);
rm.close();
}
@@ -1566,4 +1568,49 @@ public class TestClientRMService {
Assert.assertEquals("Incorrect priority has been returned", expected,
updateApplicationPriority.getApplicationPriority().getPriority());
}
+
+ private void createExcludeFile(String filename) throws IOException {
+ File file = new File(filename);
+ if (file.exists()) {
+ file.delete();
+ }
+
+ FileOutputStream out = new FileOutputStream(file);
+ out.write("decommisssionedHost".getBytes());
+ out.close();
+ }
+
+ @Test
+ public void testRMStartWithDecommissionedNode() throws Exception {
+ String excludeFile = "excludeFile";
+ createExcludeFile(excludeFile);
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
+ excludeFile);
+ MockRM rm = new MockRM(conf) {
+ protected ClientRMService createClientRMService() {
+ return new ClientRMService(this.rmContext, scheduler,
+ this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
+ this.getRMContext().getRMDelegationTokenSecretManager());
+ };
+ };
+ rm.start();
+
+ YarnRPC rpc = YarnRPC.create(conf);
+ InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
+ LOG.info("Connecting to ResourceManager at " + rmAddress);
+ ApplicationClientProtocol client =
+ (ApplicationClientProtocol) rpc
+ .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
+
+ // Make call
+ GetClusterNodesRequest request =
+ GetClusterNodesRequest.newInstance(EnumSet.allOf(NodeState.class));
+ List<NodeReport> nodeReports = client.getClusterNodes(request).getNodeReports();
+ Assert.assertEquals(1, nodeReports.size());
+
+ rm.stop();
+ rpc.stopProxy(client, conf);
+ new File(excludeFile).delete();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69f3d428/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index 6ba360b..7c03574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -690,8 +690,8 @@ public class TestRMNodeTransitions {
@Test
public void testUnknownNodeId() {
- NodesListManager.UnknownNodeId nodeId =
- new NodesListManager.UnknownNodeId("host1");
+ NodeId nodeId =
+ NodesListManager.createUnknownNodeId("host1");
RMNodeImpl node =
new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
rmContext.getInactiveRMNodes().putIfAbsent(nodeId,node);