You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2016/07/19 17:48:13 UTC
[1/2] hadoop git commit: Revert "YARN=5181. ClusterNodeTracker: add
method to get list of nodes matching a specific resourceName. (kasha via
asuresh)"
Repository: hadoop
Updated Branches:
refs/heads/trunk dc2f4b6ac -> cda0a280d
Revert "YARN=5181. ClusterNodeTracker: add method to get list of nodes matching a specific resourceName. (kasha via asuresh)"
This reverts commit e905a42a2c6c88210b5c2120d7daf0bb3739cd1d.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f2d33a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f2d33a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f2d33a5
Branch: refs/heads/trunk
Commit: 5f2d33a5510353eaea7d953953847626726d14c1
Parents: dc2f4b6
Author: Arun Suresh <as...@apache.org>
Authored: Tue Jul 19 10:43:19 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Tue Jul 19 10:43:19 2016 -0700
----------------------------------------------------------------------
.../scheduler/ClusterNodeTracker.java | 56 ++++------------
.../scheduler/TestClusterNodeTracker.java | 69 --------------------
2 files changed, 11 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f2d33a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index e487f69..413bbdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -18,13 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -52,8 +50,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
private Lock writeLock = readWriteLock.writeLock();
private HashMap<NodeId, N> nodes = new HashMap<>();
- private Map<String, N> nodeNameToNodeMap = new HashMap<>();
- private Map<String, List<N>> nodesPerRack = new HashMap<>();
+ private Map<String, Integer> nodesPerRack = new HashMap<>();
private Resource clusterCapacity = Resources.clone(Resources.none());
private Resource staleClusterCapacity = null;
@@ -69,16 +66,14 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
writeLock.lock();
try {
nodes.put(node.getNodeID(), node);
- nodeNameToNodeMap.put(node.getNodeName(), node);
// Update nodes per rack as well
String rackName = node.getRackName();
- List<N> nodesList = nodesPerRack.get(rackName);
- if (nodesList == null) {
- nodesList = new ArrayList<>();
- nodesPerRack.put(rackName, nodesList);
+ Integer numNodes = nodesPerRack.get(rackName);
+ if (numNodes == null) {
+ numNodes = 0;
}
- nodesList.add(node);
+ nodesPerRack.put(rackName, ++numNodes);
// Update cluster capacity
Resources.addTo(clusterCapacity, node.getTotalResource());
@@ -131,8 +126,8 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
readLock.lock();
String rName = rackName == null ? "NULL" : rackName;
try {
- List<N> nodesList = nodesPerRack.get(rName);
- return nodesList == null ? 0 : nodesList.size();
+ Integer nodeCount = nodesPerRack.get(rName);
+ return nodeCount == null ? 0 : nodeCount;
} finally {
readLock.unlock();
}
@@ -159,18 +154,14 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
LOG.warn("Attempting to remove a non-existent node " + nodeId);
return null;
}
- nodeNameToNodeMap.remove(node.getNodeName());
// Update nodes per rack as well
String rackName = node.getRackName();
- List<N> nodesList = nodesPerRack.get(rackName);
- if (nodesList == null) {
- LOG.error("Attempting to remove node from an empty rack " + rackName);
+ Integer numNodes = nodesPerRack.get(rackName);
+ if (numNodes > 0) {
+ nodesPerRack.put(rackName, --numNodes);
} else {
- nodesList.remove(node);
- if (nodesList.isEmpty()) {
- nodesPerRack.remove(rackName);
- }
+ LOG.error("Attempting to remove node from an empty rack " + rackName);
}
// Update cluster capacity
@@ -306,29 +297,4 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
Collections.sort(sortedList, comparator);
return sortedList;
}
-
- /**
- * Convenience method to return list of nodes corresponding to resourceName
- * passed in the {@link ResourceRequest}.
- *
- * @param resourceName Host/rack name of the resource, or
- * {@link ResourceRequest#ANY}
- * @return list of nodes that match the resourceName
- */
- public List<N> getNodesByResourceName(final String resourceName) {
- Preconditions.checkArgument(
- resourceName != null && !resourceName.isEmpty());
- List<N> retNodes = new ArrayList<>();
- if (ResourceRequest.ANY.equals(resourceName)) {
- return getAllNodes();
- } else if (nodeNameToNodeMap.containsKey(resourceName)) {
- retNodes.add(nodeNameToNodeMap.get(resourceName));
- } else if (nodesPerRack.containsKey(resourceName)) {
- return nodesPerRack.get(resourceName);
- } else {
- LOG.info(
- "Could not find a node matching given resourceName " + resourceName);
- }
- return retNodes;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f2d33a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
deleted file mode 100644
index 7f527f1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSSchedulerNode;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test class to verify ClusterNodeTracker. Using FSSchedulerNode without
- * loss of generality.
- */
-public class TestClusterNodeTracker {
- private ClusterNodeTracker<FSSchedulerNode> nodeTracker =
- new ClusterNodeTracker<>();
-
- @Before
- public void setup() {
- List<RMNode> rmNodes =
- MockNodes.newNodes(2, 4, Resource.newInstance(4096, 4));
- for (RMNode rmNode : rmNodes) {
- nodeTracker.addNode(new FSSchedulerNode(rmNode, false));
- }
- }
-
- @Test
- public void testGetNodeCount() {
- assertEquals("Incorrect number of nodes in the cluster",
- 8, nodeTracker.nodeCount());
-
- assertEquals("Incorrect number of nodes in each rack",
- 4, nodeTracker.nodeCount("rack0"));
- }
-
- @Test
- public void testGetNodesForResourceName() throws Exception {
- assertEquals("Incorrect number of nodes matching ANY",
- 8, nodeTracker.getNodesByResourceName(ResourceRequest.ANY).size());
-
- assertEquals("Incorrect number of nodes matching rack",
- 4, nodeTracker.getNodesByResourceName("rack0").size());
-
- assertEquals("Incorrect number of nodes matching node",
- 1, nodeTracker.getNodesByResourceName("host0").size());
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: YARN-5181. ClusterNodeTracker: add method to
get list of nodes matching a specific resourceName. (kasha via asuresh)
Posted by as...@apache.org.
YARN-5181. ClusterNodeTracker: add method to get list of nodes matching a specific resourceName. (kasha via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cda0a280
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cda0a280
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cda0a280
Branch: refs/heads/trunk
Commit: cda0a280ddd0c77af93d236fc80478c16bbe809a
Parents: 5f2d33a
Author: Arun Suresh <as...@apache.org>
Authored: Tue Jul 19 10:43:37 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Tue Jul 19 10:43:37 2016 -0700
----------------------------------------------------------------------
.../scheduler/ClusterNodeTracker.java | 56 ++++++++++++----
.../scheduler/TestClusterNodeTracker.java | 69 ++++++++++++++++++++
2 files changed, 114 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda0a280/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 413bbdf..e487f69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -18,11 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -50,7 +52,8 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
private Lock writeLock = readWriteLock.writeLock();
private HashMap<NodeId, N> nodes = new HashMap<>();
- private Map<String, Integer> nodesPerRack = new HashMap<>();
+ private Map<String, N> nodeNameToNodeMap = new HashMap<>();
+ private Map<String, List<N>> nodesPerRack = new HashMap<>();
private Resource clusterCapacity = Resources.clone(Resources.none());
private Resource staleClusterCapacity = null;
@@ -66,14 +69,16 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
writeLock.lock();
try {
nodes.put(node.getNodeID(), node);
+ nodeNameToNodeMap.put(node.getNodeName(), node);
// Update nodes per rack as well
String rackName = node.getRackName();
- Integer numNodes = nodesPerRack.get(rackName);
- if (numNodes == null) {
- numNodes = 0;
+ List<N> nodesList = nodesPerRack.get(rackName);
+ if (nodesList == null) {
+ nodesList = new ArrayList<>();
+ nodesPerRack.put(rackName, nodesList);
}
- nodesPerRack.put(rackName, ++numNodes);
+ nodesList.add(node);
// Update cluster capacity
Resources.addTo(clusterCapacity, node.getTotalResource());
@@ -126,8 +131,8 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
readLock.lock();
String rName = rackName == null ? "NULL" : rackName;
try {
- Integer nodeCount = nodesPerRack.get(rName);
- return nodeCount == null ? 0 : nodeCount;
+ List<N> nodesList = nodesPerRack.get(rName);
+ return nodesList == null ? 0 : nodesList.size();
} finally {
readLock.unlock();
}
@@ -154,14 +159,18 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
LOG.warn("Attempting to remove a non-existent node " + nodeId);
return null;
}
+ nodeNameToNodeMap.remove(node.getNodeName());
// Update nodes per rack as well
String rackName = node.getRackName();
- Integer numNodes = nodesPerRack.get(rackName);
- if (numNodes > 0) {
- nodesPerRack.put(rackName, --numNodes);
- } else {
+ List<N> nodesList = nodesPerRack.get(rackName);
+ if (nodesList == null) {
LOG.error("Attempting to remove node from an empty rack " + rackName);
+ } else {
+ nodesList.remove(node);
+ if (nodesList.isEmpty()) {
+ nodesPerRack.remove(rackName);
+ }
}
// Update cluster capacity
@@ -297,4 +306,29 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
Collections.sort(sortedList, comparator);
return sortedList;
}
+
+ /**
+ * Convenience method to return list of nodes corresponding to resourceName
+ * passed in the {@link ResourceRequest}.
+ *
+ * @param resourceName Host/rack name of the resource, or
+ * {@link ResourceRequest#ANY}
+ * @return list of nodes that match the resourceName
+ */
+ public List<N> getNodesByResourceName(final String resourceName) {
+ Preconditions.checkArgument(
+ resourceName != null && !resourceName.isEmpty());
+ List<N> retNodes = new ArrayList<>();
+ if (ResourceRequest.ANY.equals(resourceName)) {
+ return getAllNodes();
+ } else if (nodeNameToNodeMap.containsKey(resourceName)) {
+ retNodes.add(nodeNameToNodeMap.get(resourceName));
+ } else if (nodesPerRack.containsKey(resourceName)) {
+ return nodesPerRack.get(resourceName);
+ } else {
+ LOG.info(
+ "Could not find a node matching given resourceName " + resourceName);
+ }
+ return retNodes;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda0a280/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
new file mode 100644
index 0000000..7f527f1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSSchedulerNode;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test class to verify ClusterNodeTracker. Using FSSchedulerNode without
+ * loss of generality.
+ */
+public class TestClusterNodeTracker {
+ private ClusterNodeTracker<FSSchedulerNode> nodeTracker =
+ new ClusterNodeTracker<>();
+
+ @Before
+ public void setup() {
+ List<RMNode> rmNodes =
+ MockNodes.newNodes(2, 4, Resource.newInstance(4096, 4));
+ for (RMNode rmNode : rmNodes) {
+ nodeTracker.addNode(new FSSchedulerNode(rmNode, false));
+ }
+ }
+
+ @Test
+ public void testGetNodeCount() {
+ assertEquals("Incorrect number of nodes in the cluster",
+ 8, nodeTracker.nodeCount());
+
+ assertEquals("Incorrect number of nodes in each rack",
+ 4, nodeTracker.nodeCount("rack0"));
+ }
+
+ @Test
+ public void testGetNodesForResourceName() throws Exception {
+ assertEquals("Incorrect number of nodes matching ANY",
+ 8, nodeTracker.getNodesByResourceName(ResourceRequest.ANY).size());
+
+ assertEquals("Incorrect number of nodes matching rack",
+ 4, nodeTracker.getNodesByResourceName("rack0").size());
+
+ assertEquals("Incorrect number of nodes matching node",
+ 1, nodeTracker.getNodesByResourceName("host0").size());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org