You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2019/06/20 20:39:50 UTC
[hadoop] branch branch-3.2 updated: HADOOP-16028. Fix
NetworkTopology chooseRandom function to support excluded nodes.
Contributed by Sihai Ke.
This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new f046b88 HADOOP-16028. Fix NetworkTopology chooseRandom function to support excluded nodes. Contributed by Sihai Ke.
f046b88 is described below
commit f046b881f6bc1f6f791d32787bbd5673f106f8e5
Author: Inigo Goiri <in...@apache.org>
AuthorDate: Fri Jan 4 09:55:09 2019 -0800
HADOOP-16028. Fix NetworkTopology chooseRandom function to support excluded nodes. Contributed by Sihai Ke.
(cherry picked from commit f4e18242bd8117a5c506ec6d3f25c85011fa82d0)
---
.../org/apache/hadoop/net/NetworkTopology.java | 9 ++++--
.../org/apache/hadoop/net/TestClusterTopology.java | 35 ++++++++++++++++++++++
2 files changed, 42 insertions(+), 2 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 1f077a7..f7f60ec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -532,8 +532,13 @@ public class NetworkTopology {
if (excludedScope == null) {
availableNodes = countNumOfAvailableNodes(scope, excludedNodes);
} else {
- availableNodes =
- countNumOfAvailableNodes("~" + excludedScope, excludedNodes);
+ netlock.readLock().lock();
+ try {
+ availableNodes = countNumOfAvailableNodes(scope, excludedNodes) -
+ countNumOfAvailableNodes(excludedScope, excludedNodes);
+ } finally {
+ netlock.readLock().unlock();
+ }
}
LOG.debug("Choosing random from {} available nodes on node {},"
+ " scope={}, excludedScope={}, excludeNodes={}. numOfDatanodes={}.",
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
index 6aad6c5..fbed605 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.net;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Arrays;
import org.apache.commons.math3.stat.inference.ChiSquareTest;
import org.apache.hadoop.conf.Configuration;
@@ -194,6 +195,40 @@ public class TestClusterTopology extends Assert {
2, histogram.size());
}
+ @Test
+ public void testChooseRandomExcluded() {
+ // create the topology
+ // a1
+ // b1------|--------b2
+ // | |
+ // c1-----|-----c2 c3
+ // / \ | |
+ // / \ | |
+ // node1 node2 node3 node4
+
+ NetworkTopology cluster = NetworkTopology.getInstance(new Configuration());
+ NodeElement node1 = getNewNode("node1", "/a1/b1/c1");
+ cluster.add(node1);
+ NodeElement node2 = getNewNode("node2", "/a1/b1/c1");
+ cluster.add(node2);
+ NodeElement node3 = getNewNode("node3", "/a1/b1/c2");
+ cluster.add(node3);
+ NodeElement node4 = getNewNode("node4", "/a1/b2/c3");
+ cluster.add(node4);
+
+ Node node = cluster.chooseRandom("/a1/b1", "/a1/b1/c1", null);
+ assertSame("node3", node.getName());
+
+ node = cluster.chooseRandom("/a1/b1", "/a1/b1/c1", Arrays.asList(node1));
+ assertSame("node3", node.getName());
+
+ node = cluster.chooseRandom("/a1/b1", "/a1/b1/c1", Arrays.asList(node3));
+ assertNull(node);
+
+ node = cluster.chooseRandom("/a1/b1", "/a1/b1/c1", Arrays.asList(node4));
+ assertSame("node3", node.getName());
+ }
+
private NodeElement getNewNode(String name, String rackLocation) {
NodeElement node = new NodeElement(name);
node.setNetworkLocation(rackLocation);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org