You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2019/10/21 13:24:00 UTC
[hadoop] branch branch-3.1 updated: HDFS-14913. Correct the value
of available count in DFSNetworkTopology#chooseRandomWithStorageType().
Contributed by Ayush Saxena.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 9067a11 HDFS-14913. Correct the value of available count in DFSNetworkTopology#chooseRandomWithStorageType(). Contributed by Ayush Saxena.
9067a11 is described below
commit 9067a11b0d048e76026c4a4eb28d9333dfd75f66
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Mon Oct 21 18:35:12 2019 +0530
HDFS-14913. Correct the value of available count in DFSNetworkTopology#chooseRandomWithStorageType(). Contributed by Ayush Saxena.
---
.../apache/hadoop/hdfs/net/DFSNetworkTopology.java | 11 +++++++++--
.../hadoop/hdfs/net/TestDFSNetworkTopology.java | 19 +++++++++++++++++++
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
index aea1ff6..886411a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
@@ -212,8 +212,7 @@ public class DFSNetworkTopology extends NetworkTopology {
}
if (excludedNodes != null) {
for (Node excludedNode : excludedNodes) {
- if (excludeRoot != null
- && excludedNode.getNetworkLocation().startsWith(excludedScope)) {
+ if (excludeRoot != null && isNodeInScope(excludedNode, excludedScope)) {
continue;
}
if (excludedNode instanceof DatanodeDescriptor) {
@@ -259,6 +258,14 @@ public class DFSNetworkTopology extends NetworkTopology {
return chosen;
}
+ private boolean isNodeInScope(Node node, String scope) {
+ if (!scope.endsWith("/")) {
+ scope += "/";
+ }
+ String nodeLocation = node.getNetworkLocation() + "/";
+ return nodeLocation.startsWith(scope);
+ }
+
/**
* Choose a random node that has the required storage type, under the given
* root, with an excluded subtree root (could also just be a leaf node).
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
index fdb41a2..380466e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
@@ -581,4 +581,23 @@ public class TestDFSNetworkTopology {
assertTrue(dd.getHostName().equals("host7"));
}
}
+
+ @Test
+ public void testChooseRandomWithStorageTypeNoAvlblNode() {
+ DFSNetworkTopology dfsCluster =
+ DFSNetworkTopology.getInstance(new Configuration());
+ final String[] racks = {"/default/rack1", "/default/rack10"};
+ final String[] hosts = {"host1", "host2"};
+ final StorageType[] types = {StorageType.DISK, StorageType.DISK};
+ final DatanodeStorageInfo[] storages =
+ DFSTestUtil.createDatanodeStorageInfos(2, racks, hosts, types);
+ DatanodeDescriptor[] dns = DFSTestUtil.toDatanodeDescriptor(storages);
+ dfsCluster.add(dns[0]);
+ dfsCluster.add(dns[1]);
+ HashSet<Node> excluded = new HashSet<>();
+ excluded.add(dns[1]);
+ Node n = dfsCluster.chooseRandomWithStorageType("/default",
+ "/default/rack1", excluded, StorageType.DISK);
+ assertNull("No node should have been selected.", n);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org