You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2019/10/17 17:32:54 UTC
[hadoop] branch branch-3.1 updated: HDFS-14909.
DFSNetworkTopology#chooseRandomWithStorageType() should not decrease
storage count for excluded node which is already part of excluded scope.
Contributed by Surendra Singh Lilhore.
This is an automated email from the ASF dual-hosted git repository.
surendralilhore pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 14beaae HDFS-14909. DFSNetworkTopology#chooseRandomWithStorageType() should not decrease storage count for excluded node which is already part of excluded scope. Contributed by Surendra Singh Lilhore.
14beaae is described below
commit 14beaae65e68725635e428c237c79ba434b8b076
Author: Surendra Singh Lilhore <su...@apache.org>
AuthorDate: Thu Oct 17 22:28:30 2019 +0530
HDFS-14909. DFSNetworkTopology#chooseRandomWithStorageType() should not decrease storage count for excluded node which is already part of excluded scope. Contributed by Surendra Singh Lilhore.
(cherry picked from commit 54dc6b7d720851eb6017906d664aa0fda2698225)
---
.../apache/hadoop/hdfs/net/DFSNetworkTopology.java | 4 +++
.../apache/hadoop/hdfs/TestMissingBlocksAlert.java | 30 ++++++++++++++++++++++
2 files changed, 34 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
index 0884fc0..aea1ff6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
@@ -212,6 +212,10 @@ public class DFSNetworkTopology extends NetworkTopology {
}
if (excludedNodes != null) {
for (Node excludedNode : excludedNodes) {
+ if (excludeRoot != null
+ && excludedNode.getNetworkLocation().startsWith(excludedScope)) {
+ continue;
+ }
if (excludedNode instanceof DatanodeDescriptor) {
availableCount -= ((DatanodeDescriptor) excludedNode)
.hasStorageType(type) ? 1 : 0;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
index ca2fe92..7370a28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.junit.Assert;
import org.junit.Test;
@@ -145,4 +146,33 @@ public class TestMissingBlocksAlert {
}
}
}
+
+ @Test
+ public void testMissReplicatedBlockwithTwoRack() throws Exception {
+ Configuration conf = new Configuration();
+ //Start cluster with rack /default/rack1
+ String[] hosts = new String[] {"host0", "host1", "host2", "host3"};
+ String[] racks = new String[] {"/default/rack1", "/default/rack1",
+ "/default/rack1", "/default/rack1"};
+ conf.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ AvailableSpaceBlockPlacementPolicy.class.getName());
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4)
+ .hosts(hosts).racks(racks).build();
+ Path file = new Path("/file2");
+ try {
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ DFSTestUtil.createFile(dfs, file, 1024, (short) 2, 0);
+ dfs.getFileStatus(file);
+ //Add one more rack /default/rack2
+ cluster.startDataNodes(conf, 2, true, null,
+ new String[] {"/default/rack2", "/default/rack2"},
+ new String[] {"host4", "host5"}, null);
+ dfs.setReplication(file, (short) 3);
+ // wait for block replication
+ DFSTestUtil.waitForReplication(dfs, file, (short) 3, 60000);
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org