You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2021/11/16 15:09:21 UTC
[hadoop] branch branch-3.2 updated: HDFS-16182.numOfReplicas is given the wrong value in BlockPlacementPolicyDefault$chooseTarget can cause DataStreamer to fail with Heterogeneous Storage. (#3320)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 78c56f3 HDFS-16182.numOfReplicas is given the wrong value in BlockPlacementPolicyDefault$chooseTarget can cause DataStreamer to fail with Heterogeneous Storage. (#3320)
78c56f3 is described below
commit 78c56f39b4a01ebb313cac381238cc6a7ae7a9b7
Author: Neil <wx...@foxmail.com>
AuthorDate: Tue Nov 16 18:21:50 2021 +0800
HDFS-16182.numOfReplicas is given the wrong value in BlockPlacementPolicyDefault$chooseTarget can cause DataStreamer to fail with Heterogeneous Storage. (#3320)
(cherry picked from commit 5626734a36b9794f4b1af8b9a7d8d0055914a094)
(cherry picked from commit ce3a7155335416ed4665f90f0ddd7ac3a3edd7e6)
---
.../BlockPlacementPolicyDefault.java | 14 +++---
.../apache/hadoop/hdfs/TestBlockStoragePolicy.java | 53 ++++++++++++++++++++++
2 files changed, 60 insertions(+), 7 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index b59f0ca..353e853 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -396,7 +396,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
* @param storageTypes storage type to be considered for target
* @return local node of writer (not chosen node)
*/
- private Node chooseTarget(int numOfReplicas,
+ private Node chooseTarget(final int numOfReplicas,
Node writer,
final Set<Node> excludedNodes,
final long blocksize,
@@ -432,7 +432,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
try {
- if ((numOfReplicas = requiredStorageTypes.size()) == 0) {
+ if (requiredStorageTypes.size() == 0) {
throw new NotEnoughReplicasException(
"All required storage types are unavailable: "
+ " unavailableStorages=" + unavailableStorages
@@ -464,10 +464,10 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
for (DatanodeStorageInfo resultStorage : results) {
addToExcludedNodes(resultStorage.getDatanodeDescriptor(), oldExcludedNodes);
}
- // Set numOfReplicas, since it can get out of sync with the result list
+ // Set newNumOfReplicas, since it can get out of sync with the result list
// if the NotEnoughReplicasException was thrown in chooseRandom().
- numOfReplicas = totalReplicasExpected - results.size();
- return chooseTarget(numOfReplicas, writer, oldExcludedNodes, blocksize,
+ int newNumOfReplicas = totalReplicasExpected - results.size();
+ return chooseTarget(newNumOfReplicas, writer, oldExcludedNodes, blocksize,
maxNodesPerRack, results, false, storagePolicy, unavailableStorages,
newBlock, null);
}
@@ -486,8 +486,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
addToExcludedNodes(resultStorage.getDatanodeDescriptor(),
oldExcludedNodes);
}
- numOfReplicas = totalReplicasExpected - results.size();
- return chooseTarget(numOfReplicas, writer, oldExcludedNodes, blocksize,
+ int newNumOfReplicas = totalReplicasExpected - results.size();
+ return chooseTarget(newNumOfReplicas, writer, oldExcludedNodes, blocksize,
maxNodesPerRack, results, false, storagePolicy, unavailableStorages,
newBlock, null);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 76eb824..ab1f1ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -1300,6 +1300,59 @@ public class TestBlockStoragePolicy {
Assert.assertEquals(StorageType.DISK, targets[1].getStorageType());
}
+ /**
+ * Consider a File with All_SSD storage policy.
+ * 1. Choose 3 DISK DNs for pipeline because SSD DNs no enough at
+ * the beginning.
+ * 2. One of DISK DNs fails And it need choose one new DN for existing.
+ * pipeline {@link DataStreamer addDatanode2ExistingPipeline()}.
+ * Make sure the number of target DNs are 3.
+ * see HDFS-16182.
+ */
+ @Test
+ public void testAddDatanode2ExistingPipelineInSsd() throws Exception {
+ BlockStoragePolicy policy = POLICY_SUITE.getPolicy(ALLSSD);
+
+ final String[] racks = {"/d1/r1", "/d2/r2", "/d3/r3", "/d4/r4", "/d5/r5",
+ "/d6/r6", "/d7/r7"};
+ final String[] hosts = {"host1", "host2", "host3", "host4", "host5",
+ "host6", "host7"};
+ final StorageType[] disks = {StorageType.DISK, StorageType.DISK, StorageType.DISK};
+
+ final DatanodeStorageInfo[] diskStorages
+ = DFSTestUtil.createDatanodeStorageInfos(7, racks, hosts, disks);
+ final DatanodeDescriptor[] dataNodes
+ = DFSTestUtil.toDatanodeDescriptor(diskStorages);
+ for (int i = 0; i < dataNodes.length; i++) {
+ BlockManagerTestUtil.updateStorage(dataNodes[i],
+ new DatanodeStorage("ssd" + i + 1, DatanodeStorage.State.NORMAL,
+ StorageType.SSD));
+ }
+
+ FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+ File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ new File(baseDir, "name").getPath());
+ DFSTestUtil.formatNameNode(conf);
+ NameNode namenode = new NameNode(conf);
+
+ final BlockManager bm = namenode.getNamesystem().getBlockManager();
+ BlockPlacementPolicy replicator = bm.getBlockPlacementPolicy();
+ NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
+ for (DatanodeDescriptor datanode : dataNodes) {
+ cluster.add(datanode);
+ }
+ // chsenDs are DISK StorageType to simulate not enough SDD Storage
+ List<DatanodeStorageInfo> chsenDs = new ArrayList<>();
+ chsenDs.add(diskStorages[0]);
+ chsenDs.add(diskStorages[1]);
+ DatanodeStorageInfo[] targets = replicator.chooseTarget("/foo", 1,
+ null, chsenDs, true,
+ new HashSet<Node>(), 0, policy, null);
+ Assert.assertEquals(3, targets.length);
+ }
+
@Test
public void testGetFileStoragePolicyAfterRestartNN() throws Exception {
//HDFS8219
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org