You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2019/09/27 13:28:58 UTC
[hadoop] branch trunk updated: HDFS-14849. Erasure Coding: the
internal block is replicated many times when datanode is decommissioning.
Contributed by HuangTao.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ce58c05 HDFS-14849. Erasure Coding: the internal block is replicated many times when datanode is decommissioning. Contributed by HuangTao.
ce58c05 is described below
commit ce58c05f1d89a72c787f3571f78a9464d0ab3933
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Fri Sep 27 18:26:47 2019 +0530
HDFS-14849. Erasure Coding: the internal block is replicated many times when datanode is decommissioning. Contributed by HuangTao.
---
.../hdfs/server/blockmanagement/BlockManager.java | 12 +++--
.../hadoop/hdfs/TestBlockMissingException.java | 2 -
.../server/blockmanagement/TestBlockManager.java | 58 ++++++++++++++++++++++
3 files changed, 65 insertions(+), 7 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 0d61cad..6a62928 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2359,11 +2359,13 @@ public class BlockManager implements BlockStatsMXBean {
if (isStriped) {
blockIndex = ((BlockInfoStriped) block)
.getStorageBlockIndex(storage);
- if (!bitSet.get(blockIndex)) {
- bitSet.set(blockIndex);
- } else if (state == StoredReplicaState.LIVE) {
- numReplicas.subtract(StoredReplicaState.LIVE, 1);
- numReplicas.add(StoredReplicaState.REDUNDANT, 1);
+ if (state == StoredReplicaState.LIVE) {
+ if (!bitSet.get(blockIndex)) {
+ bitSet.set(blockIndex);
+ } else {
+ numReplicas.subtract(StoredReplicaState.LIVE, 1);
+ numReplicas.add(StoredReplicaState.REDUNDANT, 1);
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockMissingException.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockMissingException.java
index e664f99..c679f6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockMissingException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockMissingException.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertTrue;
-import java.io.File;
import java.io.IOException;
import org.slf4j.Logger;
@@ -31,7 +30,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.junit.Test;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 006513c..ba88afe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -21,6 +21,7 @@ import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.Lists;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
@@ -749,6 +750,63 @@ public class TestBlockManager {
}
@Test
+ public void testChooseSrcDNWithDupECInDecommissioningNode() throws Exception {
+ long blockId = -9223372036854775776L; // real ec block id
+ Block aBlock = new Block(blockId, 0, 0);
+ // RS-3-2 EC policy
+ ErasureCodingPolicy ecPolicy =
+ SystemErasureCodingPolicies.getPolicies().get(1);
+ // striped blockInfo
+ BlockInfoStriped aBlockInfoStriped = new BlockInfoStriped(aBlock, ecPolicy);
+ // ec storageInfo
+ DatanodeStorageInfo ds1 = DFSTestUtil.createDatanodeStorageInfo(
+ "storage1", "1.1.1.1", "rack1", "host1");
+ DatanodeStorageInfo ds2 = DFSTestUtil.createDatanodeStorageInfo(
+ "storage2", "2.2.2.2", "rack2", "host2");
+ DatanodeStorageInfo ds3 = DFSTestUtil.createDatanodeStorageInfo(
+ "storage3", "3.3.3.3", "rack3", "host3");
+ DatanodeStorageInfo ds4 = DFSTestUtil.createDatanodeStorageInfo(
+ "storage4", "4.4.4.4", "rack4", "host4");
+ DatanodeStorageInfo ds5 = DFSTestUtil.createDatanodeStorageInfo(
+ "storage5", "5.5.5.5", "rack5", "host5");
+ DatanodeStorageInfo ds6 = DFSTestUtil.createDatanodeStorageInfo(
+ "storage6", "6.6.6.6", "rack6", "host6");
+
+ // link block with storage
+ aBlockInfoStriped.addStorage(ds1, aBlock);
+ aBlockInfoStriped.addStorage(ds2, new Block(blockId + 1, 0, 0));
+ aBlockInfoStriped.addStorage(ds3, new Block(blockId + 2, 0, 0));
+ aBlockInfoStriped.addStorage(ds4, new Block(blockId + 3, 0, 0));
+ aBlockInfoStriped.addStorage(ds5, new Block(blockId + 4, 0, 0));
+ // NOTE: duplicate block 0，this DN will replace the decommission ds1 DN
+ aBlockInfoStriped.addStorage(ds6, aBlock);
+
+ addEcBlockToBM(blockId, ecPolicy);
+ // decommission datanode where store block 0
+ ds1.getDatanodeDescriptor().startDecommission();
+
+ List<DatanodeDescriptor> containingNodes =
+ new LinkedList<DatanodeDescriptor>();
+ List<DatanodeStorageInfo> nodesContainingLiveReplicas =
+ new LinkedList<DatanodeStorageInfo>();
+ NumberReplicas numReplicas = new NumberReplicas();
+ List<Byte> liveBlockIndices = new ArrayList<>();
+
+ bm.chooseSourceDatanodes(
+ aBlockInfoStriped,
+ containingNodes,
+ nodesContainingLiveReplicas,
+ numReplicas, liveBlockIndices,
+ LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY);
+ assertEquals("There are 5 live replicas in " +
+ "[ds2, ds3, ds4, ds5, ds6] datanodes ",
+ 5, numReplicas.liveReplicas());
+ assertEquals("The ds1 datanode is in decommissioning, " +
+ "so there is no redundant replica",
+ 0, numReplicas.redundantInternalBlocks());
+ }
+
+ @Test
public void testFavorDecomUntilHardLimit() throws Exception {
bm.maxReplicationStreams = 0;
bm.replicationStreamsHardLimit = 1;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org