You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/01/07 07:24:04 UTC
[2/4] hadoop git commit: HDFS-9600. do not check replication if the
block is under construction (Contributed by Phil Yang)
HDFS-9600. do not check replication if the block is under construction (Contributed by Phil Yang)
(cherry picked from commit 34cd7cd76505d01ec251e30837c94ab03319a0c1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa710bd4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa710bd4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa710bd4
Branch: refs/heads/branch-2
Commit: aa710bd461b593b0f3d7d7ac41ca68e1aa3fa9d6
Parents: 2c841e6
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 7 11:27:42 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 7 11:30:57 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 3 +-
.../blockmanagement/DecommissionManager.java | 5 +-
.../blockmanagement/TestBlockManager.java | 63 +++++++++++++++++++-
4 files changed, 69 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa710bd4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4f56910..0c8dde3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2861,6 +2861,9 @@ Release 2.6.4 - UNRELEASED
BUG FIXES
+ HDFS-9600. do not check replication if the block is under construction
+ (Phil Yang via vinayakumarb)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa710bd4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 4e7090d..40a1ae1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3595,7 +3595,8 @@ public class BlockManager implements BlockStatsMXBean {
*/
boolean isNeededReplication(BlockInfo storedBlock, int current) {
int expected = storedBlock.getReplication();
- return current < expected || !isPlacementPolicySatisfied(storedBlock);
+ return storedBlock.isComplete()
+ && (current < expected || !isPlacementPolicySatisfied(storedBlock));
}
public short getExpectedReplicaNum(BlockInfo block) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa710bd4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index c895a45..ec9a710 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -253,8 +253,9 @@ public class DecommissionManager {
NumberReplicas numberReplicas) {
final int numExpected = block.getReplication();
final int numLive = numberReplicas.liveReplicas();
- if (!blockManager.isNeededReplication(block, numLive)) {
- // Block doesn't need replication. Skip.
+ if (numLive >= numExpected
+ && blockManager.isPlacementPolicySatisfied(block)) {
+ // Block has enough replica, skip
LOG.trace("Block {} does not need replication.", block);
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa710bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 1cebac1..e9c9ec3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -32,6 +32,7 @@ import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
@@ -48,16 +49,22 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
@@ -70,8 +77,11 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetworkTopology;
@@ -396,8 +406,57 @@ public class TestBlockManager {
BlockInfo block = addBlockOnNodes(testIndex, origNodes);
assertFalse(bm.isNeededReplication(block, bm.countLiveNodes(block)));
}
-
-
+
+ @Test(timeout = 60000)
+ public void testNeededReplicationWhileAppending() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ String src = "/test-file";
+ Path file = new Path(src);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ try {
+ BlockManager bm = cluster.getNamesystem().getBlockManager();
+ FileSystem fs = cluster.getFileSystem();
+ NamenodeProtocols namenode = cluster.getNameNodeRpc();
+ DFSOutputStream out = null;
+ try {
+ out = (DFSOutputStream) (fs.create(file).
+ getWrappedStream());
+ out.write(1);
+ out.hflush();
+ out.close();
+ FSDataInputStream in = null;
+ ExtendedBlock oldBlock = null;
+ LocatedBlock oldLoactedBlock = null;
+ try {
+ in = fs.open(file);
+ oldLoactedBlock = DFSTestUtil.getAllBlocks(in).get(0);
+ oldBlock = oldLoactedBlock.getBlock();
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ String clientName =
+ ((DistributedFileSystem) fs).getClient().getClientName();
+ namenode.append(src, clientName,
+ new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND)));
+ LocatedBlock newLocatedBlock =
+ namenode.updateBlockForPipeline(oldBlock, clientName);
+ ExtendedBlock newBlock =
+ new ExtendedBlock(oldBlock.getBlockPoolId(), oldBlock.getBlockId(),
+ oldBlock.getNumBytes(),
+ newLocatedBlock.getBlock().getGenerationStamp());
+ namenode.updatePipeline(clientName, oldBlock, newBlock,
+ oldLoactedBlock.getLocations(), oldLoactedBlock.getStorageIDs());
+ BlockInfo bi = bm.getStoredBlock(newBlock.getLocalBlock());
+ assertFalse(bm.isNeededReplication(bi, bm.countLiveNodes(bi)));
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
/**
* Tell the block manager that replication is completed for the given
* pipeline.