You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/01/07 07:24:03 UTC
[1/4] hadoop git commit: HDFS-9600. do not check replication if the
block is under construction (Contributed by Phil Yang)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2c841e656 -> aa710bd46
refs/heads/branch-2.6 4be18edf2 -> 79da1283d
refs/heads/branch-2.7 258ef9f9c -> 5dc5bb281
refs/heads/trunk 2b252844e -> 34cd7cd76
HDFS-9600. do not check replication if the block is under construction (Contributed by Phil Yang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34cd7cd7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34cd7cd7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34cd7cd7
Branch: refs/heads/trunk
Commit: 34cd7cd76505d01ec251e30837c94ab03319a0c1
Parents: 2b25284
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 7 11:27:42 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 7 11:27:42 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 3 +-
.../blockmanagement/DecommissionManager.java | 5 +-
.../blockmanagement/TestBlockManager.java | 58 ++++++++++++++++++++
4 files changed, 66 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34cd7cd7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d625b93..dfa9701 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -3724,6 +3724,9 @@ Release 2.6.4 - UNRELEASED
BUG FIXES
+ HDFS-9600. do not check replication if the block is under construction
+ (Phil Yang via vinayakumarb)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34cd7cd7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 0f27240..cee03f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3946,7 +3946,8 @@ public class BlockManager implements BlockStatsMXBean {
*/
boolean isNeededReplication(BlockInfo storedBlock, int current) {
int expected = getExpectedReplicaNum(storedBlock);
- return current < expected || !isPlacementPolicySatisfied(storedBlock);
+ return storedBlock.isComplete()
+ && (current < expected || !isPlacementPolicySatisfied(storedBlock));
}
public short getExpectedReplicaNum(BlockInfo block) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34cd7cd7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index 8656f69..2a5d63c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -243,8 +243,9 @@ public class DecommissionManager {
NumberReplicas numberReplicas) {
final int numExpected = blockManager.getExpectedReplicaNum(block);
final int numLive = numberReplicas.liveReplicas();
- if (!blockManager.isNeededReplication(block, numLive)) {
- // Block doesn't need replication. Skip.
+ if (numLive >= numExpected
+ && blockManager.isPlacementPolicySatisfied(block)) {
+ // Block has enough replica, skip
LOG.trace("Block {} does not need replication.", block);
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34cd7cd7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index d92ac90..4a74d2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -32,6 +32,7 @@ import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
@@ -48,16 +49,22 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
@@ -70,8 +77,11 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetworkTopology;
@@ -397,6 +407,54 @@ public class TestBlockManager {
assertFalse(bm.isNeededReplication(block, bm.countLiveNodes(block)));
}
+ @Test(timeout = 60000)
+ public void testNeededReplicationWhileAppending() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ String src = "/test-file";
+ Path file = new Path(src);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ try {
+ BlockManager bm = cluster.getNamesystem().getBlockManager();
+ FileSystem fs = cluster.getFileSystem();
+ NamenodeProtocols namenode = cluster.getNameNodeRpc();
+ DFSOutputStream out = null;
+ try {
+ out = (DFSOutputStream) (fs.create(file).
+ getWrappedStream());
+ out.write(1);
+ out.hflush();
+ out.close();
+ FSDataInputStream in = null;
+ ExtendedBlock oldBlock = null;
+ try {
+ in = fs.open(file);
+ oldBlock = DFSTestUtil.getAllBlocks(in).get(0).getBlock();
+ } finally {
+ IOUtils.closeStream(in);
+ }
+
+ String clientName =
+ ((DistributedFileSystem) fs).getClient().getClientName();
+ namenode.append(src, clientName, new EnumSetWritable<>(
+ EnumSet.of(CreateFlag.APPEND)));
+ LocatedBlock newLocatedBlock =
+ namenode.updateBlockForPipeline(oldBlock, clientName);
+ ExtendedBlock newBlock =
+ new ExtendedBlock(oldBlock.getBlockPoolId(), oldBlock.getBlockId(),
+ oldBlock.getNumBytes(),
+ newLocatedBlock.getBlock().getGenerationStamp());
+ namenode.updatePipeline(clientName, oldBlock, newBlock,
+ newLocatedBlock.getLocations(), newLocatedBlock.getStorageIDs());
+ BlockInfo bi = bm.getStoredBlock(newBlock.getLocalBlock());
+ assertFalse(bm.isNeededReplication(bi, bm.countLiveNodes(bi)));
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
/**
* Tell the block manager that replication is completed for the given
[4/4] hadoop git commit: HDFS-9600. do not check replication if the
block is under construction (Contributed by Phil Yang)
Posted by vi...@apache.org.
HDFS-9600. do not check replication if the block is under construction (Contributed by Phil Yang)
(cherry picked from commit 34cd7cd76505d01ec251e30837c94ab03319a0c1)
(cherry picked from commit aa710bd461b593b0f3d7d7ac41ca68e1aa3fa9d6)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79da1283
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79da1283
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79da1283
Branch: refs/heads/branch-2.6
Commit: 79da1283d6e64a6b715e02d00381b4178362e394
Parents: 4be18ed
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 7 11:27:42 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 7 11:52:27 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 16 +++--
.../blockmanagement/TestBlockManager.java | 65 +++++++++++++++++++-
3 files changed, 78 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79da1283/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 68ba352..f760d36 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -27,6 +27,9 @@ Release 2.6.4 - UNRELEASED
HDFS-9445. Datanode may deadlock while handling a bad volume.
(Wlater Su via Kihwal)
+ HDFS-9600. do not check replication if the block is under construction
+ (Phil Yang via vinayakumarb)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79da1283/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index dceb85c..2d08831 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3290,8 +3290,9 @@ public class BlockManager {
NumberReplicas num = countNodes(block);
int curReplicas = num.liveReplicas();
int curExpectedReplicas = getReplication(block);
-
- if (isNeededReplication(block, curExpectedReplicas, curReplicas)) {
+
+ if (curReplicas < curExpectedReplicas
+ || !isPlacementPolicySatisfied(block)) {
if (curExpectedReplicas > curReplicas) {
if (bc.isUnderConstruction()) {
if (block.equals(bc.getLastBlock()) && curReplicas > minReplication) {
@@ -3502,8 +3503,15 @@ public class BlockManager {
* A block needs replication if the number of replicas is less than expected
* or if it does not have enough racks.
*/
- private boolean isNeededReplication(Block b, int expected, int current) {
- return current < expected || !isPlacementPolicySatisfied(b);
+ boolean isNeededReplication(Block b, int expected, int current) {
+ BlockInfo blockInfo;
+ if (b instanceof BlockInfo) {
+ blockInfo = (BlockInfo) b;
+ } else {
+ blockInfo = getStoredBlock(b);
+ }
+ return blockInfo.isComplete()
+ && (current < expected || !isPlacementPolicySatisfied(b));
}
public long getMissingBlocksCount() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79da1283/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 653a8cf..8678334 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -30,21 +30,28 @@ import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
@@ -54,8 +61,11 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetworkTopology;
import org.junit.Assert;
@@ -373,8 +383,59 @@ public class TestBlockManager {
bm.processMisReplicatedBlocks();
assertEquals(0, bm.numOfUnderReplicatedBlocks());
}
-
-
+
+ @Test(timeout = 60000)
+ public void testNeededReplicationWhileAppending() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ String src = "/test-file";
+ Path file = new Path(src);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ try {
+ BlockManager bm = cluster.getNamesystem().getBlockManager();
+ FileSystem fs = cluster.getFileSystem();
+ NamenodeProtocols namenode = cluster.getNameNodeRpc();
+ DFSOutputStream out = null;
+ try {
+ out = (DFSOutputStream) (fs.create(file).
+ getWrappedStream());
+ out.write(1);
+ out.hflush();
+ out.close();
+ FSDataInputStream in = null;
+ ExtendedBlock oldBlock = null;
+ LocatedBlock oldLoactedBlock = null;
+ try {
+ in = fs.open(file);
+ oldLoactedBlock = DFSTestUtil.getAllBlocks(in).get(0);
+ oldBlock = oldLoactedBlock.getBlock();
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ String clientName =
+ ((DistributedFileSystem) fs).getClient().getClientName();
+ namenode.append(src, clientName);
+ LocatedBlock newLocatedBlock =
+ namenode.updateBlockForPipeline(oldBlock, clientName);
+ ExtendedBlock newBlock =
+ new ExtendedBlock(oldBlock.getBlockPoolId(), oldBlock.getBlockId(),
+ oldBlock.getNumBytes(),
+ newLocatedBlock.getBlock().getGenerationStamp());
+ namenode.updatePipeline(clientName, oldBlock, newBlock,
+ oldLoactedBlock.getLocations(), oldLoactedBlock.getStorageIDs());
+ BlockInfo bi = bm.getStoredBlock(newBlock.getLocalBlock());
+ assertFalse(
+ bm.isNeededReplication(bi, oldLoactedBlock.getLocations().length,
+ bm.countLiveNodes(bi)));
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+
/**
* Tell the block manager that replication is completed for the given
* pipeline.
[2/4] hadoop git commit: HDFS-9600. do not check replication if the
block is under construction (Contributed by Phil Yang)
Posted by vi...@apache.org.
HDFS-9600. do not check replication if the block is under construction (Contributed by Phil Yang)
(cherry picked from commit 34cd7cd76505d01ec251e30837c94ab03319a0c1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa710bd4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa710bd4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa710bd4
Branch: refs/heads/branch-2
Commit: aa710bd461b593b0f3d7d7ac41ca68e1aa3fa9d6
Parents: 2c841e6
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 7 11:27:42 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 7 11:30:57 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 3 +-
.../blockmanagement/DecommissionManager.java | 5 +-
.../blockmanagement/TestBlockManager.java | 63 +++++++++++++++++++-
4 files changed, 69 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa710bd4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4f56910..0c8dde3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2861,6 +2861,9 @@ Release 2.6.4 - UNRELEASED
BUG FIXES
+ HDFS-9600. do not check replication if the block is under construction
+ (Phil Yang via vinayakumarb)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa710bd4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 4e7090d..40a1ae1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3595,7 +3595,8 @@ public class BlockManager implements BlockStatsMXBean {
*/
boolean isNeededReplication(BlockInfo storedBlock, int current) {
int expected = storedBlock.getReplication();
- return current < expected || !isPlacementPolicySatisfied(storedBlock);
+ return storedBlock.isComplete()
+ && (current < expected || !isPlacementPolicySatisfied(storedBlock));
}
public short getExpectedReplicaNum(BlockInfo block) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa710bd4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index c895a45..ec9a710 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -253,8 +253,9 @@ public class DecommissionManager {
NumberReplicas numberReplicas) {
final int numExpected = block.getReplication();
final int numLive = numberReplicas.liveReplicas();
- if (!blockManager.isNeededReplication(block, numLive)) {
- // Block doesn't need replication. Skip.
+ if (numLive >= numExpected
+ && blockManager.isPlacementPolicySatisfied(block)) {
+ // Block has enough replica, skip
LOG.trace("Block {} does not need replication.", block);
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa710bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 1cebac1..e9c9ec3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -32,6 +32,7 @@ import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
@@ -48,16 +49,22 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
@@ -70,8 +77,11 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetworkTopology;
@@ -396,8 +406,57 @@ public class TestBlockManager {
BlockInfo block = addBlockOnNodes(testIndex, origNodes);
assertFalse(bm.isNeededReplication(block, bm.countLiveNodes(block)));
}
-
-
+
+ @Test(timeout = 60000)
+ public void testNeededReplicationWhileAppending() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ String src = "/test-file";
+ Path file = new Path(src);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ try {
+ BlockManager bm = cluster.getNamesystem().getBlockManager();
+ FileSystem fs = cluster.getFileSystem();
+ NamenodeProtocols namenode = cluster.getNameNodeRpc();
+ DFSOutputStream out = null;
+ try {
+ out = (DFSOutputStream) (fs.create(file).
+ getWrappedStream());
+ out.write(1);
+ out.hflush();
+ out.close();
+ FSDataInputStream in = null;
+ ExtendedBlock oldBlock = null;
+ LocatedBlock oldLoactedBlock = null;
+ try {
+ in = fs.open(file);
+ oldLoactedBlock = DFSTestUtil.getAllBlocks(in).get(0);
+ oldBlock = oldLoactedBlock.getBlock();
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ String clientName =
+ ((DistributedFileSystem) fs).getClient().getClientName();
+ namenode.append(src, clientName,
+ new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND)));
+ LocatedBlock newLocatedBlock =
+ namenode.updateBlockForPipeline(oldBlock, clientName);
+ ExtendedBlock newBlock =
+ new ExtendedBlock(oldBlock.getBlockPoolId(), oldBlock.getBlockId(),
+ oldBlock.getNumBytes(),
+ newLocatedBlock.getBlock().getGenerationStamp());
+ namenode.updatePipeline(clientName, oldBlock, newBlock,
+ oldLoactedBlock.getLocations(), oldLoactedBlock.getStorageIDs());
+ BlockInfo bi = bm.getStoredBlock(newBlock.getLocalBlock());
+ assertFalse(bm.isNeededReplication(bi, bm.countLiveNodes(bi)));
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
/**
* Tell the block manager that replication is completed for the given
* pipeline.
[3/4] hadoop git commit: HDFS-9600. do not check replication if the
block is under construction (Contributed by Phil Yang)
Posted by vi...@apache.org.
HDFS-9600. do not check replication if the block is under construction (Contributed by Phil Yang)
(cherry picked from commit 34cd7cd76505d01ec251e30837c94ab03319a0c1)
(cherry picked from commit aa710bd461b593b0f3d7d7ac41ca68e1aa3fa9d6)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5dc5bb28
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5dc5bb28
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5dc5bb28
Branch: refs/heads/branch-2.7
Commit: 5dc5bb2812c450e14ba73e83be0480a3a882838e
Parents: 258ef9f
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 7 11:27:42 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 7 11:50:12 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 9 ++-
.../blockmanagement/DecommissionManager.java | 5 +-
.../blockmanagement/TestBlockManager.java | 64 +++++++++++++++++++-
4 files changed, 76 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dc5bb28/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cf17986..2313f13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1193,6 +1193,9 @@ Release 2.6.4 - UNRELEASED
BUG FIXES
+ HDFS-9600. do not check replication if the block is under construction
+ (Phil Yang via vinayakumarb)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dc5bb28/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index db50e44..ef784d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3444,7 +3444,14 @@ public class BlockManager {
* or if it does not have enough racks.
*/
boolean isNeededReplication(Block b, int expected, int current) {
- return current < expected || !isPlacementPolicySatisfied(b);
+ BlockInfoContiguous blockInfo;
+ if (b instanceof BlockInfoContiguous) {
+ blockInfo = (BlockInfoContiguous) b;
+ } else {
+ blockInfo = getStoredBlock(b);
+ }
+ return blockInfo.isComplete()
+ && (current < expected || !isPlacementPolicySatisfied(b));
}
public long getMissingBlocksCount() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dc5bb28/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index 7f3d778..c76b192 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -253,8 +253,9 @@ public class DecommissionManager {
NumberReplicas numberReplicas) {
final int numExpected = bc.getBlockReplication();
final int numLive = numberReplicas.liveReplicas();
- if (!blockManager.isNeededReplication(block, numExpected, numLive)) {
- // Block doesn't need replication. Skip.
+ if (numLive >= numExpected
+ && blockManager.isPlacementPolicySatisfied(block)) {
+ // Block has enough replica, skip
LOG.trace("Block {} does not need replication.", block);
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dc5bb28/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 920a18d..196a38b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -30,21 +30,28 @@ import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
@@ -53,8 +60,11 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetworkTopology;
import org.junit.Assert;
@@ -371,8 +381,58 @@ public class TestBlockManager {
bm.processMisReplicatedBlocks();
assertEquals(0, bm.numOfUnderReplicatedBlocks());
}
-
-
+
+ @Test(timeout = 60000)
+ public void testNeededReplicationWhileAppending() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ String src = "/test-file";
+ Path file = new Path(src);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ try {
+ BlockManager bm = cluster.getNamesystem().getBlockManager();
+ FileSystem fs = cluster.getFileSystem();
+ NamenodeProtocols namenode = cluster.getNameNodeRpc();
+ DFSOutputStream out = null;
+ try {
+ out = (DFSOutputStream) (fs.create(file).
+ getWrappedStream());
+ out.write(1);
+ out.hflush();
+ out.close();
+ FSDataInputStream in = null;
+ ExtendedBlock oldBlock = null;
+ LocatedBlock oldLoactedBlock = null;
+ try {
+ in = fs.open(file);
+ oldLoactedBlock = DFSTestUtil.getAllBlocks(in).get(0);
+ oldBlock = oldLoactedBlock.getBlock();
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ String clientName =
+ ((DistributedFileSystem) fs).getClient().getClientName();
+ namenode.append(src, clientName, new EnumSetWritable<>(
+ EnumSet.of(CreateFlag.APPEND)));
+ LocatedBlock newLocatedBlock =
+ namenode.updateBlockForPipeline(oldBlock, clientName);
+ ExtendedBlock newBlock =
+ new ExtendedBlock(oldBlock.getBlockPoolId(), oldBlock.getBlockId(),
+ oldBlock.getNumBytes(),
+ newLocatedBlock.getBlock().getGenerationStamp());
+ namenode.updatePipeline(clientName, oldBlock, newBlock,
+ oldLoactedBlock.getLocations(), oldLoactedBlock.getStorageIDs());
+ BlockInfoContiguous bi = bm.getStoredBlock(newBlock.getLocalBlock());
+ assertFalse(bm.isNeededReplication(bi,
+ oldLoactedBlock.getLocations().length, bm.countLiveNodes(bi)));
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
/**
* Tell the block manager that replication is completed for the given
* pipeline.