You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2015/09/26 18:05:22 UTC
[15/50] [abbrv] hadoop git commit: HDFS-7980. Incremental BlockReport
will dramatically slow down namenode startup. Contributed by Walter Su
HDFS-7980. Incremental BlockReport will dramatically slow down namenode startup. Contributed by Walter Su
(cherry picked from commit 4e1f2eb3955a97a70cf127dc97ae49201a90f5e0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ed162bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ed162bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ed162bc
Branch: refs/heads/branch-2.6
Commit: 8ed162bcbf89c5b05f2092731f60ee0a746cf4ca
Parents: c76f5b3
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu May 7 11:36:35 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Sep 7 11:46:33 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 8 +-
.../blockmanagement/TestBlockManager.java | 111 +++++++++++++++++++
3 files changed, 118 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed162bc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 759a20f..d2f07c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -153,6 +153,9 @@ Release 2.6.1 - UNRELEASED
HDFS-8270. create() always retried with hardcoded timeout when file already
exists with open lease (J.Andreina via vinayakumarb)
+ HDFS-7980. Incremental BlockReport will dramatically slow down namenode
+ startup. (Walter Su via szetszwo)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed162bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index bd9d2f7..846bc92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1820,7 +1820,7 @@ public class BlockManager {
return !node.hasStaleStorages();
}
- if (storageInfo.numBlocks() == 0) {
+ if (storageInfo.getBlockReportCount() == 0) {
// The first block report can be processed a lot more efficiently than
// ordinary block reports. This shortens restart times.
processFirstBlockReport(storageInfo, newReport);
@@ -2043,7 +2043,7 @@ public class BlockManager {
final BlockListAsLongs report) throws IOException {
if (report == null) return;
assert (namesystem.hasWriteLock());
- assert (storageInfo.numBlocks() == 0);
+ assert (storageInfo.getBlockReportCount() == 0);
BlockReportIterator itBR = report.getBlockReportIterator();
while(itBR.hasNext()) {
@@ -2463,14 +2463,14 @@ public class BlockManager {
}
// just add it
- storageInfo.addBlock(storedBlock);
+ boolean result = storageInfo.addBlock(storedBlock);
// Now check for completion of blocks and safe block count
int numCurrentReplica = countLiveNodes(storedBlock);
if (storedBlock.getBlockUCState() == BlockUCState.COMMITTED
&& numCurrentReplica >= minReplication) {
completeBlock(storedBlock.getBlockCollection(), storedBlock, false);
- } else if (storedBlock.isComplete()) {
+ } else if (storedBlock.isComplete() && result == true) {
// check whether safe replication is reached for the block
// only complete blocks are counted towards that.
// In the case that the block just became complete above, completeBlock()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed162bc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 7eec52d..eeec73c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -47,10 +47,15 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetworkTopology;
import org.junit.Assert;
@@ -653,6 +658,112 @@ public class TestBlockManager {
new BlockListAsLongs(null, null), null, false);
assertEquals(1, ds.getBlockReportCount());
}
+
+ /**
+ * test when NN starts and in same mode, it receives an incremental blockReport
+ * firstly. Then receives first full block report.
+ */
+ @Test
+ public void testSafeModeIBRBeforeFirstFullBR() throws Exception {
+ // pretend to be in safemode
+ doReturn(true).when(fsn).isInStartupSafeMode();
+
+ DatanodeDescriptor node = nodes.get(0);
+ DatanodeStorageInfo ds = node.getStorageInfos()[0];
+ node.isAlive = true;
+ DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, "");
+
+ // register new node
+ bm.getDatanodeManager().registerDatanode(nodeReg);
+ bm.getDatanodeManager().addDatanode(node);
+ assertEquals(node, bm.getDatanodeManager().getDatanode(node));
+ assertEquals(0, ds.getBlockReportCount());
+ // Build a incremental report
+ List<ReceivedDeletedBlockInfo> rdbiList =
+ new ArrayList<ReceivedDeletedBlockInfo>();
+ // Build a full report
+ List<Block> blocks = new ArrayList<Block>();
+
+ // blk_42 is finalized.
+ long receivedBlockId = 42; // arbitrary
+ BlockInfo receivedBlock = addBlockToBM(receivedBlockId);
+ rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivedBlock),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+ blocks.add(new FinalizedReplica(receivedBlock, null, null));
+
+ // blk_43 is under construction.
+ long receivingBlockId = 43;
+ BlockInfo receivingBlock = addUcBlockToBM(receivingBlockId);
+ rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingBlock),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
+ blocks.add(new ReplicaBeingWritten(receivingBlock, null, null, null));
+
+ // blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record.
+ long receivingReceivedBlockId = 44;
+ BlockInfo receivingReceivedBlock = addBlockToBM(receivingReceivedBlockId);
+ rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
+ rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+ blocks.add(new FinalizedReplica(receivingReceivedBlock, null, null));
+
+ // blk_45 is not in full BR, because it's deleted.
+ long ReceivedDeletedBlockId = 45;
+ rdbiList.add(new ReceivedDeletedBlockInfo(
+ new Block(ReceivedDeletedBlockId),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+ rdbiList.add(new ReceivedDeletedBlockInfo(
+ new Block(ReceivedDeletedBlockId),
+ ReceivedDeletedBlockInfo.BlockStatus.DELETED_BLOCK, null));
+
+ // blk_46 exists in DN for a long time, so it's in full BR, but not in IBR.
+ long existedBlockId = 46;
+ BlockInfo existedBlock = addBlockToBM(existedBlockId);
+ blocks.add(new FinalizedReplica(existedBlock, null, null));
+
+ // process IBR and full BR
+ StorageReceivedDeletedBlocks srdb =
+ new StorageReceivedDeletedBlocks(new DatanodeStorage(ds.getStorageID()),
+ rdbiList.toArray(new ReceivedDeletedBlockInfo[rdbiList.size()]));
+ bm.processIncrementalBlockReport(node, srdb);
+ // Make sure it's the first full report
+ assertEquals(0, ds.getBlockReportCount());
+ bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
+ new BlockListAsLongs(blocks,
+ (List<ReplicaInfo>) new ArrayList<ReplicaInfo>()), null, false);
+ assertEquals(1, ds.getBlockReportCount());
+
+ // verify the storage info is correct
+ assertTrue(bm.getStoredBlock(new Block(receivedBlockId)).findStorageInfo
+ (ds) >= 0);
+ assertTrue(((BlockInfoUnderConstruction) bm.
+ getStoredBlock(new Block(receivingBlockId))).getNumExpectedLocations() > 0);
+ assertTrue(bm.getStoredBlock(new Block(receivingReceivedBlockId))
+ .findStorageInfo(ds) >= 0);
+ assertNull(bm.getStoredBlock(new Block(ReceivedDeletedBlockId)));
+ assertTrue(bm.getStoredBlock(new Block(existedBlock)).findStorageInfo
+ (ds) >= 0);
+ }
+
+ private BlockInfo addBlockToBM(long blkId) {
+ Block block = new Block(blkId);
+ BlockInfo blockInfo =
+ new BlockInfo(block, (short) 3);
+ BlockCollection bc = Mockito.mock(BlockCollection.class);
+ Mockito.doReturn((short) 3).when(bc).getBlockReplication();
+ bm.blocksMap.addBlockCollection(blockInfo, bc);
+ return blockInfo;
+ }
+
+ private BlockInfo addUcBlockToBM(long blkId) {
+ Block block = new Block(blkId);
+ BlockInfoUnderConstruction blockInfo =
+ new BlockInfoUnderConstruction(block, (short) 3);
+ BlockCollection bc = Mockito.mock(BlockCollection.class);
+ Mockito.doReturn((short) 3).when(bc).getBlockReplication();
+ bm.blocksMap.addBlockCollection(blockInfo, bc);
+ return blockInfo;
+ }
/**
* Tests that a namenode doesn't choose a datanode with full disks to