You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2014/03/26 19:40:36 UTC
svn commit: r1582003 - in
/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/
Author: jing9
Date: Wed Mar 26 18:40:35 2014
New Revision: 1582003
URL: http://svn.apache.org/r1582003
Log:
HDFS-5672. Merge r1581996 from branch-2.
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1582003&r1=1582002&r2=1582003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Mar 26 18:40:35 2014
@@ -437,6 +437,8 @@ Release 2.4.0 - UNRELEASED
HDFS-6115. Call flush() for every append on block scan verification log.
(Vinayakumar B via szetszwo)
+ HDFS-5672. TestHASafeMode#testSafeBlockTracking fails in trunk. (jing9)
+
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1582003&r1=1582002&r2=1582003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Wed Mar 26 18:40:35 2014
@@ -1877,8 +1877,9 @@ public class BlockManager {
int headIndex = 0; //currently the delimiter is in the head of the list
int curIndex;
- if (newReport == null)
+ if (newReport == null) {
newReport = new BlockListAsLongs();
+ }
// scan the report and process newly reported blocks
BlockReportIterator itBR = newReport.getBlockReportIterator();
while(itBR.hasNext()) {
@@ -1971,9 +1972,11 @@ public class BlockManager {
// Ignore replicas already scheduled to be removed from the DN
if(invalidateBlocks.contains(dn.getDatanodeUuid(), block)) {
-/* TODO: following assertion is incorrect, see HDFS-2668
-assert storedBlock.findDatanode(dn) < 0 : "Block " + block
- + " in recentInvalidatesSet should not appear in DN " + dn; */
+ /*
+ * TODO: following assertion is incorrect, see HDFS-2668 assert
+ * storedBlock.findDatanode(dn) < 0 : "Block " + block +
+ * " in recentInvalidatesSet should not appear in DN " + dn;
+ */
return storedBlock;
}
@@ -1993,8 +1996,8 @@ assert storedBlock.findDatanode(dn) < 0
}
if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
- toUC.add(new StatefulBlockInfo(
- (BlockInfoUnderConstruction)storedBlock, block, reportedState));
+ toUC.add(new StatefulBlockInfo((BlockInfoUnderConstruction) storedBlock,
+ new Block(block), reportedState));
return storedBlock;
}
@@ -2878,7 +2881,7 @@ assert storedBlock.findDatanode(dn) < 0
// about new storages from heartbeats but during NN restart we may
// receive a block report or incremental report before the heartbeat.
// We must handle this for protocol compatibility. This issue was
- // uncovered by HDFS-6904.
+ // uncovered by HDFS-6094.
node.updateStorage(srdb.getStorage());
}
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java?rev=1582003&r1=1582002&r2=1582003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java Wed Mar 26 18:40:35 2014
@@ -592,7 +592,16 @@ public class TestHASafeMode {
// below 0.
assertSafeMode(nn1, 0, 0, 3, 0);
}
-
+
+ @Test
+ public void testSafeBlockTracking() throws Exception {
+ testSafeBlockTracking(false);
+ }
+
+ @Test
+ public void testSafeBlockTracking2() throws Exception {
+ testSafeBlockTracking(true);
+ }
/**
* Test that the number of safe blocks is accounted correctly even when
@@ -600,9 +609,15 @@ public class TestHASafeMode {
* If a FINALIZED report arrives at the SBN before the block is marked
* COMPLETE, then when we get the OP_CLOSE we need to count it as "safe"
* at that point. This is a regression test for HDFS-2742.
+ *
+ * @param noFirstBlockReport If this is set to true, we shutdown NN1 before
+ * closing the writing streams. In this way, when NN1 restarts, all DNs will
+ * first send it incremental block report before the first full block report.
+ * And NN1 will not treat the full block report as the first block report
+ * in BlockManager#processReport.
*/
- @Test
- public void testSafeBlockTracking() throws Exception {
+ private void testSafeBlockTracking(boolean noFirstBlockReport)
+ throws Exception {
banner("Starting with NN0 active and NN1 standby, creating some " +
"UC blocks plus some other blocks to force safemode");
DFSTestUtil.createFile(fs, new Path("/other-blocks"), 10*BLOCK_SIZE, (short) 3, 1L);
@@ -619,6 +634,9 @@ public class TestHASafeMode {
// the namespace during startup and enter safemode.
nn0.getRpcServer().rollEditLog();
} finally {
+ if (noFirstBlockReport) {
+ cluster.shutdownNameNode(1);
+ }
for (FSDataOutputStream stm : stms) {
IOUtils.closeStream(stm);
}