You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2013/01/16 01:10:00 UTC
svn commit: r1433756 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/
Author: todd
Date: Wed Jan 16 00:09:59 2013
New Revision: 1433756
URL: http://svn.apache.org/viewvc?rev=1433756&view=rev
Log:
HDFS-4288. NN accepts incremental BR as IBR in safemode. Contributed by Daryn Sharp.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1433756&r1=1433755&r2=1433756&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Jan 16 00:09:59 2013
@@ -417,6 +417,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-1245. Pluggable block id generation. (shv)
+ HDFS-4288. NN accepts incremental BR as IBR in safemode
+ (Daryn Sharp via todd)
+
BREAKDOWN OF HDFS-3077 SUBTASKS
HDFS-3077. Quorum-based protocol for reading and writing edit logs.
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1433756&r1=1433755&r2=1433756&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Wed Jan 16 00:09:59 2013
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.namenode.FSClusterStats;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
+import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
@@ -1576,7 +1577,10 @@ public class BlockManager {
}
// Log the block report processing stats from Namenode perspective
- NameNode.getNameNodeMetrics().addBlockReport((int) (endTime - startTime));
+ final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
+ if (metrics != null) {
+ metrics.addBlockReport((int) (endTime - startTime));
+ }
blockLog.info("BLOCK* processReport: from "
+ nodeID + ", blocks: " + newReport.getNumberOfBlocks()
+ ", processing time: " + (endTime - startTime) + " msecs");
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java?rev=1433756&r1=1433755&r2=1433756&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java Wed Jan 16 00:09:59 2013
@@ -551,6 +551,7 @@ public class DatanodeDescriptor extends
@Override
public void updateRegInfo(DatanodeID nodeReg) {
super.updateRegInfo(nodeReg);
+ firstBlockReport = true; // must re-process IBR after re-registration
}
/**
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1433756&r1=1433755&r2=1433756&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Wed Jan 16 00:09:59 2013
@@ -406,7 +406,7 @@ public class DatanodeManager {
}
/** Add a datanode. */
- private void addDatanode(final DatanodeDescriptor node) {
+ void addDatanode(final DatanodeDescriptor node) {
// To keep host2DatanodeMap consistent with datanodeMap,
// remove from host2DatanodeMap the datanodeDescriptor removed
// from datanodeMap before adding node to host2DatanodeMap.
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java?rev=1433756&r1=1433755&r2=1433756&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java Wed Jan 16 00:09:59 2013
@@ -34,13 +34,16 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.net.NetworkTopology;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import static org.mockito.Mockito.*;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
@@ -485,4 +488,70 @@ public class TestBlockManager {
new NumberReplicas(),
UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY));
}
+
+ @Test
+ public void testSafeModeIBR() throws Exception {
+ DatanodeDescriptor node = spy(nodes.get(0));
+ node.setStorageID("dummy-storage");
+ node.isAlive = true;
+
+ DatanodeRegistration nodeReg =
+ new DatanodeRegistration(node, null, null, "");
+
+ // pretend to be in safemode
+ doReturn(true).when(fsn).isInStartupSafeMode();
+
+ // register new node
+ bm.getDatanodeManager().registerDatanode(nodeReg);
+ bm.getDatanodeManager().addDatanode(node); // swap in spy
+ assertEquals(node, bm.getDatanodeManager().getDatanode(node));
+ assertTrue(node.isFirstBlockReport());
+ // send block report, should be processed
+ reset(node);
+ bm.processReport(node, "pool", new BlockListAsLongs());
+ verify(node).receivedBlockReport();
+ assertFalse(node.isFirstBlockReport());
+ // send block report again, should NOT be processed
+ reset(node);
+ bm.processReport(node, "pool", new BlockListAsLongs());
+ verify(node, never()).receivedBlockReport();
+ assertFalse(node.isFirstBlockReport());
+
+ // re-register as if node restarted, should update existing node
+ bm.getDatanodeManager().removeDatanode(node);
+ reset(node);
+ bm.getDatanodeManager().registerDatanode(nodeReg);
+ verify(node).updateRegInfo(nodeReg);
+ assertTrue(node.isFirstBlockReport()); // ready for report again
+ // send block report, should be processed after restart
+ reset(node);
+ bm.processReport(node, "pool", new BlockListAsLongs());
+ verify(node).receivedBlockReport();
+ assertFalse(node.isFirstBlockReport());
+ }
+
+ @Test
+ public void testSafeModeIBRAfterIncremental() throws Exception {
+ DatanodeDescriptor node = spy(nodes.get(0));
+ node.setStorageID("dummy-storage");
+ node.isAlive = true;
+
+ DatanodeRegistration nodeReg =
+ new DatanodeRegistration(node, null, null, "");
+
+ // pretend to be in safemode
+ doReturn(true).when(fsn).isInStartupSafeMode();
+
+ // register new node
+ bm.getDatanodeManager().registerDatanode(nodeReg);
+ bm.getDatanodeManager().addDatanode(node); // swap in spy
+ assertEquals(node, bm.getDatanodeManager().getDatanode(node));
+ assertTrue(node.isFirstBlockReport());
+ // send block report while pretending to already have blocks
+ reset(node);
+ doReturn(1).when(node).numBlocks();
+ bm.processReport(node, "pool", new BlockListAsLongs());
+ verify(node).receivedBlockReport();
+ assertFalse(node.isFirstBlockReport());
+ }
}