You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2013/01/29 23:44:36 UTC
svn commit: r1440192 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/
Author: kihwal
Date: Tue Jan 29 22:44:35 2013
New Revision: 1440192
URL: http://svn.apache.org/viewvc?rev=1440192&view=rev
Log:
HDFS-4288. NN accepts incremental BR as IBR in safemode. contributed by Daryn Sharp.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1440192&r1=1440191&r2=1440192&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Jan 29 22:44:35 2013
@@ -2233,6 +2233,8 @@ Release 0.23.7 - UNRELEASED
BUG FIXES
+ HDFS-4288. NN accepts incremental BR as IBR in safemode (daryn via kihwal)
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java?rev=1440192&r1=1440191&r2=1440192&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java Tue Jan 29 22:44:35 2013
@@ -88,11 +88,7 @@ public class BlockInfo extends Block imp
DatanodeDescriptor getDatanode(int index) {
assert this.triplets != null : "BlockInfo is not initialized";
assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
- DatanodeDescriptor node = (DatanodeDescriptor)triplets[index*3];
- assert node == null ||
- DatanodeDescriptor.class.getName().equals(node.getClass().getName()) :
- "DatanodeDescriptor is expected at " + index*3;
- return node;
+ return (DatanodeDescriptor)triplets[index*3];
}
private BlockInfo getPrevious(int index) {
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1440192&r1=1440191&r2=1440192&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Tue Jan 29 22:44:35 2013
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.namenode.FSClusterStats;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
+import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
@@ -1576,7 +1577,10 @@ public class BlockManager {
}
// Log the block report processing stats from Namenode perspective
- NameNode.getNameNodeMetrics().addBlockReport((int) (endTime - startTime));
+ final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
+ if (metrics != null) {
+ metrics.addBlockReport((int) (endTime - startTime));
+ }
blockLog.info("BLOCK* processReport: from "
+ nodeID + ", blocks: " + newReport.getNumberOfBlocks()
+ ", processing time: " + (endTime - startTime) + " msecs");
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java?rev=1440192&r1=1440191&r2=1440192&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java Tue Jan 29 22:44:35 2013
@@ -547,6 +547,7 @@ public class DatanodeDescriptor extends
@Override
public void updateRegInfo(DatanodeID nodeReg) {
super.updateRegInfo(nodeReg);
+ firstBlockReport = true; // must re-process IBR after re-registration
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1440192&r1=1440191&r2=1440192&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Tue Jan 29 22:44:35 2013
@@ -419,7 +419,7 @@ public class DatanodeManager {
}
/** Add a datanode. */
- private void addDatanode(final DatanodeDescriptor node) {
+ void addDatanode(final DatanodeDescriptor node) {
// To keep host2DatanodeMap consistent with datanodeMap,
// remove from host2DatanodeMap the datanodeDescriptor removed
// from datanodeMap before adding node to host2DatanodeMap.
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java?rev=1440192&r1=1440191&r2=1440192&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java Tue Jan 29 22:44:35 2013
@@ -34,13 +34,16 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.net.NetworkTopology;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import static org.mockito.Mockito.*;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
@@ -485,4 +488,70 @@ public class TestBlockManager {
new NumberReplicas(),
UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY));
}
+
+ @Test
+ public void testSafeModeIBR() throws Exception {
+ DatanodeDescriptor node = spy(nodes.get(0));
+ node.setStorageID("dummy-storage");
+ node.isAlive = true;
+
+ DatanodeRegistration nodeReg =
+ new DatanodeRegistration(node, null, null, "");
+
+ // pretend to be in safemode
+ doReturn(true).when(fsn).isInStartupSafeMode();
+
+ // register new node
+ bm.getDatanodeManager().registerDatanode(nodeReg);
+ bm.getDatanodeManager().addDatanode(node); // swap in spy
+ assertEquals(node, bm.getDatanodeManager().getDatanode(node));
+ assertTrue(node.isFirstBlockReport());
+ // send block report, should be processed
+ reset(node);
+ bm.processReport(node, "pool", new BlockListAsLongs(null, null));
+ verify(node).receivedBlockReport();
+ assertFalse(node.isFirstBlockReport());
+ // send block report again, should NOT be processed
+ reset(node);
+ bm.processReport(node, "pool", new BlockListAsLongs(null, null));
+ verify(node, never()).receivedBlockReport();
+ assertFalse(node.isFirstBlockReport());
+
+ // re-register as if node restarted, should update existing node
+ bm.getDatanodeManager().removeDatanode(node);
+ reset(node);
+ bm.getDatanodeManager().registerDatanode(nodeReg);
+ verify(node).updateRegInfo(nodeReg);
+ assertTrue(node.isFirstBlockReport()); // ready for report again
+ // send block report, should be processed after restart
+ reset(node);
+ bm.processReport(node, "pool", new BlockListAsLongs(null, null));
+ verify(node).receivedBlockReport();
+ assertFalse(node.isFirstBlockReport());
+ }
+
+ @Test
+ public void testSafeModeIBRAfterIncremental() throws Exception {
+ DatanodeDescriptor node = spy(nodes.get(0));
+ node.setStorageID("dummy-storage");
+ node.isAlive = true;
+
+ DatanodeRegistration nodeReg =
+ new DatanodeRegistration(node, null, null, "");
+
+ // pretend to be in safemode
+ doReturn(true).when(fsn).isInStartupSafeMode();
+
+ // register new node
+ bm.getDatanodeManager().registerDatanode(nodeReg);
+ bm.getDatanodeManager().addDatanode(node); // swap in spy
+ assertEquals(node, bm.getDatanodeManager().getDatanode(node));
+ assertTrue(node.isFirstBlockReport());
+ // send block report while pretending to already have blocks
+ reset(node);
+ doReturn(1).when(node).numBlocks();
+ bm.processReport(node, "pool", new BlockListAsLongs(null, null));
+ verify(node).receivedBlockReport();
+ assertFalse(node.isFirstBlockReport());
+ }
}