You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2013/06/07 22:08:19 UTC
svn commit: r1490806 - in
/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/se...
Author: kihwal
Date: Fri Jun 7 20:08:19 2013
New Revision: 1490806
URL: http://svn.apache.org/r1490806
Log:
HDFS-4832. Namenode doesn't change the number of missing blocks in safemode when DNs rejoin or leave. Contributed by Ravi Prakash.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1490806&r1=1490805&r2=1490806&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Jun 7 20:08:19 2013
@@ -14,6 +14,10 @@ Release 0.23.9 - UNRELEASED
HDFS-4862. SafeModeInfo.isManual() returns true when resources are low even
if it wasn't entered into manually (Ravi Prakash via kihwal)
+
+ HDFS-4832. Namenode doesn't change the number of missing blocks in
+ safemode when DNs rejoin or leave (Ravi Prakash via kihwal)
+
Release 0.23.8 - 2013-06-05
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1490806&r1=1490805&r2=1490806&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Fri Jun 7 20:08:19 2013
@@ -1806,7 +1806,7 @@ public class BlockManager {
return storedBlock;
}
- // do not try to handle over/under-replicated blocks during safe mode
+ // do not try to handle over/under-replicated blocks during first safe mode
if (!namesystem.isPopulatingReplQueues()) {
return storedBlock;
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1490806&r1=1490805&r2=1490806&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Fri Jun 7 20:08:19 2013
@@ -915,7 +915,13 @@ public class DatanodeManager {
heartbeatManager.updateHeartbeat(nodeinfo, capacity, dfsUsed,
remaining, blockPoolUsed, xceiverCount, failedVolumes);
-
+
+ // If we are in safemode, do not send back any recovery / replication
+ // requests. Don't even drain the existing queue of work.
+ if(namesystem.isInSafeMode()) {
+ return new DatanodeCommand[0];
+ }
+
//check lease recovery
BlockInfoUnderConstruction[] blocks = nodeinfo
.getLeaseRecoveryCommand(Integer.MAX_VALUE);
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java?rev=1490806&r1=1490805&r2=1490806&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java Fri Jun 7 20:08:19 2013
@@ -202,7 +202,7 @@ class HeartbeatManager implements Datano
final DatanodeManager dm = blockManager.getDatanodeManager();
// It's OK to check safe mode w/o taking the lock here, we re-check
// for safe mode after taking the lock before removing a datanode.
- if (namesystem.isInSafeMode()) {
+ if (namesystem.isInStartupSafeMode()) {
return;
}
boolean allAlive = false;
@@ -224,7 +224,7 @@ class HeartbeatManager implements Datano
// acquire the fsnamesystem lock, and then remove the dead node.
namesystem.writeLock();
try {
- if (namesystem.isInSafeMode()) {
+ if (namesystem.isInStartupSafeMode()) {
return;
}
synchronized(this) {
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1490806&r1=1490805&r2=1490806&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Jun 7 20:08:19 2013
@@ -2946,7 +2946,7 @@ public class FSNamesystem implements Nam
*
* @see SafeModeInfo
*/
- private SafeModeInfo(boolean resourcesLow) {
+ private SafeModeInfo(boolean resourcesLow, boolean isReplQueuesInited) {
this.threshold = 1.5f; // this threshold can never be reached
this.datanodeThreshold = Integer.MAX_VALUE;
this.extension = Integer.MAX_VALUE;
@@ -2955,6 +2955,7 @@ public class FSNamesystem implements Nam
this.blockTotal = -1;
this.blockSafe = -1;
this.resourcesLow = resourcesLow;
+ this.initializedReplQueues = isReplQueuesInited;
enter();
reportStatus("STATE* Safe mode is ON.", true);
}
@@ -3005,7 +3006,7 @@ public class FSNamesystem implements Nam
}
if(needUpgrade) {
// switch to manual safe mode
- safeMode = new SafeModeInfo(false);
+ safeMode = new SafeModeInfo(false, isPopulatingReplQueues());
return;
}
}
@@ -3356,6 +3357,10 @@ public class FSNamesystem implements Nam
&& safeMode.isOn();
}
+ /**
+ * Check if replication queues are to be populated
+ * @return true when node is HAState.Active and not in the very first safemode
+ */
@Override
public boolean isPopulatingReplQueues() {
// safeMode is volatile, and may be set to null at any time
@@ -3452,7 +3457,7 @@ public class FSNamesystem implements Nam
// is entirely stable on disk as soon as we're in safe mode.
getEditLog().logSyncAll();
if (!isInSafeMode()) {
- safeMode = new SafeModeInfo(resourcesLow);
+ safeMode = new SafeModeInfo(resourcesLow, isPopulatingReplQueues());
return;
}
if (resourcesLow) {
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java?rev=1490806&r1=1490805&r2=1490806&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java Fri Jun 7 20:08:19 2013
@@ -51,4 +51,25 @@ public class TestFSNamesystem {
assertTrue("After entering safemode due to low resources FSNamesystem."
+ "isInSafeMode still returned false", fsn.isInSafeMode());
}
+
+ @Test
+ public void testReplQueuesActiveAfterStartupSafemode() throws IOException, InterruptedException{
+ Configuration conf = new Configuration();
+
+ FSEditLog fsEditLog = Mockito.mock(FSEditLog.class);
+ FSImage fsImage = Mockito.mock(FSImage.class);
+ Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog);
+
+ FSNamesystem fsNamesystem = new FSNamesystem(fsImage, conf);
+ FSNamesystem fsn = Mockito.spy(fsNamesystem);
+
+ fsn.leaveSafeMode(false);
+ assertTrue("FSNamesystem didn't leave safemode", !fsn.isInSafeMode());
+ assertTrue("Replication queues weren't being populated even after leaving "
+ + "safemode", fsn.isPopulatingReplQueues());
+ fsn.enterSafeMode(false);
+ assertTrue("FSNamesystem didn't enter safemode", fsn.isInSafeMode());
+ assertTrue("Replication queues weren't being populated after entering "
+ + "safemode 2nd time", fsn.isPopulatingReplQueues());
+ }
}