svn commit: r1104685 - in /hadoop/hdfs/branches/branch-0.22: CHANGES.txt src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java

[to...@apache.org]

Author: todd
Date: Wed May 18 04:19:59 2011
New Revision: 1104685

URL: http://svn.apache.org/viewvc?rev=1104685&view=rev
Log:
HDFS-1505. saveNamespace appears to succeed even if all directories fail to save. Contributed by Aaron T. Myers.

Modified:
    hadoop/hdfs/branches/branch-0.22/CHANGES.txt
    hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java

Modified: hadoop/hdfs/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/CHANGES.txt?rev=1104685&r1=1104684&r2=1104685&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.22/CHANGES.txt Wed May 18 04:19:59 2011
@@ -516,6 +516,9 @@ Release 0.22.0 - Unreleased
     HDFS-1891. Disable IPV6 for junit tests to fix TestBackupNode failure.
     (suresh)
 
+    HDFS-1505. saveNamespace appears to succeed even if all directories fail
+    to save. (Aaron T. Myers via todd)
+
 Release 0.21.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1104685&r1=1104684&r2=1104685&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Wed May 18 04:19:59 2011
@@ -1239,9 +1239,21 @@ public class FSImage extends Storage {
         errorSDs.add(sd);
       }
     }
-    processIOError(errorSDs, false);
-    if(!editLog.isOpen()) editLog.open();
-    ckptState = CheckpointStates.UPLOAD_DONE;
+    
+    try {
+      processIOError(errorSDs, false);
+
+      // If there was an error in every storage dir, each one will have been
+      // removed from the list of storage directories.
+      if (getNumStorageDirs(NameNodeDirType.IMAGE) == 0 ||
+          getNumStorageDirs(NameNodeDirType.EDITS) == 0) {
+        throw new IOException("Failed to save any storage directories while saving namespace");
+      }
+
+      if(!editLog.isOpen()) editLog.open();
+    } finally {
+      ckptState = CheckpointStates.UPLOAD_DONE;
+    }
   }
 
   /**

Modified: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1104685&r1=1104684&r2=1104685&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java Wed May 18 04:19:59 2011
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.na
 
 import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
@@ -40,6 +41,7 @@ import org.apache.hadoop.hdfs.protocol.F
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.junit.Test;
+import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
@@ -157,6 +159,62 @@ public class TestSaveNamespace {
   }
 
   @Test
+  public void testFailedSaveNamespace() throws Exception {
+    Configuration conf = getConf();
+    NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
+    NameNode.format(conf);
+    FSNamesystem fsn = new FSNamesystem(conf);
+
+    // Replace the FSImage with a spy
+    final FSImage originalImage = fsn.dir.fsImage;
+    originalImage.unlockAll();
+    FSImage spyImage = spy(originalImage);
+    fsn.dir.fsImage = spyImage;
+    spyImage.setStorageDirectories(
+        FSNamesystem.getNamespaceDirs(conf),
+        FSNamesystem.getNamespaceEditsDirs(conf));
+
+    doThrow(new IOException("Injected fault: saveFSImage")).
+        when(spyImage).saveFSImage((File)anyObject());
+
+    try {
+      doAnEdit(fsn, 1);
+
+      // Save namespace
+      fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+      try {
+        fsn.saveNamespace();
+        fail("saveNamespace did not fail even when all directories failed!");
+      } catch (IOException ioe) {
+        LOG.info("Got expected exception", ioe);
+      }
+
+      // Ensure that, if storage dirs come back online, things work again.
+      Mockito.reset(spyImage);
+      spyImage.setRestoreFailedStorage(true);
+      spyImage.attemptRestoreRemovedStorage();
+      fsn.saveNamespace();
+      checkEditExists(fsn, 1);
+
+      // Now shut down and restart the NN
+      originalImage.close();
+      fsn.close();
+      fsn = null;
+
+      // Start a new namesystem, which should be able to recover
+      // the namespace from the previous incarnation.
+      fsn = new FSNamesystem(conf);
+
+      // Make sure the image loaded including our edits.
+      checkEditExists(fsn, 1);
+    } finally {
+      if (fsn != null) {
+        fsn.close();
+      }
+    }
+  }
+
+  @Test
   public void testSaveWhileEditsRolled() throws Exception {
     Configuration conf = getConf();
     NameNode.initMetrics(conf, NamenodeRole.ACTIVE);