You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by at...@apache.org on 2012/08/29 00:12:11 UTC
svn commit: r1378365 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/server/namenode/
Author: atm
Date: Tue Aug 28 22:12:10 2012
New Revision: 1378365
URL: http://svn.apache.org/viewvc?rev=1378365&view=rev
Log:
HDFS-3849. When re-loading the FSImage, we should clear the existing genStamp and leases. Contributed by Colin Patrick McCabe.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1378365&r1=1378364&r2=1378365&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Aug 28 22:12:10 2012
@@ -519,6 +519,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3860. HeartbeatManager#Monitor may wrongly hold the writelock of
namesystem. (Jing Zhao via atm)
+ HDFS-3849. When re-loading the FSImage, we should clear the existing
+ genStamp and leases. (Colin Patrick McCabe via atm)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1378365&r1=1378364&r2=1378365&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Tue Aug 28 22:12:10 2012
@@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
+import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
@@ -535,9 +536,7 @@ public class FSImage implements Closeabl
* file.
*/
void reloadFromImageFile(File file, FSNamesystem target) throws IOException {
- target.dir.reset();
- target.dtSecretManager.reset();
-
+ target.clear();
LOG.debug("Reloading namespace from " + file);
loadFSImage(file, target, null);
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1378365&r1=1378364&r2=1378365&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Aug 28 22:12:10 2012
@@ -351,6 +351,23 @@ public class FSNamesystem implements Nam
private final boolean haEnabled;
/**
+ * Clear all loaded data
+ */
+ void clear() {
+ dir.reset();
+ dtSecretManager.reset();
+ generationStamp.setStamp(GenerationStamp.FIRST_VALID_STAMP);
+ leaseManager.removeAllLeases();
+ }
+
+ @VisibleForTesting
+ LeaseManager getLeaseManager() {
+ return leaseManager;
+ }
+
+ /**
+
+ /**
* Instantiates an FSNamesystem loaded from the image and edits
* directories specified in the passed Configuration.
*
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1378365&r1=1378364&r2=1378365&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Tue Aug 28 22:12:10 2012
@@ -159,6 +159,12 @@ public class LeaseManager {
}
}
+ synchronized void removeAllLeases() {
+ sortedLeases.clear();
+ sortedLeasesByPath.clear();
+ leases.clear();
+ }
+
/**
* Reassign lease for file src to the new holder.
*/
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1378365&r1=1378364&r2=1378365&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Tue Aug 28 22:12:10 2012
@@ -138,6 +138,11 @@ public class SecondaryNameNode implement
FSImage getFSImage() {
return checkpointImage;
}
+
+ @VisibleForTesting
+ FSNamesystem getFSNamesystem() {
+ return namesystem;
+ }
@VisibleForTesting
void setFSImage(CheckpointStorage image) {
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1378365&r1=1378364&r2=1378365&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Tue Aug 28 22:12:10 2012
@@ -1878,6 +1878,59 @@ public class TestCheckpoint {
}
}
}
+
+ /**
+ * Regression test for HDFS-3849. This makes sure that when we re-load the
+ * FSImage in the 2NN, we clear the existing leases.
+ */
+ @Test
+ public void testSecondaryNameNodeWithSavedLeases() throws IOException {
+ MiniDFSCluster cluster = null;
+ SecondaryNameNode secondary = null;
+ FSDataOutputStream fos = null;
+ Configuration conf = new HdfsConfiguration();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes)
+ .format(true).build();
+ FileSystem fs = cluster.getFileSystem();
+ fos = fs.create(new Path("tmpfile"));
+ fos.write(new byte[] { 0, 1, 2, 3 });
+ fos.hflush();
+ assertEquals(1, cluster.getNamesystem().getLeaseManager().countLease());
+
+ secondary = startSecondaryNameNode(conf);
+ assertEquals(0, secondary.getFSNamesystem().getLeaseManager().countLease());
+
+ // Checkpoint once, so the 2NN loads the lease into its in-memory sate.
+ secondary.doCheckpoint();
+ assertEquals(1, secondary.getFSNamesystem().getLeaseManager().countLease());
+ fos.close();
+ fos = null;
+
+ // Perform a saveNamespace, so that the NN has a new fsimage, and the 2NN
+ // therefore needs to download a new fsimage the next time it performs a
+ // checkpoint.
+ cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ cluster.getNameNodeRpc().saveNamespace();
+ cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ // Ensure that the 2NN can still perform a checkpoint.
+ secondary.doCheckpoint();
+
+ // And the leases have been cleared...
+ assertEquals(0, secondary.getFSNamesystem().getLeaseManager().countLease());
+ } finally {
+ if (fos != null) {
+ fos.close();
+ }
+ if (secondary != null) {
+ secondary.shutdown();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
@Test
public void testCommandLineParsing() throws ParseException {
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java?rev=1378365&r1=1378364&r2=1378365&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java Tue Aug 28 22:12:10 2012
@@ -26,6 +26,9 @@ import java.net.URI;
import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.junit.Test;
public class TestFSNamesystem {
@@ -45,4 +48,20 @@ public class TestFSNamesystem {
assertEquals(2, editsDirs.size());
}
+ /**
+ * Test that FSNamesystem#clear clears all leases.
+ */
+ @Test
+ public void testFSNamespaceClearLeases() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
+ DFSTestUtil.formatNameNode(conf);
+ FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
+ LeaseManager leaseMan = fsn.getLeaseManager();
+ leaseMan.addLease("client1", "importantFile");
+ assertEquals(1, leaseMan.countLease());
+ fsn.clear();
+ leaseMan = fsn.getLeaseManager();
+ assertEquals(0, leaseMan.countLease());
+ }
}