You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/05/26 20:55:45 UTC
svn commit: r1128032 - in /hadoop/hdfs/branches/HDFS-1073: ./
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Author: todd
Date: Thu May 26 18:55:44 2011
New Revision: 1128032
URL: http://svn.apache.org/viewvc?rev=1128032&view=rev
Log:
HDFS-1991. Some refactoring of Secondary NameNode to be able to share more code with the BackupNode or CheckpointNode. Contributed by Todd Lipcon.
Modified:
hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1128032&r1=1128031&r2=1128032&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original)
+++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Thu May 26 18:55:44 2011
@@ -36,3 +36,5 @@ HDFS-1987. Re-enable TestCheckpoint.test
running previously. (todd)
HDFS-1993. TestCheckpoint needs to clean up between cases (todd)
HDFS-1992. Remove vestiges of NNStorageListener. (todd)
+HDFS-1991. Some refactoring of Secondary NameNode to be able to share more
+ code with the BackupNode or CheckpointNode. (todd)
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1128032&r1=1128031&r2=1128032&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Thu May 26 18:55:44 2011
@@ -641,10 +641,9 @@ public class FSImage implements Closeabl
// storage directory to read properties from
sdForProperties.read();
File imageFile = loadPlan.getImageFile();
- MD5Hash expectedMD5 = MD5FileUtils.readStoredMd5ForFile(imageFile);
try {
- loadFSImage(imageFile, expectedMD5);
+ loadFSImage(imageFile);
} catch (IOException ioe) {
throw new IOException("Failed to load image from " + loadPlan.getImageFile(), ioe);
}
@@ -692,6 +691,15 @@ public class FSImage implements Closeabl
/**
+ * Load the image namespace from the given image file, verifying
+ * it against the MD5 sum stored in its associated .md5 file.
+ */
+ void loadFSImage(File imageFile) throws IOException {
+ MD5Hash expectedMD5 = MD5FileUtils.readStoredMd5ForFile(imageFile);
+ loadFSImage(imageFile, expectedMD5);
+ }
+
+ /**
* Load in the filesystem image from file. It's a big list of
* filenames and blocks. Return whether we should
* "re-save" and consolidate the edit-logs
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1128032&r1=1128031&r2=1128032&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Thu May 26 18:55:44 2011
@@ -43,11 +43,9 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.Storage.StorageState;
-import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
-import org.apache.hadoop.hdfs.util.MD5FileUtils;
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.ipc.RPC;
@@ -104,7 +102,6 @@ public class SecondaryNameNode implement
private int imagePort;
private String infoBindAddress;
- private FSNamesystem namesystem;
private Collection<URI> checkpointDirs;
private Collection<URI> checkpointEditsDirs;
private long checkpointPeriod; // in seconds
@@ -345,7 +342,7 @@ public class SecondaryNameNode implement
* @return true if a new image has been downloaded and needs to be loaded
* @throws IOException
*/
- private static boolean downloadCheckpointFiles(
+ static boolean downloadCheckpointFiles(
final String nnHostPort,
final FSImage dstImage,
final CheckpointSignature sig,
@@ -455,8 +452,8 @@ public class SecondaryNameNode implement
boolean loadImage = downloadCheckpointFiles(
fsName, checkpointImage, sig, manifest); // Fetch fsimage and edits
- doMerge(sig, manifest, loadImage); // Do the merge
-
+ doMerge(conf, sig, manifest, loadImage, checkpointImage);
+
//
// Upload the new image into the NameNode. Then tell the Namenode
// to make this new uploaded image as the most current image.
@@ -483,21 +480,6 @@ public class SecondaryNameNode implement
}
/**
- * Merge downloaded image and edits and write the new image into
- * current storage directory.
- * @param manifest
- */
- private void doMerge(
- CheckpointSignature sig, RemoteEditLogManifest manifest, boolean loadImage)
- throws IOException {
- if (loadImage) {
- namesystem = new FSNamesystem(checkpointImage, conf);
- }
- assert namesystem.dir.fsImage == checkpointImage;
- checkpointImage.doMerge(sig, manifest, loadImage);
- }
-
- /**
* @param argv The parameters passed to this program.
* @exception Exception if the filesystem does not exist.
* @return 0 on success, non zero on error.
@@ -701,57 +683,42 @@ public class SecondaryNameNode implement
storage.moveLastCheckpoint(sd);
}
}
-
- /**
- * Merge image and edits, and verify consistency with the signature.
- * @param manifest
- */
- private void doMerge(CheckpointSignature sig,
- RemoteEditLogManifest manifest,
- boolean loadImage)
- throws IOException {
- StorageDirectory sdName = null;
- StorageDirectory sdEdits = null;
- Iterator<StorageDirectory> it = null;
- if (loadImage) {
- it = getStorage().dirIterator(NameNodeDirType.IMAGE);
- if (it.hasNext())
- sdName = it.next();
- if (sdName == null) {
- throw new IOException("Could not locate checkpoint fsimage");
- }
- }
- it = getStorage().dirIterator(NameNodeDirType.EDITS);
- if (it.hasNext())
- sdEdits = it.next();
- if (sdEdits == null)
- throw new IOException("Could not locate checkpoint edits");
-
- this.getStorage().setStorageInfo(sig);
- if (loadImage) {
- File file = getStorage().findImageFile(sig.mostRecentCheckpointTxId);
- MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
- LOG.debug("2NN loading image from " + file);
- loadFSImage(file, hash);
- }
- List<File> editsFiles = Lists.newArrayList();
- for (RemoteEditLog log : manifest.getLogs()) {
- File f = getStorage().findFinalizedEditsFile(
- log.getStartTxId(), log.getEndTxId());
- editsFiles.add(f);
- }
- LOG.info("SecondaryNameNode about to load edits from " +
- editsFiles.size() + " file(s).");
- loadEdits(editsFiles);
-
- storage.setClusterID(sig.getClusterID());
- storage.setBlockPoolID(sig.getBlockpoolID());
-
- // TODO I think the below validation might be too strict -- it would disallow
- // multiple concurrent checkpointers with big fsimages. Test this.
- sig.validateStorageInfo(this);
- saveFSImageInAllDirs(editLog.getLastWrittenTxId());
- getStorage().writeAll();
- }
+ }
+
+ static void doMerge(Configuration conf,
+ CheckpointSignature sig, RemoteEditLogManifest manifest,
+ boolean loadImage, FSImage dstImage) throws IOException {
+ NNStorage dstStorage = dstImage.getStorage();
+
+ dstStorage.setStorageInfo(sig);
+ if (loadImage) {
+ // TODO: dstImage.namesystem.close(); ??
+ dstImage.namesystem = new FSNamesystem(dstImage, conf);
+ dstImage.editLog = new FSEditLog(dstStorage);
+
+ File file = dstStorage.findImageFile(sig.mostRecentCheckpointTxId);
+ LOG.debug("2NN loading image from " + file);
+ dstImage.loadFSImage(file);
+ }
+ List<File> editsFiles = Lists.newArrayList();
+ for (RemoteEditLog log : manifest.getLogs()) {
+ File f = dstStorage.findFinalizedEditsFile(
+ log.getStartTxId(), log.getEndTxId());
+ editsFiles.add(f);
+ }
+ LOG.info("SecondaryNameNode about to load edits from " +
+ editsFiles.size() + " file(s).");
+ dstImage.loadEdits(editsFiles);
+
+ // TODO: why do we need the following two lines? We shouldn't have even
+ // been able to download an image from a NN that had a different
+ // cluster ID or blockpool ID! this should only be done for the
+ // very first checkpoint.
+ dstStorage.setClusterID(sig.getClusterID());
+ dstStorage.setBlockPoolID(sig.getBlockpoolID());
+
+ sig.validateStorageInfo(dstImage);
+ dstImage.saveFSImageInAllDirs(dstImage.getEditLog().getLastWrittenTxId());
+ dstStorage.writeAll();
}
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1128032&r1=1128031&r2=1128032&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Thu May 26 18:55:44 2011
@@ -463,7 +463,7 @@ public class TestCheckpoint extends Test
// start new instance of secondary and verify that
// a new rollEditLog suceedes inspite of the fact that
- // edits.new already exists.
+ // edits.new already exists. TODO update this comment!
//
secondary = startSecondaryNameNode(conf);
secondary.doCheckpoint(); // this should work correctly
@@ -988,7 +988,8 @@ public class TestCheckpoint extends Test
File secondaryFsImageAfter = new File(secondaryCurrent,
"fsimage_" + (expectedTxIdToDownload + 2));
- assertFalse("Secondary should start with empty current/ dir",
+ assertFalse("Secondary should start with empty current/ dir " +
+ "but " + secondaryFsImageBefore + " exists",
secondaryFsImageBefore.exists());
assertTrue("Secondary should have loaded an image",