You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/06/08 00:30:26 UTC
svn commit: r1133183 - in /hadoop/hdfs/branches/HDFS-1073: ./ src/java/
src/java/org/apache/hadoop/hdfs/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Author: todd
Date: Tue Jun 7 22:30:25 2011
New Revision: 1133183
URL: http://svn.apache.org/viewvc?rev=1133183&view=rev
Log:
HDFS-2016. Add infrastructure to remove or archive old and unneeded storage files within the name directories. Contributed by Todd Lipcon.
Added:
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java
hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java
Modified:
hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java
Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original)
+++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Tue Jun 7 22:30:25 2011
@@ -43,3 +43,5 @@ HDFS-1994. Fix race conditions when runn
HDFS-2001. Remove use of previous.checkpoint and lastcheckpoint.tmp directories
(todd)
HDFS-2015. Remove checkpointTxId from VERSION file. (todd)
+HDFS-2016. Add infrastructure to remove or archive old and unneeded storage
+ files within the name directories. (todd)
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml Tue Jun 7 22:30:25 2011
@@ -590,6 +590,16 @@ creations/deletions), or "all".</descrip
</property>
<property>
+ <name>dfs.namenode.num.checkpoints.retained</name>
+ <value>2</value>
+ <description>The number of image checkpoint files that will be retained by
+ the NameNode and Secondary NameNode in their storage directories. All edit
+ logs necessary to recover an up-to-date namespace from the oldest retained
+ checkpoint will also be retained.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.delegation.key.update-interval</name>
<value>86400000</value>
<description>The update interval for master key for delegation tokens
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Jun 7 22:30:25 2011
@@ -106,6 +106,9 @@ public class DFSConfigKeys extends Commo
public static final boolean DFS_NAMENODE_NAME_DIR_RESTORE_DEFAULT = false;
public static final String DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY = "dfs.namenode.support.allow.format";
public static final boolean DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT = true;
+ public static final String DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY = "dfs.namenode.num.checkpoints.retained";
+ public static final int DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT = 2;
+
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
public static final String DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY = "dfs.datanode.failed.volumes.tolerated";
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Tue Jun 7 22:30:25 2011
@@ -567,51 +567,6 @@ public class FSImage implements Closeabl
storage.writeTransactionIdFileToStorage(editLog.getCurSegmentTxId());
};
- private FSImageStorageInspector inspectStorageDirs() throws IOException {
- int minLayoutVersion = Integer.MAX_VALUE; // the newest
- int maxLayoutVersion = Integer.MIN_VALUE; // the oldest
-
- // First determine what range of layout versions we're going to inspect
- for (Iterator<StorageDirectory> it = storage.dirIterator();
- it.hasNext();) {
- StorageDirectory sd = it.next();
- if (!sd.getVersionFile().exists()) {
- LOG.warn("Storage directory " + sd + " contains no VERSION file. Skipping...");
- continue;
- }
- sd.read(); // sets layoutVersion
- minLayoutVersion = Math.min(minLayoutVersion, storage.getLayoutVersion());
- maxLayoutVersion = Math.max(maxLayoutVersion, storage.getLayoutVersion());
- }
-
- if (minLayoutVersion > maxLayoutVersion) {
- throw new IOException("No storage directories contained VERSION information");
- }
- assert minLayoutVersion <= maxLayoutVersion;
-
- // If we have any storage directories with the new layout version
- // (ie edits_<txnid>) then use the new inspector, which will ignore
- // the old format dirs.
- FSImageStorageInspector inspector;
- if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, minLayoutVersion)) {
- inspector = new FSImageTransactionalStorageInspector();
- if (!LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, maxLayoutVersion)) {
- LOG.warn("Ignoring one or more storage directories with old layouts");
- }
- } else {
- inspector = new FSImageOldStorageInspector();
- }
-
- // Process each of the storage directories to find the pair of
- // newest image file and edit file
- for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
- StorageDirectory sd = it.next();
- inspector.inspectDirectory(sd);
- }
-
- return inspector;
- }
-
/**
* Choose latest image from one of the directories,
* load it and merge with the edits from that directory.
@@ -628,7 +583,7 @@ public class FSImage implements Closeabl
* @throws IOException
*/
boolean loadFSImage() throws IOException {
- FSImageStorageInspector inspector = inspectStorageDirs();
+ FSImageStorageInspector inspector = storage.readAndInspectDirs();
isUpgradeFinalized = inspector.isUpgradeFinalized();
@@ -854,6 +809,10 @@ public class FSImage implements Closeabl
// TODO Double-check for regressions against HDFS-1505 and HDFS-1921.
renameCheckpoint(txid);
+
+ // Since we now have a new checkpoint, we can clean up some
+ // old edit logs and checkpoints.
+ storage.archiveOldStorage();
}
/**
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java Tue Jun 7 22:30:25 2011
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
class FSImageTransactionalStorageInspector extends FSImageStorageInspector {
@@ -180,6 +181,14 @@ class FSImageTransactionalStorageInspect
}
return ret;
}
+
+ public List<FoundFSImage> getFoundImages() {
+ return ImmutableList.copyOf(foundImages);
+ }
+
+ public List<FoundEditLog> getFoundEditLogs() {
+ return ImmutableList.copyOf(foundEditLogs);
+ }
@Override
public LoadPlan createLoadPlan() throws IOException {
@@ -442,6 +451,11 @@ class FSImageTransactionalStorageInspect
public long getTxId() {
return txId;
}
+
+ @Override
+ public String toString() {
+ return file.toString();
+ }
}
/**
@@ -513,6 +527,11 @@ class FSImageTransactionalStorageInspect
}
file = dst;
}
+
+ @Override
+ public String toString() {
+ return file.toString();
+ }
}
static class TransactionalLoadPlan extends LoadPlan {
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java Tue Jun 7 22:30:25 2011
@@ -130,6 +130,10 @@ public class GetImageServlet extends Htt
}
});
nnImage.saveDigestAndRenameCheckpointImage(txid, downloadImageDigest);
+
+ // Now that we have a new checkpoint, we might be able to
+ // remove some old ones.
+ nnImage.getStorage().archiveOldStorage();
} finally {
currentlyDownloadingCheckpoints.remove(txid);
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Tue Jun 7 22:30:25 2011
@@ -121,6 +121,8 @@ public class NNStorage extends Storage i
private UpgradeManager upgradeManager = null;
protected String blockpoolID = ""; // id of the block pool
+
+ private final NNStorageArchivalManager archivalManager;
/**
* flag that controls if we try to restore failed storages
@@ -158,18 +160,8 @@ public class NNStorage extends Storage i
storageDirs = new CopyOnWriteArrayList<StorageDirectory>();
setStorageDirectories(imageDirs, editsDirs);
- }
-
- /**
- * Construct the NNStorage.
- * @param storageInfo storage information
- * @param bpid block pool Id
- */
- public NNStorage(StorageInfo storageInfo, String bpid) {
- super(NodeType.NAME_NODE, storageInfo);
-
- storageDirs = new CopyOnWriteArrayList<StorageDirectory>();
- this.blockpoolID = bpid;
+
+ archivalManager = new NNStorageArchivalManager(conf, this);
}
@Override // Storage
@@ -542,6 +534,19 @@ public class NNStorage extends Storage i
}
/**
+ * Archive any files in the storage directories that are no longer
+ * necessary.
+ */
+ public void archiveOldStorage() {
+ try {
+ archivalManager.archiveOldStorage();
+ } catch (Exception e) {
+ LOG.warn("Unable to archive old storage", e);
+ }
+ }
+
+
+ /**
* Generate new namespaceID.
*
* namespaceID is a persistent attribute of the namespace.
@@ -942,4 +947,67 @@ public class NNStorage extends Storage i
public String getBlockPoolID() {
return blockpoolID;
}
+
+ /**
+ * Iterate over all current storage directories, inspecting them
+ * with the given inspector.
+ */
+ void inspectStorageDirs(FSImageStorageInspector inspector)
+ throws IOException {
+
+ // Process each of the storage directories to find the pair of
+ // newest image file and edit file
+ for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) {
+ StorageDirectory sd = it.next();
+ inspector.inspectDirectory(sd);
+ }
+ }
+
+ /**
+ * Iterate over all of the storage dirs, reading their contents to determine
+ * their layout versions. Returns an FSImageStorageInspector which has
+ * inspected each directory.
+ *
+ * <b>Note:</b> this can mutate the storage info fields (ctime, version, etc).
+ * @throws IOException if no valid storage dirs are found
+ */
+ FSImageStorageInspector readAndInspectDirs()
+ throws IOException {
+ int minLayoutVersion = Integer.MAX_VALUE; // the newest
+ int maxLayoutVersion = Integer.MIN_VALUE; // the oldest
+
+ // First determine what range of layout versions we're going to inspect
+ for (Iterator<StorageDirectory> it = dirIterator();
+ it.hasNext();) {
+ StorageDirectory sd = it.next();
+ if (!sd.getVersionFile().exists()) {
+ FSImage.LOG.warn("Storage directory " + sd + " contains no VERSION file. Skipping...");
+ continue;
+ }
+ sd.read(); // sets layoutVersion
+ minLayoutVersion = Math.min(minLayoutVersion, getLayoutVersion());
+ maxLayoutVersion = Math.max(maxLayoutVersion, getLayoutVersion());
+ }
+
+ if (minLayoutVersion > maxLayoutVersion) {
+ throw new IOException("No storage directories contained VERSION information");
+ }
+ assert minLayoutVersion <= maxLayoutVersion;
+
+ // If we have any storage directories with the new layout version
+ // (ie edits_<txnid>) then use the new inspector, which will ignore
+ // the old format dirs.
+ FSImageStorageInspector inspector;
+ if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, minLayoutVersion)) {
+ inspector = new FSImageTransactionalStorageInspector();
+ if (!LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, maxLayoutVersion)) {
+ FSImage.LOG.warn("Ignoring one or more storage directories with old layouts");
+ }
+ } else {
+ inspector = new FSImageOldStorageInspector();
+ }
+
+ inspectStorageDirs(inspector);
+ return inspector;
+ }
}
Added: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java?rev=1133183&view=auto
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java (added)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java Tue Jun 7 22:30:25 2011
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage;
+import org.apache.hadoop.hdfs.util.MD5FileUtils;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+/**
+ * The NNStorageArchivalManager is responsible for inspecting the storage
+ * directories of the NN and enforcing a retention policy on checkpoints
+ * and edit logs.
+ *
+ * It delegates the actual removal of files to a {@link #StorageArchiver}
+ * implementation, which might delete the files or instead copy them to
+ * a filer or HDFS for later analysis.
+ */
+public class NNStorageArchivalManager {
+
+ private final int numCheckpointsToRetain;
+ private static final Log LOG = LogFactory.getLog(NNStorageArchivalManager.class);
+ private final NNStorage storage;
+ private final StorageArchiver archiver;
+
+ public NNStorageArchivalManager(
+ Configuration conf,
+ NNStorage storage,
+ StorageArchiver archiver) {
+ this.numCheckpointsToRetain = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY,
+ DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT);
+ this.storage = storage;
+ this.archiver = archiver;
+ }
+
+ public NNStorageArchivalManager(Configuration conf, NNStorage storage) {
+ this(conf, storage, new DeletionStorageArchiver());
+ }
+
+ public void archiveOldStorage() throws IOException {
+ FSImageTransactionalStorageInspector inspector =
+ new FSImageTransactionalStorageInspector();
+ storage.inspectStorageDirs(inspector);
+
+ long minImageTxId = getImageTxIdToRetain(inspector);
+ archiveCheckpointsOlderThan(inspector, minImageTxId);
+ archiveLogsOlderThan(inspector, minImageTxId);
+ }
+
+ private void archiveLogsOlderThan(
+ FSImageTransactionalStorageInspector inspector,
+ long minImageTxId) {
+ for (FoundEditLog log : inspector.getFoundEditLogs()) {
+ if (log.getStartTxId() < minImageTxId) {
+ LOG.info("Purging old edit log " + log);
+ archiver.archiveLog(log);
+ }
+ }
+ }
+
+ private void archiveCheckpointsOlderThan(
+ FSImageTransactionalStorageInspector inspector,
+ long minTxId) {
+ for (FoundFSImage image : inspector.getFoundImages()) {
+ if (image.getTxId() < minTxId) {
+ LOG.info("Purging old image " + image);
+ archiver.archiveImage(image);
+ }
+ }
+ }
+
+ /**
+ * @param inspector inspector that has already inspected all storage dirs
+ * @return the transaction ID corresponding to the oldest checkpoint
+ * that should be retained.
+ */
+ private long getImageTxIdToRetain(FSImageTransactionalStorageInspector inspector) {
+
+ List<FoundFSImage> images = inspector.getFoundImages();
+ TreeSet<Long> imageTxIds = Sets.newTreeSet();
+ for (FoundFSImage image : images) {
+ imageTxIds.add(image.getTxId());
+ }
+
+ List<Long> imageTxIdsList = Lists.newArrayList(imageTxIds);
+ if (imageTxIdsList.isEmpty()) {
+ return 0;
+ }
+
+ Collections.reverse(imageTxIdsList);
+ int toRetain = Math.min(numCheckpointsToRetain, imageTxIdsList.size());
+ long minTxId = imageTxIdsList.get(toRetain - 1);
+ LOG.info("Going to retain " + toRetain + " images with txid >= " +
+ minTxId);
+ return minTxId;
+ }
+
+ /**
+ * Interface responsible for archiving old checkpoints and edit logs.
+ */
+ static interface StorageArchiver {
+ void archiveLog(FoundEditLog log);
+ void archiveImage(FoundFSImage image);
+ }
+
+ static class DeletionStorageArchiver implements StorageArchiver {
+ @Override
+ public void archiveLog(FoundEditLog log) {
+ log.getFile().delete();
+ }
+
+ @Override
+ public void archiveImage(FoundFSImage image) {
+ image.getFile().delete();
+ MD5FileUtils.getDigestFileForFile(image.getFile()).delete();
+ }
+ }
+}
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Tue Jun 7 22:30:25 2011
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.Storage.StorageState;
+import org.apache.hadoop.hdfs.server.namenode.NNStorageArchivalManager.StorageArchiver;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
@@ -492,6 +493,10 @@ public class SecondaryNameNode implement
LOG.warn("Checkpoint done. New Image Size: "
+ checkpointImage.getStorage().getFsImageName(txid).length());
+ // Since we've successfully checkpointed, we can remove some old
+ // image files
+ checkpointImage.getStorage().archiveOldStorage();
+
return loadImage;
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Tue Jun 7 22:30:25 2011
@@ -1117,6 +1117,9 @@ public class TestCheckpoint extends Test
// a checkpoint with a lower txid finished most recently)
NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
assertEquals(4, storage.getMostRecentCheckpointTxId());
+
+ // Should have accepted both checkpoints
+ assertNNHasCheckpoints(cluster, ImmutableList.of(2,4));
// Now have second one checkpoint one more time just to make sure that
// the NN isn't left in a broken state
@@ -1134,9 +1137,10 @@ public class TestCheckpoint extends Test
// Validate invariant that files named the same are the same.
assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2));
- // Validate that the NN received checkpoints at expected txids
- // (i.e that both checkpoints went through)
- assertNNHasCheckpoints(cluster, ImmutableList.of(2,4,6));
+
+ // NN should have removed the checkpoint at txid 2 at this point, but has
+ // one at txid 6
+ assertNNHasCheckpoints(cluster, ImmutableList.of(4,6));
}
@@ -1226,6 +1230,9 @@ public class TestCheckpoint extends Test
secondary2.doCheckpoint();
assertEquals(6, storage.getMostRecentCheckpointTxId());
+ // Should have accepted both checkpoints
+ assertNNHasCheckpoints(cluster, ImmutableList.of(4,6));
+
// Let the first one also go again on its own to make sure it can
// continue at next checkpoint
secondary1.setNameNode(origNN);
@@ -1245,7 +1252,7 @@ public class TestCheckpoint extends Test
assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2));
// Validate that the NN received checkpoints at expected txids
// (i.e that both checkpoints went through)
- assertNNHasCheckpoints(cluster, ImmutableList.of(4,6,8));
+ assertNNHasCheckpoints(cluster, ImmutableList.of(6,8));
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java Tue Jun 7 22:30:25 2011
@@ -358,7 +358,7 @@ public class TestFSImageStorageInspector
* @param previousExists should we mock that the previous/ dir exists?
* @param fileNames the names of files contained in current/
*/
- private StorageDirectory mockDirectory(
+ static StorageDirectory mockDirectory(
StorageDirType type,
boolean previousExists,
String... fileNames) {
Added: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java?rev=1133183&view=auto
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java (added)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java Tue Jun 7 22:30:25 2011
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
+import org.apache.hadoop.hdfs.server.namenode.NNStorageArchivalManager.StorageArchiver;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Sets;
+
+
+public class TestNNStorageArchivalManager {
+ /**
+ * Test the "easy case" where we have more images in the
+ * directory than we need to keep. Should archive the
+ * old ones.
+ */
+ @Test
+ public void testArchiveEasyCase() throws IOException {
+ TestCaseDescription tc = new TestCaseDescription();
+ tc.addImage("/foo1/current/fsimage_100", true);
+ tc.addImage("/foo1/current/fsimage_200", true);
+ tc.addImage("/foo1/current/fsimage_300", false);
+ tc.addImage("/foo1/current/fsimage_400", false);
+ tc.addLog("/foo1/current/edits_101-200", true);
+ tc.addLog("/foo1/current/edits_201-300", true);
+ tc.addLog("/foo1/current/edits_301-400", false);
+ tc.addLog("/foo1/current/edits_inprogress_401", false);
+
+ // Test that other files don't get archived
+ tc.addLog("/foo1/current/VERSION", false);
+ runTest(tc);
+ }
+
+ /**
+ * Same as above, but across multiple directories
+ */
+ @Test
+ public void testArchiveMultipleDirs() throws IOException {
+ TestCaseDescription tc = new TestCaseDescription();
+ tc.addImage("/foo1/current/fsimage_100", true);
+ tc.addImage("/foo1/current/fsimage_200", true);
+ tc.addImage("/foo2/current/fsimage_200", true);
+ tc.addImage("/foo1/current/fsimage_300", false);
+ tc.addImage("/foo1/current/fsimage_400", false);
+ tc.addLog("/foo1/current/edits_101-200", true);
+ tc.addLog("/foo1/current/edits_201-300", true);
+ tc.addLog("/foo2/current/edits_201-300", true);
+ tc.addLog("/foo1/current/edits_301-400", false);
+ tc.addLog("/foo2/current/edits_301-400", false);
+ tc.addLog("/foo1/current/edits_inprogress_401", false);
+ runTest(tc);
+ }
+
+ /**
+ * Test that if we have fewer fsimages than the configured
+ * retention, we don't archive any of them
+ */
+ @Test
+ public void testArchiveLessThanRetention() throws IOException {
+ TestCaseDescription tc = new TestCaseDescription();
+ tc.addImage("/foo1/current/fsimage_100", false);
+ tc.addLog("/foo1/current/edits_101-200", false);
+ tc.addLog("/foo1/current/edits_201-300", false);
+ tc.addLog("/foo1/current/edits_301-400", false);
+ tc.addLog("/foo1/current/edits_inprogress_401", false);
+ runTest(tc);
+ }
+
+ /**
+ * Check for edge case with no logs present at all.
+ */
+ @Test
+ public void testNoLogs() throws IOException {
+ TestCaseDescription tc = new TestCaseDescription();
+ tc.addImage("/foo1/current/fsimage_100", true);
+ tc.addImage("/foo1/current/fsimage_200", true);
+ tc.addImage("/foo1/current/fsimage_300", false);
+ tc.addImage("/foo1/current/fsimage_400", false);
+ runTest(tc);
+ }
+
+ /**
+ * Check for edge case with no logs or images present at all.
+ */
+ @Test
+ public void testEmptyDir() throws IOException {
+ TestCaseDescription tc = new TestCaseDescription();
+ runTest(tc);
+ }
+
+ /**
+ * Test that old in-progress logs are properly archived
+ */
+ @Test
+ public void testOldInProgress() throws IOException {
+ TestCaseDescription tc = new TestCaseDescription();
+ tc.addImage("/foo1/current/fsimage_100", true);
+ tc.addImage("/foo1/current/fsimage_200", true);
+ tc.addImage("/foo1/current/fsimage_300", false);
+ tc.addImage("/foo1/current/fsimage_400", false);
+ tc.addLog("/foo1/current/edits_inprogress_101", true);
+ runTest(tc);
+ }
+
+ private void runTest(TestCaseDescription tc) throws IOException {
+ Configuration conf = new Configuration();
+
+ StorageArchiver mockArchiver =
+ Mockito.mock(NNStorageArchivalManager.StorageArchiver.class);
+ ArgumentCaptor<FoundFSImage> imagesArchivedCaptor =
+ ArgumentCaptor.forClass(FoundFSImage.class);
+ ArgumentCaptor<FoundEditLog> logsArchivedCaptor =
+ ArgumentCaptor.forClass(FoundEditLog.class);
+
+ // Ask the manager to archive files we don't need any more
+ new NNStorageArchivalManager(conf, tc.mockStorage(), mockArchiver)
+ .archiveOldStorage();
+
+ // Verify that it asked the archiver to remove the correct files
+ Mockito.verify(mockArchiver, Mockito.atLeast(0))
+ .archiveImage(imagesArchivedCaptor.capture());
+ Mockito.verify(mockArchiver, Mockito.atLeast(0))
+ .archiveLog(logsArchivedCaptor.capture());
+
+ // Check images
+ Set<String> archivedPaths = Sets.newHashSet();
+ for (FoundFSImage archived : imagesArchivedCaptor.getAllValues()) {
+ archivedPaths.add(archived.getFile().toString());
+ }
+ Assert.assertEquals(Joiner.on(",").join(tc.expectedArchivedImages),
+ Joiner.on(",").join(archivedPaths));
+
+ // Check images
+ archivedPaths.clear();
+ for (FoundEditLog archived : logsArchivedCaptor.getAllValues()) {
+ archivedPaths.add(archived.getFile().toString());
+ }
+ Assert.assertEquals(Joiner.on(",").join(tc.expectedArchivedLogs),
+ Joiner.on(",").join(archivedPaths));
+ }
+
+ private static class TestCaseDescription {
+ private Set<String> files = Sets.newHashSet();
+ private Set<String> expectedArchivedLogs = Sets.newHashSet();
+ private Set<String> expectedArchivedImages = Sets.newHashSet();
+
+ void addLog(String path, boolean expectArchive) {
+ files.add(path);
+ if (expectArchive) {
+ expectedArchivedLogs.add(path);
+ }
+ }
+
+ private String[] getPaths() {
+ return files.toArray(new String[0]);
+ }
+
+ void addImage(String path, boolean expectArchive) {
+ files.add(path);
+ if (expectArchive) {
+ expectedArchivedImages.add(path);
+ }
+ }
+
+ NNStorage mockStorage() throws IOException {
+ String[] paths = getPaths();
+ StorageDirectory mockDir = TestFSImageStorageInspector.mockDirectory(
+ NameNodeDirType.IMAGE_AND_EDITS, false, paths);
+ return mockStorageForDirs(mockDir);
+ }
+ }
+
+ private static NNStorage mockStorageForDirs(final StorageDirectory ... mockDirs)
+ throws IOException {
+ NNStorage mockStorage = Mockito.mock(NNStorage.class);
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ FSImageStorageInspector inspector =
+ (FSImageStorageInspector) invocation.getArguments()[0];
+ for (StorageDirectory sd : mockDirs) {
+ inspector.inspectDirectory(sd);
+ }
+ return null;
+ }
+ }).when(mockStorage).inspectStorageDirs(
+ Mockito.<FSImageStorageInspector>anyObject());
+ return mockStorage;
+ }
+}