You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/07/14 20:53:13 UTC
svn commit: r1146845 [2/2] - in /hadoop/common/branches/HDFS-1073/hdfs: ./
src/java/org/apache/hadoop/hdfs/server/namenode/
src/java/org/apache/hadoop/hdfs/server/protocol/
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Jul 14 18:53:11 2011
@@ -106,6 +106,7 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
@@ -5337,31 +5338,29 @@ public class FSNamesystem implements FSC
}
/**
- * Register a name-node.
- * <p>
- * Registration is allowed if there is no ongoing streaming to
- * another backup node.
- * We currently allow only one backup node, but multiple chackpointers
- * if there are no backups.
+ * Register a Backup name-node, verifying that it belongs
+ * to the correct namespace, and adding it to the set of
+ * active journals if necessary.
*
- * @param registration
- * @throws IOException
+ * @param bnReg registration of the new BackupNode
+ * @param nnReg registration of this NameNode
+ * @throws IOException if the namespace IDs do not match
*/
- void registerBackupNode(NamenodeRegistration registration)
- throws IOException {
+ void registerBackupNode(NamenodeRegistration bnReg,
+ NamenodeRegistration nnReg) throws IOException {
writeLock();
try {
if(getFSImage().getStorage().getNamespaceID()
- != registration.getNamespaceID())
+ != bnReg.getNamespaceID())
throw new IOException("Incompatible namespaceIDs: "
+ " Namenode namespaceID = "
+ getFSImage().getStorage().getNamespaceID() + "; "
- + registration.getRole() +
- " node namespaceID = " + registration.getNamespaceID());
- boolean regAllowed = getEditLog().checkBackupRegistration(registration);
- if(!regAllowed)
- throw new IOException("Registration is not allowed. " +
- "Another node is registered as a backup.");
+ + bnReg.getRole() +
+ " node namespaceID = " + bnReg.getNamespaceID());
+ if (bnReg.getRole() == NamenodeRole.BACKUP) {
+ getFSImage().getEditLog().registerBackupNode(
+ bnReg, nnReg);
+ }
} finally {
writeUnlock();
}
@@ -5943,4 +5942,6 @@ public class FSNamesystem implements FSC
}
}
}
+
+
}
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java Thu Jul 14 18:53:11 2011
@@ -104,4 +104,11 @@ public class FileJournalManager implemen
}
}
+ @Override
+ public EditLogInputStream getInProgressInputStream(long segmentStartsAtTxId)
+ throws IOException {
+ File f = NNStorage.getInProgressEditsFile(sd, segmentStartsAtTxId);
+ return new EditLogFileInputStream(f);
+ }
+
}
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java Thu Jul 14 18:53:11 2011
@@ -52,4 +52,12 @@ public interface JournalManager {
*/
void archiveLogsOlderThan(long minTxIdToKeep, StorageArchiver archiver)
throws IOException;
+
+ /**
+ * @return an EditLogInputStream that reads from the same log that
+ * the edit log is currently writing. May return null if this journal
+ * manager does not support this operation.
+ */
+ EditLogInputStream getInProgressInputStream(long segmentStartsAtTxId)
+ throws IOException;
}
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Thu Jul 14 18:53:11 2011
@@ -744,8 +744,9 @@ public class NameNode implements Namenod
public NamenodeRegistration register(NamenodeRegistration registration)
throws IOException {
verifyVersion(registration.getVersion());
- namesystem.registerBackupNode(registration);
- return setRegistration();
+ NamenodeRegistration myRegistration = setRegistration();
+ namesystem.registerBackupNode(registration, myRegistration);
+ return myRegistration;
}
@Override // NamenodeProtocol
@@ -766,15 +767,6 @@ public class NameNode implements Namenod
namesystem.endCheckpoint(registration, sig);
}
- @Override // NamenodeProtocol
- public void journal(NamenodeRegistration registration,
- int jAction,
- int length,
- byte[] args) throws IOException {
- // Active name-node cannot journal.
- throw new UnsupportedActionException("journal");
- }
-
@Override // ClientProtocol
public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException {
@@ -1151,7 +1143,6 @@ public class NameNode implements Namenod
return namesystem.getTransactionID();
}
- @Deprecated
@Override // NamenodeProtocol
public CheckpointSignature rollEditLog() throws IOException {
return namesystem.rollEditLog();
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Thu Jul 14 18:53:11 2011
@@ -28,8 +28,6 @@ import java.util.Date;
import java.util.Iterator;
import java.util.List;
-import javax.jws.soap.SOAPBinding.Use;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -45,7 +43,6 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.Storage.StorageState;
-import org.apache.hadoop.hdfs.server.namenode.NNStorageArchivalManager.StorageArchiver;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
@@ -390,6 +387,20 @@ public class SecondaryNameNode implement
final CheckpointSignature sig,
final RemoteEditLogManifest manifest
) throws IOException {
+
+ // Sanity check manifest - these could happen if, eg, someone on the
+ // NN side accidentally rmed the storage directories
+ if (manifest.getLogs().isEmpty()) {
+ throw new IOException("Found no edit logs to download on NN since txid "
+ + sig.mostRecentCheckpointTxId);
+ }
+
+ long expectedTxId = sig.mostRecentCheckpointTxId + 1;
+ if (manifest.getLogs().get(0).getStartTxId() != expectedTxId) {
+ throw new IOException("Bad edit log manifest (expected txid = " +
+ expectedTxId + ": " + manifest);
+ }
+
try {
Boolean b = UserGroupInformation.getCurrentUser().doAs(
new PrivilegedExceptionAction<Boolean>() {
@@ -469,6 +480,7 @@ public class SecondaryNameNode implement
*/
boolean doCheckpoint() throws IOException {
checkpointImage.ensureCurrentDirExists();
+ NNStorage dstStorage = checkpointImage.getStorage();
// Tell the namenode to start logging transactions in a new edit file
// Returns a token that would be used to upload the merged image.
@@ -482,7 +494,9 @@ public class SecondaryNameNode implement
} else {
// if we're a fresh 2NN, just take the storage info from the server
// we first talk to.
- checkpointImage.getStorage().setStorageInfo(sig);
+ dstStorage.setStorageInfo(sig);
+ dstStorage.setClusterID(sig.getClusterID());
+ dstStorage.setBlockPoolID(sig.getBlockpoolID());
}
// error simulation code for junit test
@@ -494,28 +508,17 @@ public class SecondaryNameNode implement
RemoteEditLogManifest manifest =
namenode.getEditLogManifest(sig.mostRecentCheckpointTxId + 1);
- // Sanity check manifest - these could happen if, eg, someone on the
- // NN side accidentally rmed the storage directories
- if (manifest.getLogs().isEmpty()) {
- throw new IOException("Found no edit logs to download on NN since txid "
- + sig.mostRecentCheckpointTxId);
- }
- if (manifest.getLogs().get(0).getStartTxId() != sig.mostRecentCheckpointTxId + 1) {
- throw new IOException("Bad edit log manifest (expected txid = " +
- (sig.mostRecentCheckpointTxId + 1) + ": " + manifest);
- }
-
boolean loadImage = downloadCheckpointFiles(
fsName, checkpointImage, sig, manifest); // Fetch fsimage and edits
- doMerge(conf, sig, manifest, loadImage, checkpointImage);
+ doMerge(sig, manifest, loadImage, checkpointImage);
//
// Upload the new image into the NameNode. Then tell the Namenode
// to make this new uploaded image as the most current image.
//
- long txid = checkpointImage.getStorage().getMostRecentCheckpointTxId();
+ long txid = checkpointImage.getLastAppliedTxId();
TransferFsImage.uploadImageFromStorage(fsName, getImageListenAddress(),
- checkpointImage.getStorage(), txid);
+ dstStorage, txid);
// error simulation code for junit test
if (ErrorSimulator.getErrorSimulation(1)) {
@@ -524,7 +527,7 @@ public class SecondaryNameNode implement
}
LOG.warn("Checkpoint done. New Image Size: "
- + checkpointImage.getStorage().getFsImageName(txid).length());
+ + dstStorage.getFsImageName(txid).length());
// Since we've successfully checkpointed, we can remove some old
// image files
@@ -678,6 +681,12 @@ public class SecondaryNameNode implement
Collection<URI> imageDirs,
Collection<URI> editsDirs) throws IOException {
super(conf, (FSNamesystem)null, imageDirs, editsDirs);
+ setFSNamesystem(new FSNamesystem(this, conf));
+
+ // the 2NN never writes edits -- it only downloads them. So
+ // we shouldn't have any editLog instance. Setting to null
+ // makes sure we don't accidentally depend on it.
+ editLog = null;
}
/**
@@ -749,45 +758,42 @@ public class SecondaryNameNode implement
}
}
- static void doMerge(Configuration conf,
+ static void doMerge(
CheckpointSignature sig, RemoteEditLogManifest manifest,
boolean loadImage, FSImage dstImage) throws IOException {
NNStorage dstStorage = dstImage.getStorage();
dstStorage.setStorageInfo(sig);
if (loadImage) {
- // TODO: dstImage.namesystem.close(); ??
- dstImage.namesystem = new FSNamesystem(dstImage, conf);
- dstImage.editLog = new FSEditLog(dstStorage);
-
File file = dstStorage.findImageFile(sig.mostRecentCheckpointTxId);
if (file == null) {
throw new IOException("Couldn't find image file at txid " +
sig.mostRecentCheckpointTxId + " even though it should have " +
"just been downloaded");
}
- LOG.debug("2NN loading image from " + file);
- dstImage.loadFSImage(file);
+ dstImage.reloadFromImageFile(file);
}
+
+ rollForwardByApplyingLogs(manifest, dstImage);
+ dstImage.saveFSImageInAllDirs(dstImage.getLastAppliedTxId());
+ dstStorage.writeAll();
+ }
+
+ static void rollForwardByApplyingLogs(
+ RemoteEditLogManifest manifest,
+ FSImage dstImage) throws IOException {
+ NNStorage dstStorage = dstImage.getStorage();
+
List<File> editsFiles = Lists.newArrayList();
for (RemoteEditLog log : manifest.getLogs()) {
File f = dstStorage.findFinalizedEditsFile(
log.getStartTxId(), log.getEndTxId());
- editsFiles.add(f);
+ if (log.getStartTxId() > dstImage.getLastAppliedTxId()) {
+ editsFiles.add(f);
+ }
}
LOG.info("SecondaryNameNode about to load edits from " +
editsFiles.size() + " file(s).");
dstImage.loadEdits(editsFiles);
-
- // TODO: why do we need the following two lines? We shouldn't have even
- // been able to download an image from a NN that had a different
- // cluster ID or blockpool ID! this should only be done for the
- // very first checkpoint.
- dstStorage.setClusterID(sig.getClusterID());
- dstStorage.setBlockPoolID(sig.getBlockpoolID());
-
- sig.validateStorageInfo(dstImage);
- dstImage.saveFSImageInAllDirs(dstImage.getEditLog().getLastWrittenTxId());
- dstStorage.writeAll();
}
}
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java Thu Jul 14 18:53:11 2011
@@ -84,6 +84,16 @@ class TransferFsImage implements FSConst
List<File> dstFiles = dstStorage.getFiles(NameNodeDirType.EDITS, fileName);
assert !dstFiles.isEmpty() : "No checkpoint targets.";
+
+ for (File f : dstFiles) {
+ if (f.exists() && f.canRead()) {
+ LOG.info("Skipping download of remote edit log " +
+ log + " since it already is stored locally at " + f);
+ return;
+ } else {
+ LOG.debug("Dest file: " + f);
+ }
+ }
getFileClient(fsName, fileid, dstFiles, dstStorage, false);
LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " +
Added: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/BackupNodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/BackupNodeProtocol.java?rev=1146845&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/BackupNodeProtocol.java (added)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/BackupNodeProtocol.java Thu Jul 14 18:53:11 2011
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.security.KerberosInfo;
+
+@KerberosInfo(
+ serverPrincipal = DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY,
+ clientPrincipal = DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY)
+@InterfaceAudience.Private
+public interface BackupNodeProtocol extends VersionedProtocol {
+ public static final long versionID = 1L;
+
+ /**
+ * Journal edit records.
+ * This message is sent by the active name-node to the backup node
+ * via {@code EditLogBackupOutputStream} in order to synchronize meta-data
+ * changes with the backup namespace image.
+ *
+ * @param registration active node registration
+ * @param firstTxnId the first transaction of this batch
+ * @param numTxns number of transactions
+ * @param records byte array containing serialized journal records
+ */
+ public void journal(NamenodeRegistration registration,
+ long firstTxnId,
+ int numTxns,
+ byte[] records) throws IOException;
+
+ /**
+ * Notify the BackupNode that the NameNode has rolled its edit logs
+ * and is now writing a new log segment.
+ * @param registration the registration of the active NameNode
+ * @param txid the first txid in the new log
+ */
+ public void startLogSegment(NamenodeRegistration registration,
+ long txid) throws IOException;
+}
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java Thu Jul 14 18:53:11 2011
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
import org.apache.hadoop.ipc.VersionedProtocol;
import org.apache.hadoop.security.KerberosInfo;
@@ -50,11 +51,6 @@ public interface NamenodeProtocol extend
final static int NOTIFY = 0;
final static int FATAL = 1;
- // Journal action codes. See journal().
- public static byte JA_IS_ALIVE = 100; // check whether the journal is alive
- public static byte JA_JOURNAL = 101; // just journal
- public static byte JA_JSPOOL_START = 102; // = FSEditLogOpCodes.OP_JSPOOL_START
-
public final static int ACT_UNKNOWN = 0; // unknown action
public final static int ACT_SHUTDOWN = 50; // shutdown node
public final static int ACT_CHECKPOINT = 51; // do checkpoint
@@ -167,22 +163,5 @@ public interface NamenodeProtocol extend
*/
public RemoteEditLogManifest getEditLogManifest(long sinceTxId)
throws IOException;
-
- /**
- * Journal edit records.
- * This message is sent by the active name-node to the backup node
- * via {@code EditLogBackupOutputStream} in order to synchronize meta-data
- * changes with the backup namespace image.
- *
- * @param registration active node registration
- * @param jAction journal action
- * @param length length of the byte array
- * @param records byte array containing serialized journal records
- * @throws IOException
- */
- public void journal(NamenodeRegistration registration,
- int jAction,
- int length,
- byte[] records) throws IOException;
}
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java Thu Jul 14 18:53:11 2011
@@ -27,12 +27,15 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
import org.apache.hadoop.io.WritableUtils;
+import com.google.common.collect.ImmutableMap;
+
/**
* NamespaceInfo is returned by the name-node in reply
* to a data-node handshake.
@@ -98,4 +101,26 @@ public class NamespaceInfo extends Stora
public String toString(){
return super.toString() + ";bpid=" + blockPoolID;
}
+
+ public void validateStorage(NNStorage storage) throws IOException {
+ if (layoutVersion != storage.getLayoutVersion() ||
+ namespaceID != storage.getNamespaceID() ||
+ cTime != storage.cTime ||
+ !clusterID.equals(storage.getClusterID()) ||
+ !blockPoolID.equals(storage.getBlockPoolID())) {
+ throw new IOException("Inconsistent namespace information:\n" +
+ "NamespaceInfo has:\n" +
+ "LV=" + layoutVersion + ";" +
+ "NS=" + namespaceID + ";" +
+ "cTime=" + cTime + ";" +
+ "CID=" + clusterID + ";" +
+ "BPID=" + blockPoolID +
+ ".\nStorage has:\n" +
+ "LV=" + storage.getLayoutVersion() + ";" +
+ "NS=" + storage.getNamespaceID() + ";" +
+ "cTime=" + storage.getCTime() + ";" +
+ "CID=" + storage.getClusterID() + ";" +
+ "BPID=" + storage.getBlockPoolID() + ".");
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Thu Jul 14 18:53:11 2011
@@ -25,8 +25,11 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigInteger;
+import java.net.URI;
import java.security.MessageDigest;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -34,6 +37,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog;
import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage;
@@ -44,6 +48,7 @@ import org.apache.hadoop.io.MD5Hash;
import org.mockito.Mockito;
import com.google.common.base.Joiner;
+import com.google.common.collect.ComparisonChain;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -255,6 +260,31 @@ public abstract class FSImageTestUtil {
}
/**
+ * Assert that the NameNode has checkpoints at the expected
+ * transaction IDs.
+ */
+ static void assertNNHasCheckpoints(MiniDFSCluster cluster,
+ List<Integer> txids) {
+
+ for (File nameDir : getNameNodeCurrentDirs(cluster)) {
+ // Should have fsimage_N for the three checkpoints
+ for (long checkpointTxId : txids) {
+ File image = new File(nameDir,
+ NNStorage.getImageFileName(checkpointTxId));
+ assertTrue("Expected non-empty " + image, image.length() > 0);
+ }
+ }
+ }
+
+ static List<File> getNameNodeCurrentDirs(MiniDFSCluster cluster) {
+ List<File> nameDirs = Lists.newArrayList();
+ for (URI u : cluster.getNameDirs(0)) {
+ nameDirs.add(new File(u.getPath(), "current"));
+ }
+ return nameDirs;
+ }
+
+ /**
* @return the latest edits log, finalized or otherwise, from the given
* storage directory.
*/
@@ -264,7 +294,17 @@ public abstract class FSImageTestUtil {
new FSImageTransactionalStorageInspector();
inspector.inspectDirectory(sd);
- return inspector.foundEditLogs.get(inspector.foundEditLogs.size() - 1);
+ List<FoundEditLog> foundEditLogs = Lists.newArrayList(
+ inspector.getFoundEditLogs());
+ return Collections.max(foundEditLogs, new Comparator<FoundEditLog>() {
+ @Override
+ public int compare(FoundEditLog a, FoundEditLog b) {
+ return ComparisonChain.start()
+ .compare(a.getStartTxId(), b.getStartTxId())
+ .compare(a.getLastTxId(), b.getLastTxId())
+ .result();
+ }
+ });
}
/**
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Thu Jul 14 18:53:11 2011
@@ -19,9 +19,12 @@ package org.apache.hadoop.hdfs.server.na
import java.io.File;
import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
@@ -29,14 +32,28 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
-import org.apache.hadoop.hdfs.server.namenode.FSImage.CheckpointStates;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
import junit.framework.TestCase;
public class TestBackupNode extends TestCase {
public static final Log LOG = LogFactory.getLog(TestBackupNode.class);
+
+ static {
+ ((Log4JLogger)Checkpointer.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger)BackupImage.LOG).getLogger().setLevel(Level.ALL);
+ }
+
static final String BASE_DIR = MiniDFSCluster.getBaseDirectory();
protected void setUp() throws Exception {
@@ -53,73 +70,185 @@ public class TestBackupNode extends Test
dirB.mkdirs();
}
- protected void tearDown() throws Exception {
- super.tearDown();
- File baseDir = new File(BASE_DIR);
- if(!(FileUtil.fullyDelete(baseDir)))
- throw new IOException("Cannot remove directory: " + baseDir);
- }
-
- static void writeFile(FileSystem fileSys, Path name, int repl)
- throws IOException {
- TestCheckpoint.writeFile(fileSys, name, repl);
- }
-
-
- static void checkFile(FileSystem fileSys, Path name, int repl)
- throws IOException {
- TestCheckpoint.checkFile(fileSys, name, repl);
- }
-
- void cleanupFile(FileSystem fileSys, Path name)
- throws IOException {
- TestCheckpoint.cleanupFile(fileSys, name);
- }
-
- static String getBackupNodeDir(StartupOption t, int i) {
- return BASE_DIR + "name" + t.getName() + i + "/";
+ static String getBackupNodeDir(StartupOption t, int idx) {
+ return BASE_DIR + "name" + t.getName() + idx + "/";
}
BackupNode startBackupNode(Configuration conf,
- StartupOption t, int i) throws IOException {
+ StartupOption startupOpt,
+ int idx) throws IOException {
Configuration c = new HdfsConfiguration(conf);
- String dirs = getBackupNodeDir(t, i);
+ String dirs = getBackupNodeDir(startupOpt, idx);
c.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, dirs);
c.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
"${" + DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + "}");
- return (BackupNode)NameNode.createNameNode(new String[]{t.getName()}, c);
+ c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,
+ "127.0.0.1:0");
+
+ return (BackupNode)NameNode.createNameNode(new String[]{startupOpt.getName()}, c);
}
- void waitCheckpointDone(BackupNode backup) {
-/* do {
+ void waitCheckpointDone(
+ MiniDFSCluster cluster, BackupNode backup, long txid) {
+ long thisCheckpointTxId;
+ do {
try {
- LOG.info("Waiting checkpoint to complete...");
+ LOG.info("Waiting checkpoint to complete... " +
+ "checkpoint txid should increase above " + txid);
Thread.sleep(1000);
} catch (Exception e) {}
- } while(backup.getCheckpointState() != CheckpointStates.START); */
+ thisCheckpointTxId = backup.getFSImage().getStorage()
+ .getMostRecentCheckpointTxId();
+
+ } while (thisCheckpointTxId < txid);
+
+ // Check that the checkpoint got uploaded to NN successfully
+ FSImageTestUtil.assertNNHasCheckpoints(cluster,
+ Collections.singletonList((int)thisCheckpointTxId));
}
- public void testCheckpoint() throws IOException {
+ public void testCheckpointNode() throws Exception {
testCheckpoint(StartupOption.CHECKPOINT);
- testCheckpoint(StartupOption.BACKUP);
}
+
+ /**
+ * Ensure that the backupnode will tail edits from the NN
+ * and keep in sync, even while the NN rolls, checkpoints
+ * occur, etc.
+ */
+ public void testBackupNodeTailsEdits() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ FileSystem fileSys = null;
+ BackupNode backup = null;
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(0).build();
+ fileSys = cluster.getFileSystem();
+ backup = startBackupNode(conf, StartupOption.BACKUP, 1);
+
+ BackupImage bnImage = backup.getBNImage();
+ testBNInSync(cluster, backup, 1);
+
+ // Force a roll -- BN should roll with NN.
+ NameNode nn = cluster.getNameNode();
+ nn.rollEditLog();
+ assertEquals(bnImage.getEditLog().getCurSegmentTxId(),
+ nn.getFSImage().getEditLog().getCurSegmentTxId());
+
+ // BN should stay in sync after roll
+ testBNInSync(cluster, backup, 2);
+
+ long nnImageBefore =
+ nn.getFSImage().getStorage().getMostRecentCheckpointTxId();
+ // BN checkpoint
+ backup.doCheckpoint();
+
+ // NN should have received a new image
+ long nnImageAfter =
+ nn.getFSImage().getStorage().getMostRecentCheckpointTxId();
+
+ assertTrue("nn should have received new checkpoint. before: " +
+ nnImageBefore + " after: " + nnImageAfter,
+ nnImageAfter > nnImageBefore);
+
+ // BN should stay in sync after checkpoint
+ testBNInSync(cluster, backup, 3);
+
+ // Stop BN
+ StorageDirectory sd = bnImage.getStorage().getStorageDir(0);
+ backup.stop();
+ backup = null;
+
+ // When shutting down the BN, it shouldn't finalize logs that are
+ // still open on the NN
+ FoundEditLog editsLog = FSImageTestUtil.findLatestEditsLog(sd);
+ assertEquals(editsLog.getStartTxId(),
+ nn.getFSImage().getEditLog().getCurSegmentTxId());
+ assertTrue("Should not have finalized " + editsLog,
+ editsLog.isInProgress());
+
+ // do some edits
+ assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down")));
+
+ // start a new backup node
+ backup = startBackupNode(conf, StartupOption.BACKUP, 1);
+
+ testBNInSync(cluster, backup, 4);
+ assertNotNull(backup.getNamesystem().getFileInfo("/edit-while-bn-down", false));
+ } finally {
+ LOG.info("Shutting down...");
+ if (backup != null) backup.stop();
+ if (fileSys != null) fileSys.close();
+ if (cluster != null) cluster.shutdown();
+ }
+
+ assertStorageDirsMatch(cluster.getNameNode(), backup);
+ }
+
+ private void testBNInSync(MiniDFSCluster cluster, final BackupNode backup,
+ int testIdx) throws Exception {
+
+ final NameNode nn = cluster.getNameNode();
+ final FileSystem fs = cluster.getFileSystem();
+
+ // Do a bunch of namespace operations, make sure they're replicated
+ // to the BN.
+ for (int i = 0; i < 10; i++) {
+ final String src = "/test_" + testIdx + "_" + i;
+ LOG.info("Creating " + src + " on NN");
+ Path p = new Path(src);
+ assertTrue(fs.mkdirs(p));
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ LOG.info("Checking for " + src + " on BN");
+ try {
+ boolean hasFile = backup.getNamesystem().getFileInfo(src, false) != null;
+ boolean txnIdMatch = backup.getTransactionID() == nn.getTransactionID();
+ return hasFile && txnIdMatch;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }, 30, 10000);
+ }
+
+ assertStorageDirsMatch(nn, backup);
+ }
+
+ private void assertStorageDirsMatch(final NameNode nn, final BackupNode backup)
+ throws Exception {
+ // Check that the stored files in the name dirs are identical
+ List<File> dirs = Lists.newArrayList(
+ FSImageTestUtil.getCurrentDirs(nn.getFSImage().getStorage(),
+ null));
+ dirs.addAll(FSImageTestUtil.getCurrentDirs(backup.getFSImage().getStorage(),
+ null));
+ FSImageTestUtil.assertParallelFilesAreIdentical(dirs, ImmutableSet.of("VERSION"));
+ }
+
+ public void testBackupNode() throws Exception {
+ testCheckpoint(StartupOption.BACKUP);
+ }
- void testCheckpoint(StartupOption op) throws IOException {
+ void testCheckpoint(StartupOption op) throws Exception {
Path file1 = new Path("checkpoint.dat");
Path file2 = new Path("checkpoint2.dat");
Configuration conf = new HdfsConfiguration();
- short replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
conf.set(DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY, "0");
conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); // disable block scanner
- int numDatanodes = Math.max(3, replication);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
BackupNode backup = null;
try {
cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(numDatanodes).build();
+ .numDataNodes(0).build();
fileSys = cluster.getFileSystem();
//
// verify that 'format' really blew away all pre-existing files
@@ -130,14 +259,14 @@ public class TestBackupNode extends Test
//
// Create file1
//
- writeFile(fileSys, file1, replication);
- checkFile(fileSys, file1, replication);
+ assertTrue(fileSys.mkdirs(file1));
//
// Take a checkpoint
//
+ long txid = cluster.getNameNode().getTransactionID();
backup = startBackupNode(conf, op, 1);
- waitCheckpointDone(backup);
+ waitCheckpointDone(cluster, backup, txid);
} catch(IOException e) {
LOG.error("Error in TestBackupNode:", e);
assertTrue(e.getLocalizedMessage(), false);
@@ -146,39 +275,45 @@ public class TestBackupNode extends Test
if(fileSys != null) fileSys.close();
if(cluster != null) cluster.shutdown();
}
- File imageFileNN = new File(BASE_DIR, "name1/current/fsimage");
- File imageFileBN = new File(getBackupNodeDir(op, 1), "/current/fsimage");
- LOG.info("NameNode fsimage length = " + imageFileNN.length());
- LOG.info("Backup Node fsimage length = " + imageFileBN.length());
- assertTrue(imageFileNN.length() == imageFileBN.length());
+ File nnCurDir = new File(BASE_DIR, "name1/current/");
+ File bnCurDir = new File(getBackupNodeDir(op, 1), "/current/");
+ FSImageTestUtil.assertParallelFilesAreIdentical(
+ ImmutableList.of(bnCurDir, nnCurDir),
+ ImmutableSet.<String>of("VERSION"));
+
try {
//
// Restart cluster and verify that file1 still exist.
//
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes)
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(false).build();
fileSys = cluster.getFileSystem();
// check that file1 still exists
- checkFile(fileSys, file1, replication);
- cleanupFile(fileSys, file1);
+ assertTrue(fileSys.exists(file1));
+ fileSys.delete(file1, true);
// create new file file2
- writeFile(fileSys, file2, replication);
- checkFile(fileSys, file2, replication);
+ fileSys.mkdirs(file2);
//
// Take a checkpoint
//
backup = startBackupNode(conf, op, 1);
- waitCheckpointDone(backup);
+ long txid = cluster.getNameNode().getTransactionID();
+ waitCheckpointDone(cluster, backup, txid);
for (int i = 0; i < 10; i++) {
- writeFile(fileSys, new Path("file_" + i), replication);
+ fileSys.mkdirs(new Path("file_" + i));
}
+ txid = cluster.getNameNode().getTransactionID();
+ backup.doCheckpoint();
+ waitCheckpointDone(cluster, backup, txid);
+
+ txid = cluster.getNameNode().getTransactionID();
backup.doCheckpoint();
- waitCheckpointDone(backup);
+ waitCheckpointDone(cluster, backup, txid);
} catch(IOException e) {
LOG.error("Error in TestBackupNode:", e);
@@ -188,22 +323,22 @@ public class TestBackupNode extends Test
if(fileSys != null) fileSys.close();
if(cluster != null) cluster.shutdown();
}
- LOG.info("NameNode fsimage length = " + imageFileNN.length());
- LOG.info("Backup Node fsimage length = " + imageFileBN.length());
- assertTrue(imageFileNN.length() == imageFileBN.length());
+ FSImageTestUtil.assertParallelFilesAreIdentical(
+ ImmutableList.of(bnCurDir, nnCurDir),
+ ImmutableSet.<String>of("VERSION"));
try {
//
// Restart cluster and verify that file2 exists and
// file1 does not exist.
//
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).build();
fileSys = cluster.getFileSystem();
assertTrue(!fileSys.exists(file1));
// verify that file2 exists
- checkFile(fileSys, file2, replication);
+ assertTrue(fileSys.exists(file2));
} catch(IOException e) {
LOG.error("Error in TestBackupNode:", e);
assertTrue(e.getLocalizedMessage(), false);
@@ -212,52 +347,4 @@ public class TestBackupNode extends Test
cluster.shutdown();
}
}
-
- /**
- * Test that only one backup node can register.
- * @throws IOException
- */
- public void testBackupRegistration() throws IOException {
- Configuration conf1 = new HdfsConfiguration();
- Configuration conf2 = null;
- MiniDFSCluster cluster = null;
- BackupNode backup1 = null;
- BackupNode backup2 = null;
- try {
- // start name-node and backup node 1
- cluster = new MiniDFSCluster.Builder(conf1).numDataNodes(0).build();
- conf1.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY, "0.0.0.0:7771");
- conf1.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY, "0.0.0.0:7775");
- backup1 = startBackupNode(conf1, StartupOption.BACKUP, 1);
- // try to start backup node 2
- conf2 = new HdfsConfiguration(conf1);
- conf2.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY, "0.0.0.0:7772");
- conf2.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY, "0.0.0.0:7776");
- try {
- backup2 = startBackupNode(conf2, StartupOption.BACKUP, 2);
- backup2.stop();
- backup2 = null;
- assertTrue("Only one backup node should be able to start", false);
- } catch(IOException e) {
- assertTrue(
- e.getLocalizedMessage().contains("Registration is not allowed"));
- // should fail - doing good
- }
- // stop backup node 1; backup node 2 should be able to start
- backup1.stop();
- backup1 = null;
- try {
- backup2 = startBackupNode(conf2, StartupOption.BACKUP, 2);
- } catch(IOException e) {
- assertTrue("Backup node 2 should be able to start", false);
- }
- } catch(IOException e) {
- LOG.error("Error in TestBackupNode:", e);
- assertTrue(e.getLocalizedMessage(), false);
- } finally {
- if(backup1 != null) backup1.stop();
- if(backup2 != null) backup2.stop();
- if(cluster != null) cluster.shutdown();
- }
- }
}
Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1146845&r1=1146844&r2=1146845&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original)
+++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Thu Jul 14 18:53:11 2011
@@ -46,8 +46,6 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
-import org.apache.hadoop.hdfs.server.common.Storage;
-import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -56,7 +54,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
-import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.DelayAnswer;
import org.apache.hadoop.util.StringUtils;
@@ -71,6 +68,9 @@ import com.google.common.collect.Immutab
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil.assertNNHasCheckpoints;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil.getNameNodeCurrentDirs;
+
/**
* This class tests the creation and validation of a checkpoint.
*/
@@ -1699,22 +1699,6 @@ public class TestCheckpoint extends Test
}
}
- /**
- * Assert that the NameNode has checkpoints at the expected
- * transaction IDs.
- */
- private void assertNNHasCheckpoints(MiniDFSCluster cluster,
- List<Integer> txids) {
-
- for (File nameDir : getNameNodeCurrentDirs(cluster)) {
- // Should have fsimage_N for the three checkpoints
- for (long checkpointTxId : txids) {
- File image = new File(nameDir,
- NNStorage.getImageFileName(checkpointTxId));
- assertTrue("Expected non-empty " + image, image.length() > 0);
- }
- }
- }
/**
* Assert that if any two files have the same name across the 2NNs
@@ -1732,14 +1716,6 @@ public class TestCheckpoint extends Test
ImmutableSet.of("VERSION"));
}
- private List<File> getNameNodeCurrentDirs(MiniDFSCluster cluster) {
- List<File> nameDirs = Lists.newArrayList();
- for (URI u : cluster.getNameDirs(0)) {
- nameDirs.add(new File(u.getPath(), "current"));
- }
- return nameDirs;
- }
-
@SuppressWarnings("deprecation")
private List<File> getCheckpointCurrentDirs(SecondaryNameNode secondary) {
List<File> ret = Lists.newArrayList();