You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/06/06 07:17:54 UTC
svn commit: r1132525 - in /hadoop/hdfs/trunk: ./
src/java/org/apache/hadoop/hdfs/server/common/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/
Author: todd
Date: Mon Jun 6 05:17:53 2011
New Revision: 1132525
URL: http://svn.apache.org/viewvc?rev=1132525&view=rev
Log:
HDFS-1969. Running rollback on new-version namenode destroys the namespace. Contributed by Todd Lipcon.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Mon Jun 6 05:17:53 2011
@@ -1139,6 +1139,9 @@ Release 0.22.0 - Unreleased
HDFS-977. DataNode.createInterDataNodeProtocolProxy() guards a log
at the wrong level. (Harsh J Chouraria via todd)
+ HDFS-1969. Running rollback on new-version namenode destroys the
+ namespace. (todd)
+
Release 0.21.1 - Unreleased
HDFS-1466. TestFcHdfsSymlink relies on /tmp/test not existing. (eli)
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java Mon Jun 6 05:17:53 2011
@@ -873,7 +873,7 @@ public abstract class Storage extends St
String property = props.getProperty(name);
if (property == null) {
throw new InconsistentFSStateException(sd.root, "file "
- + STORAGE_FILE_VERSION + " has " + name + " mising.");
+ + STORAGE_FILE_VERSION + " has " + name + " missing.");
}
return property;
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Mon Jun 6 05:17:53 2011
@@ -438,6 +438,14 @@ public class FSImage implements NNStorag
// read and verify consistency of the prev dir
sdPrev.read(sdPrev.getPreviousVersionFile());
+ if (prevState.getLayoutVersion() != FSConstants.LAYOUT_VERSION) {
+ throw new IOException(
+ "Cannot rollback to storage version " +
+ prevState.getLayoutVersion() +
+ " using this version of the NameNode, which uses storage version " +
+ FSConstants.LAYOUT_VERSION + ". " +
+ "Please use the previous version of HDFS to perform the rollback.");
+ }
canRollback = true;
}
if (!canRollback)
@@ -960,6 +968,11 @@ public class FSImage implements NNStorag
* Save current image and empty journal into {@code current} directory.
*/
protected void saveCurrent(StorageDirectory sd) throws IOException {
+ if (storage.getLayoutVersion() != FSConstants.LAYOUT_VERSION) {
+ throw new IllegalStateException(
+ "NN with storage version " + FSConstants.LAYOUT_VERSION +
+ "cannot save an image with version " + storage.getLayoutVersion());
+ }
File curDir = sd.getCurrentDir();
NameNodeDirType dirType = (NameNodeDirType)sd.getStorageDirType();
// save new image or new edits
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Mon Jun 6 05:17:53 2011
@@ -732,13 +732,17 @@ public class NNStorage extends Storage i
props.setProperty("distributedUpgradeVersion",
Integer.toString(uVersion));
}
- if (imageDigest == null) {
- imageDigest = MD5Hash.digest(
- new FileInputStream(getStorageFile(sd, NameNodeFile.IMAGE)));
+ if (LayoutVersion.supports(Feature.FSIMAGE_CHECKSUM, layoutVersion)) {
+ // Though the current NN supports this feature, this function
+ // is called with old layoutVersions from the upgrade tests.
+ if (imageDigest == null) {
+ // May be null on the first save after an upgrade.
+ imageDigest = MD5Hash.digest(
+ new FileInputStream(getStorageFile(sd, NameNodeFile.IMAGE)));
+ }
+ props.setProperty(MESSAGE_DIGEST_PROPERTY, imageDigest.toString());
}
- props.setProperty(MESSAGE_DIGEST_PROPERTY, imageDigest.toString());
-
writeCheckpointTime(sd);
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java Mon Jun 6 05:17:53 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.apache.hadoop.util.StringUtils;
/**
* This test ensures the appropriate response (successful or failure) from
@@ -89,7 +90,7 @@ public class TestDFSRollback extends Tes
* Attempts to start a NameNode with the given operation. Starting
* the NameNode should throw an exception.
*/
- void startNameNodeShouldFail(StartupOption operation) {
+ void startNameNodeShouldFail(StartupOption operation, String searchString) {
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.startupOption(operation)
@@ -99,6 +100,10 @@ public class TestDFSRollback extends Tes
.build(); // should fail
throw new AssertionError("NameNode should have failed to start");
} catch (Exception expected) {
+ if (!expected.getMessage().contains(searchString)) {
+ fail("Expected substring '" + searchString + "' in exception " +
+ "but got: " + StringUtils.stringifyException(expected));
+ }
// expected
}
}
@@ -165,7 +170,8 @@ public class TestDFSRollback extends Tes
log("NameNode rollback without existing previous dir", numDirs);
UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
- startNameNodeShouldFail(StartupOption.ROLLBACK);
+ startNameNodeShouldFail(StartupOption.ROLLBACK,
+ "None of the storage directories contain previous fs state");
UpgradeUtilities.createEmptyDirs(nameNodeDirs);
log("DataNode rollback without existing previous dir", numDirs);
@@ -238,7 +244,8 @@ public class TestDFSRollback extends Tes
for (File f : baseDirs) {
FileUtil.fullyDelete(new File(f,"edits"));
}
- startNameNodeShouldFail(StartupOption.ROLLBACK);
+ startNameNodeShouldFail(StartupOption.ROLLBACK,
+ "Edits file is not found");
UpgradeUtilities.createEmptyDirs(nameNodeDirs);
log("NameNode rollback with no image file", numDirs);
@@ -247,7 +254,8 @@ public class TestDFSRollback extends Tes
for (File f : baseDirs) {
FileUtil.fullyDelete(new File(f,"fsimage"));
}
- startNameNodeShouldFail(StartupOption.ROLLBACK);
+ startNameNodeShouldFail(StartupOption.ROLLBACK,
+ "Image file is not found");
UpgradeUtilities.createEmptyDirs(nameNodeDirs);
log("NameNode rollback with corrupt version file", numDirs);
@@ -256,7 +264,8 @@ public class TestDFSRollback extends Tes
for (File f : baseDirs) {
UpgradeUtilities.corruptFile(new File(f,"VERSION"));
}
- startNameNodeShouldFail(StartupOption.ROLLBACK);
+ startNameNodeShouldFail(StartupOption.ROLLBACK,
+ "file VERSION has layoutVersion missing");
UpgradeUtilities.createEmptyDirs(nameNodeDirs);
log("NameNode rollback with old layout version in previous", numDirs);
@@ -269,7 +278,8 @@ public class TestDFSRollback extends Tes
UpgradeUtilities.createNameNodeVersionFile(conf, baseDirs,
storageInfo, UpgradeUtilities.getCurrentBlockPoolID(cluster));
- startNameNodeShouldFail(StartupOption.UPGRADE);
+ startNameNodeShouldFail(StartupOption.ROLLBACK,
+ "Cannot rollback to storage version 1 using this version");
UpgradeUtilities.createEmptyDirs(nameNodeDirs);
} // end numDir loop
}