You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2011/04/26 20:45:26 UTC
svn commit: r1096846 - in /hadoop/hdfs/trunk: ./
src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/server/common/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/ src/jav...
Author: suresh
Date: Tue Apr 26 18:45:25 2011
New Revision: 1096846
URL: http://svn.apache.org/viewvc?rev=1096846&view=rev
Log:
HDFS-1842. Handle editlog opcode conflict with 0.20.203 during upgrade, by throwing an error to indicate the editlog needs to be empty. Contributed by Suresh Srinivas.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue Apr 26 18:45:25 2011
@@ -210,6 +210,10 @@ Trunk (unreleased changes)
HDFS-1594. When the disk becomes full Namenode is getting shutdown and
not able to recover. (Aaron T. Myers via eli)
+ HDFS-1842. Handle editlog opcode conflict with 0.20.203 during upgrade,
+ by throwing an error to indicate the editlog needs to be empty.
+ (suresh)
+
Release 0.22.0 - Unreleased
NEW FEATURES
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java Tue Apr 26 18:45:25 2011
@@ -88,7 +88,7 @@ public interface FSConstants {
// Version is reflected in the data storage file.
// Versions are negative.
// Decrement LAYOUT_VERSION to define a new version.
- public static final int LAYOUT_VERSION = -30;
+ public static final int LAYOUT_VERSION = -34;
// Current version:
- // -30: store only last component of a path in image
+ // -31, -32 and -33 are reserved for 0.20.203, 0.20.204 and 0.22.
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java Tue Apr 26 18:45:25 2011
@@ -78,6 +78,9 @@ public abstract class Storage extends St
// last layout version that did not support persistent rbw replicas
public static final int PRE_RBW_LAYOUT_VERSION = -19;
+ /** Layout versions of 0.20.203 release */
+ public static final int[] LAYOUT_VERSIONS_203 = {-19, -31};
+
private static final String STORAGE_FILE_LOCK = "in_use.lock";
protected static final String STORAGE_FILE_VERSION = "VERSION";
public static final String STORAGE_DIR_CURRENT = "current";
@@ -861,4 +864,13 @@ public abstract class Storage extends St
+ "-" + Integer.toString(storage.getLayoutVersion())
+ "-" + Long.toString(storage.getCTime());
}
+
+ public static boolean is203LayoutVersion(int layoutVersion) {
+ for (int lv203 : LAYOUT_VERSIONS_203) {
+ if (lv203 == layoutVersion) {
+ return true;
+ }
+ }
+ return false;
+ }
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Tue Apr 26 18:45:25 2011
@@ -509,6 +509,8 @@ public class FSEditLogLoader {
}
validateChecksum(in, checksum, numEdits);
}
+ } catch (IOException ex) {
+ check203UpgradeFailure(logVersion, ex);
} finally {
if(closeOnExit)
in.close();
@@ -652,4 +654,25 @@ public class FSEditLogLoader {
}
return blocks;
}
+
+ /**
+ * Throw appropriate exception during upgrade from 203, when editlog loading
+ * could fail due to opcode conflicts.
+ */
+ private void check203UpgradeFailure(int logVersion, IOException ex)
+ throws IOException {
+ // 0.20.203 version version has conflicting opcodes with the later releases.
+ // The editlog must be emptied by restarting the namenode, before proceeding
+ // with the upgrade.
+ if (Storage.is203LayoutVersion(logVersion)
+ && logVersion != FSConstants.LAYOUT_VERSION) {
+ String msg = "During upgrade failed to load the editlog version "
+ + logVersion + " from release 0.20.203. Please go back to the old "
+ + " release and restart the namenode. This empties the editlog "
+ + " and saves the namespace. Resume the upgrade after this step.";
+ throw new IOException(msg, ex);
+ } else {
+ throw ex;
+ }
+ }
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java Tue Apr 26 18:45:25 2011
@@ -37,8 +37,8 @@ import static org.apache.hadoop.hdfs.too
@InterfaceStability.Unstable
class EditsLoaderCurrent implements EditsLoader {
- private static int [] supportedVersions = {
- -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30 };
+ private static int[] supportedVersions = { -18, -19, -20, -21, -22, -23, -24,
+ -25, -26, -27, -28, -30, -31, -32, -33, -34 };
private EditsVisitor v;
private int editsVersion = 0;
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Tue Apr 26 18:45:25 2011
@@ -119,8 +119,8 @@ import org.apache.hadoop.security.token.
class ImageLoaderCurrent implements ImageLoader {
protected final DateFormat dateFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
- private static int [] versions =
- {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30};
+ private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23,
+ -24, -25, -26, -27, -28, -30, -31, -32, -33, -34 };
private int imageVersion = 0;
/* (non-Javadoc)
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java Tue Apr 26 18:45:25 2011
@@ -29,17 +29,20 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* This test ensures the appropriate response (successful or failure) from
* the system when the system is upgraded under various storage state and
* version conditions.
*/
-public class TestDFSUpgrade extends TestCase {
+public class TestDFSUpgrade {
private static final Log LOG = LogFactory.getLog(
"org.apache.hadoop.hdfs.TestDFSUpgrade");
@@ -141,6 +144,7 @@ public class TestDFSUpgrade extends Test
* This test attempts to upgrade the NameNode and DataNode under
* a number of valid and invalid conditions.
*/
+ @Test
public void testUpgrade() throws Exception {
File[] baseDirs;
UpgradeUtilities.initialize();
@@ -256,15 +260,23 @@ public class TestDFSUpgrade extends Test
} // end numDir loop
}
- protected void tearDown() throws Exception {
- LOG.info("Shutting down MiniDFSCluster");
- if (cluster != null) cluster.shutdown();
+ @Test(expected=IOException.class)
+ public void testUpgradeFromPreUpgradeLVFails() throws IOException {
+ // Upgrade from versions prior to Storage#LAST_UPGRADABLE_LAYOUT_VERSION
+ // is not allowed
+ Storage.checkVersionUpgradable(Storage.LAST_PRE_UPGRADE_LAYOUT_VERSION + 1);
+ fail("Expected IOException is not thrown");
}
-
+
+ public void test203LayoutVersion() {
+ for (int lv : Storage.LAYOUT_VERSIONS_203) {
+ assertTrue(Storage.is203LayoutVersion(lv));
+ }
+ }
+
public static void main(String[] args) throws Exception {
new TestDFSUpgrade().testUpgrade();
}
-
}