You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2011/04/23 01:10:22 UTC
svn commit: r1096079 - in /hadoop/common/branches/branch-0.20-security: ./
src/hdfs/org/apache/hadoop/hdfs/protocol/
src/hdfs/org/apache/hadoop/hdfs/server/common/
src/hdfs/org/apache/hadoop/hdfs/server/namenode/
src/test/org/apache/hadoop/hdfs/server/...
Author: suresh
Date: Fri Apr 22 23:10:22 2011
New Revision: 1096079
URL: http://svn.apache.org/viewvc?rev=1096079&view=rev
Log:
HDFS-1842. Handle editlog opcode conflict with 0.20.203 during upgrade, by throwing an error to indicate the editlog needs to be empty. Contributed by Suresh Srinivas.
Modified:
hadoop/common/branches/branch-0.20-security/CHANGES.txt
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Fri Apr 22 23:10:22 2011
@@ -55,6 +55,10 @@ Release 0.20.204.0 - unreleased
MAPREDUCE-2443. Fix TaskAspect for TaskUmbilicalProtocol.ping(..).
(Siddharth Seth via szetszwo)
+ HDFS-1842. Handle editlog opcode conflict with 0.20.203 during upgrade,
+ by throwing an error to indicate the editlog needs to be empty.
+ (suresh)
+
IMPROVEMENTS
HDFS-1541. Not marking datanodes dead when namenode in safemode.
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java Fri Apr 22 23:10:22 2011
@@ -77,8 +77,8 @@ public interface FSConstants {
// Version is reflected in the data storage file.
// Versions are negative.
// Decrement LAYOUT_VERSION to define a new version.
- public static final int LAYOUT_VERSION = -19;
+ public static final int LAYOUT_VERSION = -32;
// Current version:
- // -19: added new OP_[GET|RENEW|CANCEL]_DELEGATION_TOKEN and
- // OP_UPDATE_MASTER_KEY.
+ // -32: to handle editlog opcode conflicts with 0.20.203 during upgrades and
+ // to disallow upgrade to release 0.21.
}
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java Fri Apr 22 23:10:22 2011
@@ -73,6 +73,9 @@ public abstract class Storage extends St
* any upgrade code that uses this constant should also be removed. */
public static final int PRE_GENERATIONSTAMP_LAYOUT_VERSION = -13;
+ /** Layout versions of 203 release */
+ public static final int[] LAYOUT_VERSIONS_203 = {-19, -31};
+
private static final String STORAGE_FILE_LOCK = "in_use.lock";
protected static final String STORAGE_FILE_VERSION = "VERSION";
public static final String STORAGE_DIR_CURRENT = "current";
@@ -849,4 +852,13 @@ public abstract class Storage extends St
file.writeBytes(messageForPreUpgradeVersion);
file.getFD().sync();
}
+
+ public static boolean is203LayoutVersion(int layoutVersion) {
+ for (int lv : LAYOUT_VERSIONS_203) {
+ if (lv == layoutVersion) {
+ return true;
+ }
+ }
+ return false;
+ }
}
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Fri Apr 22 23:10:22 2011
@@ -70,10 +70,10 @@ public class FSEditLog {
private static final byte OP_CLEAR_NS_QUOTA = 12; // clear namespace quota
private static final byte OP_TIMES = 13; // sets mod & access time on a file
private static final byte OP_SET_QUOTA = 14; // sets name and disk quotas.
- private static final byte OP_GET_DELEGATION_TOKEN = 15; //new delegation token
- private static final byte OP_RENEW_DELEGATION_TOKEN = 16; //renew delegation token
- private static final byte OP_CANCEL_DELEGATION_TOKEN = 17; //cancel delegation token
- private static final byte OP_UPDATE_MASTER_KEY = 18; //update master key
+ private static final byte OP_GET_DELEGATION_TOKEN = 18; //new delegation token
+ private static final byte OP_RENEW_DELEGATION_TOKEN = 19; //renew delegation token
+ private static final byte OP_CANCEL_DELEGATION_TOKEN = 20; //cancel delegation token
+ private static final byte OP_UPDATE_MASTER_KEY = 21; //update master key
private static int sizeFlushBuffer = 512*1024;
@@ -832,6 +832,21 @@ public class FSEditLog {
}
}
}
+ } catch (IOException ex) {
+ // Failed to load 0.20.203 version edits during upgrade. This version has
+ // conflicting opcodes with the later releases. The editlog must be
+ // emptied by restarting the namenode, before proceeding with the upgrade.
+ if (Storage.is203LayoutVersion(logVersion) &&
+ logVersion != FSConstants.LAYOUT_VERSION) {
+ String msg = "During upgrade, failed to load the editlog version " +
+ logVersion + " from release 0.20.203. Please go back to the old " +
+ " release and restart the namenode. This empties the editlog " +
+ " and saves the namespace. Resume the upgrade after this step.";
+ throw new IOException(msg, ex);
+ } else {
+ throw ex;
+ }
+
} finally {
in.close();
}
Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Fri Apr 22 23:10:22 2011
@@ -21,16 +21,13 @@ import junit.framework.TestCase;
import java.io.*;
import java.util.Collection;
import java.util.Iterator;
-import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.*;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.io.ArrayWritable;
-import org.apache.hadoop.io.UTF8;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.hdfs.server.namenode.FSEditLog.EditLogFileInputStream;
+import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile;
@@ -156,4 +153,10 @@ public class TestEditLog extends TestCas
}
}
+
+ public void test203LayoutVersion() {
+ for (int lv : Storage.LAYOUT_VERSIONS_203) {
+ assertTrue(Storage.is203LayoutVersion(lv));
+ }
+ }
}