You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2012/10/23 03:10:38 UTC
svn commit: r1401136 - in /hadoop/common/branches/branch-1: ./ src/hdfs/
src/hdfs/org/apache/hadoop/hdfs/
src/hdfs/org/apache/hadoop/hdfs/server/namenode/
src/test/org/apache/hadoop/hdfs/server/namenode/
Author: szetszwo
Date: Tue Oct 23 01:10:37 2012
New Revision: 1401136
URL: http://svn.apache.org/viewvc?rev=1401136&view=rev
Log:
HDFS-3540. Change the default value of dfs.namenode.edits.toleration.length from -1 to 0, i.e. enable the end of edit log check but do not tolerate any corruption.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Oct 23 01:10:37 2012
@@ -97,6 +97,10 @@ Release 1.2.0 - unreleased
HDFS-4057. NameNode.namesystem should be private. (Brandon Li via suresh)
+ HDFS-3540. Change the default value of dfs.namenode.edits.toleration.length
+ from -1 to 0, i.e. enable the end of edit log check but do not tolerate
+ any corruption. (szetszwo)
+
OPTIMIZATIONS
HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
Modified: hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml (original)
+++ hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml Tue Oct 23 01:10:37 2012
@@ -163,6 +163,25 @@ creations/deletions), or "all".</descrip
directories, for redundancy. Default value is same as dfs.name.dir
</description>
</property>
+
+<property>
+ <name>dfs.namenode.edits.toleration.length</name>
+ <value>0</value>
+ <description>
+ The length in bytes that namenode is willing to tolerate when the edit log
+ is corrupted. The edit log toleration feature checks the entire edit log.
+ It computes read length (the length of valid data), corruption length and
+ padding length. In case that corruption length is non-zero, the corruption
+ will be tolerated only if the corruption length is less than or equal to
+ the toleration length.
+
+ For disabling edit log toleration feature, set this property to -1. When
+ the feature is disabled, the end of edit log will not be checked. In this
+ case, namenode will startup normally even if the end of edit log is
+ corrupted.
+ </description>
+</property>
+
<property>
<name>dfs.web.ugi</name>
<value>webuser,webgroup</value>
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Oct 23 01:10:37 2012
@@ -81,7 +81,7 @@ public class DFSConfigKeys extends Commo
public static final String DFS_NAMENODE_CHECKPOINT_SIZE_KEY = "dfs.namenode.checkpoint.size";
public static final long DFS_NAMENODE_CHECKPOINT_SIZE_DEFAULT = 4194304;
public static final String DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY = "dfs.namenode.edits.toleration.length";
- public static final int DFS_NAMENODE_EDITS_TOLERATION_LENGTH_DEFAULT = -1; //i.e. disable it
+ public static final int DFS_NAMENODE_EDITS_TOLERATION_LENGTH_DEFAULT = 0;
public static final String DFS_NAMENODE_UPGRADE_PERMISSION_KEY = "dfs.namenode.upgrade.permission";
public static final int DFS_NAMENODE_UPGRADE_PERMISSION_DEFAULT = 00777;
public static final String DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY = "dfs.namenode.heartbeat.recheck-interval";
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Tue Oct 23 01:10:37 2012
@@ -1357,6 +1357,21 @@ public class NameNode implements ClientP
return;
}
}
+ final int tolerationLength = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY,
+ DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_DEFAULT);
+ if (tolerationLength >= 0) {
+ if (!confirmPrompt("You have selected Metadata Recovery mode and have set "
+ + DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY + " = "
+ + tolerationLength + ". However, Metadata Recovery mode and the"
+ + " Edit Log Toleration feature cannot be enabled at the same time."
+ + " Disable Edit Log Toleration? (Y/N)\n")) {
+ System.err.println("Recovery aborted at user request.\n");
+ return;
+ }
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY, -1);
+ }
+
MetaRecoveryContext.LOG.info("starting recovery...");
Collection<File> namespaceDirs = FSNamesystem.getNamespaceDirs(conf);
Collection<File> editDirs =
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java Tue Oct 23 01:10:37 2012
@@ -17,7 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
@@ -26,6 +27,7 @@ import java.io.RandomAccessFile;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.Test;
@@ -38,6 +40,8 @@ public class TestEditLogLoading {
// start a cluster
Configuration conf = new Configuration();
conf.set("dfs.name.dir", new File(MiniDFSCluster.getBaseDir(), "name").getPath());
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY, -1);
+
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java Tue Oct 23 01:10:37 2012
@@ -18,22 +18,23 @@
package org.apache.hadoop.hdfs.server.namenode;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.List;
-import static org.junit.Assert.*;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.util.StringUtils;
import org.junit.Test;
@@ -130,6 +131,7 @@ public class TestNameNodeRecovery {
// Start up the mini dfs cluster
Configuration conf = new Configuration();
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY, -1);
MiniDFSCluster cluster;
cluster = new MiniDFSCluster(0, conf, 0, true, true, false,
StartupOption.FORMAT, null, null, null);