You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2010/11/18 22:26:43 UTC
svn commit: r1036643 - in /hadoop/hdfs/branches/branch-0.22: ./
src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/
src/java/org/apache/hadoop/hdfs/
src/java/org/apache/hadoop/hdfs/server/datanode/
src/java/org/apache/hadoop/hdfs/server/namenode/ src/tes...
Author: eli
Date: Thu Nov 18 21:26:43 2010
New Revision: 1036643
URL: http://svn.apache.org/viewvc?rev=1036643&view=rev
Log:
HDFS-528. svn merge -c 1036631 from trunk
Added:
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java
- copied unchanged from r1036631, hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java
Modified:
hadoop/hdfs/branches/branch-0.22/ (props changed)
hadoop/hdfs/branches/branch-0.22/CHANGES.txt
hadoop/hdfs/branches/branch-0.22/build.xml (props changed)
hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/hdfs-default.xml
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/branch-0.22/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
/hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:1036213,1036303,1036310
+/hadoop/hdfs/trunk:1036213,1036303,1036310,1036631
Modified: hadoop/hdfs/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/CHANGES.txt?rev=1036643&r1=1036642&r2=1036643&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.22/CHANGES.txt Thu Nov 18 21:26:43 2010
@@ -51,6 +51,9 @@ Release 0.22.0 - Unreleased
HDFS-895. Allow hflush/sync to occur in parallel with new writes
to the file. (Todd Lipcon via hairong)
+ HDFS-528. Add ability for safemode to wait for a minimum number of
+ live datanodes (Todd Lipcon via eli)
+
IMPROVEMENTS
HDFS-1304. Add a new unit test for HftpFileSystem.open(..). (szetszwo)
Propchange: hadoop/hdfs/branches/branch-0.22/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
/hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:1036213,1036303,1036310
+/hadoop/hdfs/trunk/build.xml:1036213,1036303,1036310,1036631
Propchange: hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303,1036310
+/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303,1036310,1036631
Propchange: hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303,1036310
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303,1036310,1036631
Propchange: hadoop/hdfs/branches/branch-0.22/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:1036213,1036303,1036310
+/hadoop/hdfs/trunk/src/java:1036213,1036303,1036310,1036631
Modified: hadoop/hdfs/branches/branch-0.22/src/java/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/hdfs-default.xml?rev=1036643&r1=1036642&r2=1036643&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/hdfs-default.xml (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/hdfs-default.xml Thu Nov 18 21:26:43 2010
@@ -344,12 +344,27 @@ creations/deletions), or "all".</descrip
<description>
Specifies the percentage of blocks that should satisfy
the minimal replication requirement defined by dfs.namenode.replication.min.
- Values less than or equal to 0 mean not to start in safe mode.
+ Values less than or equal to 0 mean not to wait for any particular
+ percentage of blocks before exiting safemode.
Values greater than 1 will make safe mode permanent.
</description>
</property>
<property>
+ <name>dfs.namenode.safemode.min.datanodes</name>
+ <value>0</value>
+ <description>
+ Specifies the number of datanodes that must be considered alive
+ before the name node exits safemode.
+ Values less than or equal to 0 mean not to take the number of live
+ datanodes into account when deciding whether to remain in safe mode
+ during startup.
+ Values greater than the number of datanodes in the cluster
+ will make safe mode permanent.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.safemode.extension</name>
<value>30000</value>
<description>
Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1036643&r1=1036642&r2=1036643&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Nov 18 21:26:43 2010
@@ -57,6 +57,8 @@ public class DFSConfigKeys extends Commo
public static final int DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT = 30000;
public static final String DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY = "dfs.namenode.safemode.threshold-pct";
public static final float DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_DEFAULT = 0.999f;
+ public static final String DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY = "dfs.namenode.safemode.min.datanodes";
+ public static final int DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT = 0;
public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY = "dfs.namenode.secondary.http-address";
public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_DEFAULT = "0.0.0.0:50090";
public static final String DFS_NAMENODE_CHECKPOINT_PERIOD_KEY = "dfs.namenode.checkpoint.period";
Propchange: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -4,4 +4,4 @@
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303,1036310
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303,1036310,1036631
Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1036643&r1=1036642&r2=1036643&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Nov 18 21:26:43 2010
@@ -2595,6 +2595,10 @@ public class FSNamesystem implements FSC
// no need to update its timestamp
// because its is done when the descriptor is created
}
+
+ if (safeMode != null) {
+ safeMode.checkMode();
+ }
return;
} finally {
writeUnlock();
@@ -2929,6 +2933,10 @@ public class FSNamesystem implements FSC
}
unprotectedRemoveDatanode(nodeInfo);
clusterMap.remove(nodeInfo);
+
+ if (safeMode != null) {
+ safeMode.checkMode();
+ }
}
void unprotectedRemoveDatanode(DatanodeDescriptor nodeDescr) {
@@ -3730,6 +3738,8 @@ public class FSNamesystem implements FSC
// configuration fields
/** Safe mode threshold condition %.*/
private double threshold;
+ /** Safe mode minimum number of datanodes alive */
+ private int datanodeThreshold;
/** Safe mode extension after the threshold. */
private int extension;
/** Min replication required by safe mode. */
@@ -3759,6 +3769,9 @@ public class FSNamesystem implements FSC
*/
SafeModeInfo(Configuration conf) {
this.threshold = conf.getFloat(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, 0.95f);
+ this.datanodeThreshold = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY,
+ DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT);
this.extension = conf.getInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0);
this.safeReplication = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,
DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
@@ -3776,6 +3789,7 @@ public class FSNamesystem implements FSC
*/
private SafeModeInfo() {
this.threshold = 1.5f; // this threshold can never be reached
+ this.datanodeThreshold = Integer.MAX_VALUE;
this.extension = Integer.MAX_VALUE;
this.safeReplication = Short.MAX_VALUE + 1; // more than maxReplication
this.blockTotal = -1;
@@ -3868,7 +3882,8 @@ public class FSNamesystem implements FSC
* if DFS is empty or {@link #threshold} == 0
*/
boolean needEnter() {
- return threshold != 0 && blockSafe < blockThreshold;
+ return (threshold != 0 && blockSafe < blockThreshold) ||
+ (getNumLiveDataNodes() < datanodeThreshold);
}
/**
@@ -3958,17 +3973,37 @@ public class FSNamesystem implements FSC
}
if(blockTotal < 0)
return leaveMsg + ".";
-
- String msg = null;
+
+ int numLive = getNumLiveDataNodes();
+ String msg = "";
if (reached == 0) {
- msg = String.format("The reported blocks %d needs additional %d"
- + " blocks to reach the threshold %.4f of total blocks %d. %s",
- blockSafe, (blockThreshold - blockSafe), threshold, blockTotal,
- leaveMsg);
+ if (blockSafe < blockThreshold) {
+ msg += String.format(
+ "The reported blocks %d needs additional %d"
+ + " blocks to reach the threshold %.4f of total blocks %d.",
+ blockSafe, (blockThreshold - blockSafe), threshold, blockTotal);
+ }
+ if (numLive < datanodeThreshold) {
+ if (!"".equals(msg)) {
+ msg += "\n";
+ }
+ msg += String.format(
+ "The number of live datanodes %d needs an additional %d live "
+ + "datanodes to reach the minimum number %d.",
+ numLive, datanodeThreshold - numLive, datanodeThreshold);
+ }
+ msg += " " + leaveMsg;
} else {
msg = String.format("The reported blocks %d has reached the threshold"
- + " %.4f of total blocks %d. %s", blockSafe, threshold,
- blockTotal, leaveMsg);
+ + " %.4f of total blocks %d.", blockSafe, threshold,
+ blockTotal);
+
+ if (datanodeThreshold > 0) {
+ msg += String.format(" The number of live datanodes %d has reached "
+ + "the minimum number %d.",
+ numLive, datanodeThreshold);
+ }
+ msg += " " + leaveMsg;
}
if(reached == 0 || isManual()) { // threshold is not reached or manual
return msg + ".";
Propchange: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303,1036310
+/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303,1036310,1036631
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303,1036310
+/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303,1036310,1036631
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303,1036310
+/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303,1036310,1036631
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 18 21:26:43 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303,1036310
+/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303,1036310,1036631