You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/08/18 18:18:45 UTC
svn commit: r1374605 - in /hadoop/common/branches/branch-1: ./ src/hdfs/
src/hdfs/org/apache/hadoop/hdfs/
src/hdfs/org/apache/hadoop/hdfs/server/namenode/
src/test/org/apache/hadoop/hdfs/server/namenode/
Author: suresh
Date: Sat Aug 18 16:18:45 2012
New Revision: 1374605
URL: http://svn.apache.org/viewvc?rev=1374605&view=rev
Log:
HDFS-3814. Make the replication monitor multipliers configurable in 1.x. Contributed by Jing Zhao
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1374605&r1=1374604&r2=1374605&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Sat Aug 18 16:18:45 2012
@@ -313,6 +313,9 @@ Release 1.1.0 - unreleased
MAPREDUCE-4400. Avoid task finish sleep to improve small job/workflow
latency. (llu)
+ HDFS-3814. Make the replication monitor multipliers configurable in 1.x.
+ (Jing Zhao via suresh)
+
BUG FIXES
MAPREDUCE-4087. [Gridmix] GenerateDistCacheData job of Gridmix can
Modified: hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml?rev=1374605&r1=1374604&r2=1374605&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml (original)
+++ hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml Sat Aug 18 16:18:45 2012
@@ -581,4 +581,35 @@ creations/deletions), or "all".</descrip
<value>${dfs.web.authentication.kerberos.principal}</value>
</property>
+<property>
+ <name>dfs.namenode.invalidate.work.pct.per.iteration</name>
+ <value>0.32f</value>
+ <description>
+ *Note*: Advanced property. Change with caution.
+ This determines the percentage amount of block
+ invalidations (deletes) to do over a single DN heartbeat
+ deletion command. The final deletion count is determined by applying this
+ percentage to the number of live nodes in the system.
+ The resultant number is the number of blocks from the deletion list
+ chosen for proper invalidation over a single heartbeat of a single DN.
+ Value should be a positive, non-zero percentage in float notation (X.Yf),
+ with 1.0f meaning 100%.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.replication.work.multiplier.per.iteration</name>
+ <value>2</value>
+ <description>
+ *Note*: Advanced property. Change with caution.
+ This determines the total amount of block transfers to begin in
+ parallel at a DN, for replication, when such a command list is being
+ sent over a DN heartbeat by the NN. The actual number is obtained by
+ multiplying this multiplier with the total number of live nodes in the
+ cluster. The result number is the number of blocks to begin transfers
+ immediately for, per DN heartbeat. This number can be any positive,
+ non-zero integer.
+ </description>
+</property>
+
</configuration>
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1374605&r1=1374604&r2=1374605&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java Sat Aug 18 16:18:45 2012
@@ -201,6 +201,14 @@ public class DFSConfigKeys extends Commo
public static final String DFS_DATANODE_IPC_ADDRESS_KEY = "dfs.datanode.ipc.address";
public static final String DFS_DATANODE_IPC_ADDRESS_DEFAULT = "0.0.0.0:50020";
+ //Replication monitoring related keys
+ public static final String DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION =
+ "dfs.namenode.invalidate.work.pct.per.iteration";
+ public static final float DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT = 0.32f;
+ public static final String DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION =
+ "dfs.namenode.replication.work.multiplier.per.iteration";
+ public static final int DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT = 2;
+
public static final String DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY = "dfs.block.access.token.enable";
public static final boolean DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT = false;
public static final String DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_KEY = "dfs.block.access.key.update.interval";
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java?rev=1374605&r1=1374604&r2=1374605&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java Sat Aug 18 16:18:45 2012
@@ -24,6 +24,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.StringTokenizer;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -123,5 +124,48 @@ public class DFSUtil {
throw new IllegalArgumentException(ue);
}
}
+
+ /**
+ * Get DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION from configuration.
+ *
+ * @param conf Configuration
+ * @return Value of DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION
+ * @throws IllegalArgumentException If the value is not positive
+ */
+ public static float getInvalidateWorkPctPerIteration(Configuration conf) {
+ float blocksInvalidateWorkPct = conf.getFloat(
+ DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,
+ DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT);
+ if (blocksInvalidateWorkPct <= 0) {
+ throw new IllegalArgumentException(
+ DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION + " = '"
+ + blocksInvalidateWorkPct + "' is invalid. "
+ + "It should be a positive, non-zero float value "
+ + "indicating a percentage.");
+ }
+ return blocksInvalidateWorkPct;
+ }
+
+ /**
+ * Get DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION
+ * from configuration.
+ *
+ * @param conf Configuration
+ * @return Value of DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION
+ * @throws IllegalArgumentException If the value is not positive
+ */
+ public static int getReplWorkMultiplier(Configuration conf) {
+ int blocksReplWorkMultiplier = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
+ DFSConfigKeys.
+ DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT);
+ if (blocksReplWorkMultiplier <= 0) {
+ throw new IllegalArgumentException(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION
+ + " = '" + blocksReplWorkMultiplier + "' is invalid. "
+ + "It should be a positive, non-zero integer value.");
+ }
+ return blocksReplWorkMultiplier;
+ }
}
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1374605&r1=1374604&r2=1374605&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sat Aug 18 16:18:45 2012
@@ -182,6 +182,9 @@ public class FSNamesystem implements FSC
// Default initial capacity and load factor of map
public static final int DEFAULT_INITIAL_MAP_CAPACITY = 16;
public static final float DEFAULT_MAP_LOAD_FACTOR = 0.75f;
+
+ private float blocksInvalidateWorkPct;
+ private int blocksReplWorkMultiplier;
private boolean isPermissionEnabled;
private boolean persistBlocks;
@@ -500,6 +503,10 @@ public class FSNamesystem implements FSC
this.defaultPermission = PermissionStatus.createImmutable(
fsOwner.getShortUserName(), supergroup, new FsPermission(filePermission));
+ this.blocksInvalidateWorkPct =
+ DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+
this.replicator = BlockPlacementPolicy.getInstance(conf, this, clusterMap);
this.defaultReplication = conf.getInt("dfs.replication", 3);
this.maxReplication = conf.getInt("dfs.replication.max", 512);
@@ -2751,8 +2758,6 @@ public class FSNamesystem implements FSC
* Periodically calls computeReplicationWork().
*/
class ReplicationMonitor implements Runnable {
- static final int INVALIDATE_WORK_PCT_PER_ITERATION = 32;
- static final float REPLICATION_WORK_MULTIPLIER_PER_ITERATION = 2;
ReplicateQueueProcessingStats replicateQueueStats =
new ReplicateQueueProcessingStats();
InvalidateQueueProcessingStats invalidateQueueStats =
@@ -2859,9 +2864,9 @@ public class FSNamesystem implements FSC
}
synchronized(heartbeats) {
blocksToProcess = (int)(heartbeats.size()
- * ReplicationMonitor.REPLICATION_WORK_MULTIPLIER_PER_ITERATION);
+ * this.blocksReplWorkMultiplier);
nodesToProcess = (int)Math.ceil((double)heartbeats.size()
- * ReplicationMonitor.INVALIDATE_WORK_PCT_PER_ITERATION / 100);
+ * this.blocksInvalidateWorkPct);
}
replmon.replicateQueueStats.startCycle(blocksToProcess);
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java?rev=1374605&r1=1374604&r2=1374605&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java Sat Aug 18 16:18:45 2012
@@ -23,14 +23,16 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import junit.framework.TestCase;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetworkTopology;
-import org.apache.hadoop.net.Node;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.FSConstants;
-
-import junit.framework.TestCase;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.Node;
public class TestReplicationPolicy extends TestCase {
private static final int BLOCK_SIZE = 1024;
@@ -422,4 +424,48 @@ public class TestReplicationPolicy exten
assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0]));
}
+ /**
+ * This testcase tests whether the value returned by
+ * DFSUtil.getInvalidateWorkPctPerIteration() is positive
+ */
+ public void testGetInvalidateWorkPctPerIteration() {
+ Configuration conf = new Configuration();
+ float blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ assertTrue(blocksInvalidateWorkPct > 0);
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "0.0");
+ try {
+ blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ fail("Should throw IllegalArgumentException.");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "0.5f");
+ blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ assertEquals(blocksInvalidateWorkPct, 0.5f);
+ }
+
+ /**
+ * This testcase tests whether the value returned by
+ * DFSUtil.getReplWorkMultiplier() is positive
+ */
+ public void testGetReplWorkMultiplier() {
+ Configuration conf = new Configuration();
+ int blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+ assertTrue(blocksReplWorkMultiplier > 0);
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, "-1");
+ try {
+ blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+ fail("Should throw IllegalArgumentException.");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, "3");
+ blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+ assertEquals(blocksReplWorkMultiplier, 3);
+ }
+
}