You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/09/05 02:04:11 UTC
svn commit: r1380940 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
CHANGES.txt src/main/java/
src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
Author: szetszwo
Date: Wed Sep 5 00:04:11 2012
New Revision: 1380940
URL: http://svn.apache.org/viewvc?rev=1380940&view=rev
Log:
svn merge -c 1380939 from trunk for HDFS-3888. Clean up BlockPlacementPolicyDefault.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1380939
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1380940&r1=1380939&r2=1380940&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Sep 5 00:04:11 2012
@@ -242,6 +242,8 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3887. Remove redundant chooseTarget methods in BlockPlacementPolicy.
(Jing Zhao via szetszwo)
+ HDFS-3888. Clean up BlockPlacementPolicyDefault. (Jing Zhao via szetszwo)
+
OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1380939
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1380940&r1=1380939&r2=1380940&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Wed Sep 5 00:04:11 2012
@@ -27,8 +27,6 @@ import java.util.List;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -55,9 +53,6 @@ import com.google.common.annotations.Vis
@InterfaceAudience.Private
public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
- private static final Log LOG =
- LogFactory.getLog(BlockPlacementPolicyDefault.class.getName());
-
private static final String enableDebugLogging =
"For more information, please enable DEBUG log level on "
+ LOG.getClass().getName();
@@ -124,7 +119,6 @@ public class BlockPlacementPolicyDefault
excludedNodes, blocksize);
}
-
/** This is the implementation. */
DatanodeDescriptor[] chooseTarget(int numOfReplicas,
DatanodeDescriptor writer,
@@ -161,7 +155,8 @@ public class BlockPlacementPolicyDefault
}
DatanodeDescriptor localNode = chooseTarget(numOfReplicas, writer,
- excludedNodes, blocksize, maxNodesPerRack, results);
+ excludedNodes, blocksize,
+ maxNodesPerRack, results);
if (!returnChosenNodes) {
results.removeAll(chosenNodes);
}
@@ -438,14 +433,29 @@ public class BlockPlacementPolicyDefault
* does not have too much load, and the rack does not have too many nodes
*/
private boolean isGoodTarget(DatanodeDescriptor node,
- long blockSize, int maxTargetPerLoc,
+ long blockSize, int maxTargetPerRack,
List<DatanodeDescriptor> results) {
- return isGoodTarget(node, blockSize, maxTargetPerLoc,
+ return isGoodTarget(node, blockSize, maxTargetPerRack,
this.considerLoad, results);
}
-
+
+ /**
+ * Determine if a node is a good target.
+ *
+ * @param node The target node
+ * @param blockSize Size of block
+ * @param maxTargetPerRack Maximum number of targets per rack. The value of
+ * this parameter depends on the number of racks in
+ * the cluster and total number of replicas for a block
+ * @param considerLoad whether or not to consider load of the target node
+ * @param results A list containing currently chosen nodes. Used to check if
+ * too many nodes has been chosen in the target rack.
+ * @return Return true if <i>node</i> has enough space,
+ * does not have too much load,
+ * and the rack does not have too many nodes.
+ */
private boolean isGoodTarget(DatanodeDescriptor node,
- long blockSize, int maxTargetPerLoc,
+ long blockSize, int maxTargetPerRack,
boolean considerLoad,
List<DatanodeDescriptor> results) {
// check if the node is (being) decommissed
@@ -497,7 +507,7 @@ public class BlockPlacementPolicyDefault
counter++;
}
}
- if (counter>maxTargetPerLoc) {
+ if (counter>maxTargetPerRack) {
if(LOG.isDebugEnabled()) {
threadLocalBuilder.get().append(node.toString()).append(": ")
.append("Node ").append(NodeBase.getPath(node))