You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2009/04/07 19:38:17 UTC
svn commit: r762864 - in /hadoop/core/trunk: CHANGES.txt
src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java
Author: hairong
Date: Tue Apr 7 17:38:17 2009
New Revision: 762864
URL: http://svn.apache.org/viewvc?rev=762864&view=rev
Log:
HADOOP-5603. Improve NameNode's block placement performance by reducing the number of iterations on datanodes. Contributed by Hairong Kuang.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=762864&r1=762863&r2=762864&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Apr 7 17:38:17 2009
@@ -214,6 +214,8 @@
HADOOP-5595. NameNode does not need to run a replicator to choose a
random DataNode. (hairong)
+ HADOOP-5603. Improve NameNode's block placement performance. (hairong)
+
BUG FIXES
HADOOP-5379. CBZip2InputStream to throw IOException on data crc error.
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java?rev=762864&r1=762863&r2=762864&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java Tue Apr 7 17:38:17 2009
@@ -308,18 +308,24 @@
int maxNodesPerRack,
List<DatanodeDescriptor> results)
throws NotEnoughReplicasException {
- DatanodeDescriptor result;
- do {
- DatanodeDescriptor[] selectedNodes =
- chooseRandom(1, nodes, excludedNodes);
- if (selectedNodes.length == 0) {
- throw new NotEnoughReplicasException(
- "Not able to place enough replicas");
- }
- result = (DatanodeDescriptor)(selectedNodes[0]);
- } while(!isGoodTarget(result, blocksize, maxNodesPerRack, results));
- results.add(result);
- return result;
+ int numOfAvailableNodes =
+ clusterMap.countNumOfAvailableNodes(nodes, excludedNodes);
+ while(numOfAvailableNodes > 0) {
+ DatanodeDescriptor choosenNode =
+ (DatanodeDescriptor)(clusterMap.chooseRandom(nodes));
+ if (!excludedNodes.contains(choosenNode)) {
+ excludedNodes.add(choosenNode);
+ numOfAvailableNodes--;
+
+ if (isGoodTarget(choosenNode, blocksize, maxNodesPerRack, results)) {
+ results.add(choosenNode);
+ return choosenNode;
+ }
+ }
+ }
+
+ throw new NotEnoughReplicasException(
+ "Not able to place enough replicas");
}
/* Randomly choose <i>numOfReplicas</i> targets from <i>nodes</i>.
@@ -331,51 +337,27 @@
int maxNodesPerRack,
List<DatanodeDescriptor> results)
throws NotEnoughReplicasException {
- boolean toContinue = true;
- do {
- DatanodeDescriptor[] selectedNodes =
- chooseRandom(numOfReplicas, nodes, excludedNodes);
- if (selectedNodes.length < numOfReplicas) {
- toContinue = false;
- }
- for(int i=0; i<selectedNodes.length; i++) {
- DatanodeDescriptor result = selectedNodes[i];
- if (isGoodTarget(result, blocksize, maxNodesPerRack, results)) {
- numOfReplicas--;
- results.add(result);
- }
- } // end of for
- } while (numOfReplicas>0 && toContinue);
- if (numOfReplicas>0) {
- throw new NotEnoughReplicasException(
- "Not able to place enough replicas");
- }
- }
-
- /* Randomly choose <i>numOfNodes</i> nodes from <i>scope</i>.
- * @return the choosen nodes
- */
- private DatanodeDescriptor[] chooseRandom(int numOfReplicas,
- String nodes,
- List<Node> excludedNodes) {
- List<DatanodeDescriptor> results =
- new ArrayList<DatanodeDescriptor>();
int numOfAvailableNodes =
clusterMap.countNumOfAvailableNodes(nodes, excludedNodes);
- numOfReplicas = (numOfAvailableNodes<numOfReplicas)?
- numOfAvailableNodes:numOfReplicas;
- while(numOfReplicas > 0) {
+ while(numOfReplicas > 0 && numOfAvailableNodes > 0) {
DatanodeDescriptor choosenNode =
(DatanodeDescriptor)(clusterMap.chooseRandom(nodes));
if (!excludedNodes.contains(choosenNode)) {
- results.add(choosenNode);
excludedNodes.add(choosenNode);
- numOfReplicas--;
+ numOfAvailableNodes--;
+
+ if (isGoodTarget(choosenNode, blocksize, maxNodesPerRack, results)) {
+ numOfReplicas--;
+ results.add(choosenNode);
+ }
}
}
- return (DatanodeDescriptor[])results.toArray(
- new DatanodeDescriptor[results.size()]);
+
+ if (numOfReplicas>0) {
+ throw new NotEnoughReplicasException(
+ "Not able to place enough replicas");
+ }
}
/* judge if a node is a good target.