You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2011/02/17 08:03:01 UTC
svn commit: r1071518 - in /hadoop/hdfs/trunk: ./
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Author: szetszwo
Date: Thu Feb 17 07:03:00 2011
New Revision: 1071518
URL: http://svn.apache.org/viewvc?rev=1071518&view=rev
Log:
HDFS-1629. Add a method to BlockPlacementPolicy for keeping the chosen nodes in the output array.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1071518&r1=1071517&r2=1071518&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Thu Feb 17 07:03:00 2011
@@ -53,6 +53,9 @@ Trunk (unreleased changes)
HDFS-560 Enhancements/tuning to hadoop-hdfs/build.xml
+ HDFS-1629. Add a method to BlockPlacementPolicy for keeping the chosen
+ nodes in the output array. (szetszwo)
+
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java?rev=1071518&r1=1071517&r2=1071518&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java Thu Feb 17 07:03:00 2011
@@ -17,6 +17,11 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -24,7 +29,6 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.util.ReflectionUtils;
-import java.util.*;
/**
* This interface is used for choosing the desired number of targets
@@ -61,6 +65,21 @@ public abstract class BlockPlacementPoli
long blocksize);
/**
+ * Same as
+ * {{@link #chooseTarget(String, int, DatanodeDescriptor, List, boolean, HashMap, long)}
+ * with returnChosenNodes equal to false.
+ */
+ final DatanodeDescriptor[] chooseTarget(String srcPath,
+ int numOfReplicas,
+ DatanodeDescriptor writer,
+ List<DatanodeDescriptor> chosenNodes,
+ HashMap<Node, Node> excludedNodes,
+ long blocksize) {
+ return chooseTarget(srcPath, numOfReplicas, writer, chosenNodes, false,
+ excludedNodes, blocksize);
+ }
+
+ /**
* choose <i>numOfReplicas</i> data nodes for <i>writer</i>
* to re-replicate a block with size <i>blocksize</i>
* If not, return as many as we can.
@@ -69,7 +88,8 @@ public abstract class BlockPlacementPoli
* @param numOfReplicas additional number of replicas wanted.
* @param writer the writer's machine, null if not in the cluster.
* @param chosenNodes datanodes that have been chosen as targets.
- * @param excludedNodes: datanodes that should not be considered as targets.
+ * @param returnChosenNodes decide if the chosenNodes are returned.
+ * @param excludedNodes datanodes that should not be considered as targets.
* @param blocksize size of the data to be written.
* @return array of DatanodeDescriptor instances chosen as target
* and sorted as a pipeline.
@@ -78,6 +98,7 @@ public abstract class BlockPlacementPoli
int numOfReplicas,
DatanodeDescriptor writer,
List<DatanodeDescriptor> chosenNodes,
+ boolean returnChosenNodes,
HashMap<Node, Node> excludedNodes,
long blocksize);
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java?rev=1071518&r1=1071517&r2=1071518&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java Thu Feb 17 07:03:00 2011
@@ -17,9 +17,17 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import org.apache.commons.logging.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.FSConstants;
@@ -27,8 +35,6 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import java.util.*;
/** The class is responsible for choosing the desired number of targets
* for placing block replicas.
@@ -66,20 +72,23 @@ public class BlockPlacementPolicyDefault
DatanodeDescriptor writer,
List<DatanodeDescriptor> chosenNodes,
long blocksize) {
- return chooseTarget(numOfReplicas, writer, chosenNodes, null, blocksize);
+ return chooseTarget(numOfReplicas, writer, chosenNodes, false,
+ null, blocksize);
}
/** {@inheritDoc} */
- public DatanodeDescriptor[] chooseTarget(String srcPath,
+ @Override
+ DatanodeDescriptor[] chooseTarget(String srcPath,
int numOfReplicas,
DatanodeDescriptor writer,
List<DatanodeDescriptor> chosenNodes,
+ boolean returnChosenNodes,
HashMap<Node, Node> excludedNodes,
long blocksize) {
- return chooseTarget(numOfReplicas, writer, chosenNodes, excludedNodes, blocksize);
+ return chooseTarget(numOfReplicas, writer, chosenNodes, returnChosenNodes,
+ excludedNodes, blocksize);
}
-
/** {@inheritDoc} */
@Override
public DatanodeDescriptor[] chooseTarget(FSInodeInfo srcInode,
@@ -87,15 +96,15 @@ public class BlockPlacementPolicyDefault
DatanodeDescriptor writer,
List<DatanodeDescriptor> chosenNodes,
long blocksize) {
- return chooseTarget(numOfReplicas, writer, chosenNodes, null, blocksize);
+ return chooseTarget(numOfReplicas, writer, chosenNodes, false,
+ null, blocksize);
}
-
- /**
- * This is not part of the public API but is used by the unit tests.
- */
+
+ /** This is the implementation. */
DatanodeDescriptor[] chooseTarget(int numOfReplicas,
DatanodeDescriptor writer,
List<DatanodeDescriptor> chosenNodes,
+ boolean returnChosenNodes,
HashMap<Node, Node> excludedNodes,
long blocksize) {
if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) {
@@ -128,8 +137,9 @@ public class BlockPlacementPolicyDefault
DatanodeDescriptor localNode = chooseTarget(numOfReplicas, writer,
excludedNodes, blocksize, maxNodesPerRack, results);
-
- results.removeAll(chosenNodes);
+ if (!returnChosenNodes) {
+ results.removeAll(chosenNodes);
+ }
// sorting nodes to form a pipeline
return getPipeline((writer==null)?localNode:writer,
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java?rev=1071518&r1=1071517&r2=1071518&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java Thu Feb 17 07:03:00 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.na
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -129,6 +130,17 @@ public class TestReplicationPolicy exten
FSConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0);
}
+ private static DatanodeDescriptor[] chooseTarget(
+ BlockPlacementPolicyDefault policy,
+ int numOfReplicas,
+ DatanodeDescriptor writer,
+ List<DatanodeDescriptor> chosenNodes,
+ HashMap<Node, Node> excludedNodes,
+ long blocksize) {
+ return policy.chooseTarget(numOfReplicas, writer, chosenNodes, false,
+ excludedNodes, blocksize);
+ }
+
/**
* In this testcase, client is dataNodes[0], but the dataNodes[1] is
* not allowed to be chosen. So the 1st replica should be
@@ -145,23 +157,23 @@ public class TestReplicationPolicy exten
excludedNodes = new HashMap<Node, Node>();
excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = repl.chooseTarget(
- 0, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE);
+ targets = chooseTarget(repl, 0, dataNodes[0], chosenNodes, excludedNodes,
+ BLOCK_SIZE);
assertEquals(targets.length, 0);
excludedNodes.clear();
chosenNodes.clear();
excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = repl.chooseTarget(
- 1, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE);
+ targets = chooseTarget(repl, 1, dataNodes[0], chosenNodes, excludedNodes,
+ BLOCK_SIZE);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[0]);
excludedNodes.clear();
chosenNodes.clear();
excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = repl.chooseTarget(
- 2, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE);
+ targets = chooseTarget(repl, 2, dataNodes[0], chosenNodes, excludedNodes,
+ BLOCK_SIZE);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
@@ -169,8 +181,8 @@ public class TestReplicationPolicy exten
excludedNodes.clear();
chosenNodes.clear();
excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = repl.chooseTarget(
- 3, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE);
+ targets = chooseTarget(repl, 3, dataNodes[0], chosenNodes, excludedNodes,
+ BLOCK_SIZE);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
@@ -179,8 +191,8 @@ public class TestReplicationPolicy exten
excludedNodes.clear();
chosenNodes.clear();
excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = repl.chooseTarget(
- 4, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE);
+ targets = chooseTarget(repl, 4, dataNodes[0], chosenNodes, excludedNodes,
+ BLOCK_SIZE);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
for(int i=1; i<4; i++) {
@@ -189,6 +201,19 @@ public class TestReplicationPolicy exten
assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
cluster.isOnSameRack(targets[2], targets[3]));
assertFalse(cluster.isOnSameRack(targets[1], targets[3]));
+
+ excludedNodes.clear();
+ chosenNodes.clear();
+ excludedNodes.put(dataNodes[1], dataNodes[1]);
+ chosenNodes.add(dataNodes[2]);
+ targets = repl.chooseTarget(1, dataNodes[0], chosenNodes, true,
+ excludedNodes, BLOCK_SIZE);
+ System.out.println("targets=" + Arrays.asList(targets));
+ assertEquals(2, targets.length);
+ //make sure that the chosen node is in the target.
+ int i = 0;
+ for(; i < targets.length && !dataNodes[2].equals(targets[i]); i++);
+ assertTrue(i < targets.length);
}
/**