You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/01/23 23:06:54 UTC
svn commit: r1235024 - in
/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/
Author: todd
Date: Mon Jan 23 22:06:53 2012
New Revision: 1235024
URL: http://svn.apache.org/viewvc?rev=1235024&view=rev
Log:
HDFS-2825. Add test hook to turn off the writer preferring its local DN. Contributed by Todd Lipcon.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1235024&r1=1235023&r2=1235024&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Jan 23 22:06:53 2012
@@ -110,6 +110,9 @@ Release 0.23.1 - UNRELEASED
for a client on the same node as the block file. (Andrew Purtell,
Suresh Srinivas and Jitendra Nath Pandey via szetszwo)
+ HDFS-2825. Add test hook to turn off the writer preferring its local
+ DN. (todd)
+
BUG FIXES
HDFS-2541. For a sufficiently large value of blocks, the DN Scanner
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1235024&r1=1235023&r2=1235024&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Mon Jan 23 22:06:53 2012
@@ -38,6 +38,8 @@ import org.apache.hadoop.net.NetworkTopo
import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
+import com.google.common.annotations.VisibleForTesting;
+
/** The class is responsible for choosing the desired number of targets
* for placing block replicas.
* The replica placement strategy is that if the writer is on a datanode,
@@ -49,6 +51,7 @@ import org.apache.hadoop.net.NodeBase;
@InterfaceAudience.Private
public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
private boolean considerLoad;
+ private boolean preferLocalNode = true;
private NetworkTopology clusterMap;
private FSClusterStats stats;
static final String enableDebugLogging = "For more information, please enable"
@@ -224,17 +227,17 @@ public class BlockPlacementPolicyDefault
if (localMachine == null)
return chooseRandom(NodeBase.ROOT, excludedNodes,
blocksize, maxNodesPerRack, results);
-
- // otherwise try local machine first
- Node oldNode = excludedNodes.put(localMachine, localMachine);
- if (oldNode == null) { // was not in the excluded list
- if (isGoodTarget(localMachine, blocksize,
- maxNodesPerRack, false, results)) {
- results.add(localMachine);
- return localMachine;
- }
- }
-
+ if (preferLocalNode) {
+ // otherwise try local machine first
+ Node oldNode = excludedNodes.put(localMachine, localMachine);
+ if (oldNode == null) { // was not in the excluded list
+ if (isGoodTarget(localMachine, blocksize,
+ maxNodesPerRack, false, results)) {
+ results.add(localMachine);
+ return localMachine;
+ }
+ }
+ }
// try a node on local rack
return chooseLocalRack(localMachine, excludedNodes,
blocksize, maxNodesPerRack, results);
@@ -569,5 +572,10 @@ public class BlockPlacementPolicyDefault
}
return cur;
}
+
+ @VisibleForTesting
+ void setPreferLocalNode(boolean prefer) {
+ this.preferLocalNode = prefer;
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java?rev=1235024&r1=1235023&r2=1235024&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java Mon Jan 23 22:06:53 2012
@@ -27,6 +27,8 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.util.Daemon;
+import com.google.common.base.Preconditions;
+
public class BlockManagerTestUtil {
public static void setNodeReplicationLimit(final BlockManager blockManager,
final int limit) {
@@ -122,4 +124,17 @@ public class BlockManagerTestUtil {
return blockManager.computeDatanodeWork();
}
+
+ /**
+ * Change whether the block placement policy will prefer the writer's
+ * local Datanode or not.
+ * @param prefer
+ */
+ public static void setWritingPrefersLocalNode(
+ BlockManager bm, boolean prefer) {
+ BlockPlacementPolicy bpp = bm.getBlockPlacementPolicy();
+ Preconditions.checkState(bpp instanceof BlockPlacementPolicyDefault,
+ "Must use default policy, got %s", bpp.getClass());
+ ((BlockPlacementPolicyDefault)bpp).setPreferLocalNode(prefer);
+ }
}