You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/09/20 22:21:34 UTC
[12/50] [abbrv] hadoop git commit: HDFS-13833. Improve
BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.
HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.
Signed-off-by: Xiao Chen <xi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27978bcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27978bcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27978bcb
Branch: refs/heads/HDDS-4
Commit: 27978bcb66a9130cbf26d37ec454c0b7fcdc2530
Parents: 3929653
Author: Shweta <sh...@cloudera.com>
Authored: Tue Sep 18 20:22:25 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 18 20:23:50 2018 -0700
----------------------------------------------------------------------
.../BlockPlacementPolicyDefault.java | 29 ++++++++++++++------
.../blockmanagement/TestReplicationPolicy.java | 28 +++++++++++++++++++
2 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d00f961..d396845 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -913,6 +913,24 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
/**
+ * Determine if a datanode should be chosen based on current workload.
+ *
+ * @param node The target datanode
+ * @return Return true if the datanode should be excluded, otherwise false
+ */
+ boolean excludeNodeByLoad(DatanodeDescriptor node){
+ final double maxLoad = considerLoadFactor *
+ stats.getInServiceXceiverAverage();
+ final int nodeLoad = node.getXceiverCount();
+ if ((nodeLoad > maxLoad) && (maxLoad > 0)) {
+ logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
+ "(load: " + nodeLoad + " > " + maxLoad + ")");
+ return true;
+ }
+ return false;
+ }
+
+ /**
* Determine if a datanode is good for placing block.
*
* @param node The target datanode
@@ -923,7 +941,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
* @param results A list containing currently chosen nodes. Used to check if
* too many nodes has been chosen in the target rack.
* @param avoidStaleNodes Whether or not to avoid choosing stale nodes
- * @return Reture true if the datanode is good candidate, otherwise false
+ * @return Return true if the datanode is good candidate, otherwise false
*/
boolean isGoodDatanode(DatanodeDescriptor node,
int maxTargetPerRack, boolean considerLoad,
@@ -943,13 +961,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
// check the communication traffic of the target machine
- if (considerLoad) {
- final double maxLoad = considerLoadFactor *
- stats.getInServiceXceiverAverage();
- final int nodeLoad = node.getXceiverCount();
- if (nodeLoad > maxLoad) {
- logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
- "(load: " + nodeLoad + " > " + maxLoad + ")");
+ if(considerLoad){
+ if(excludeNodeByLoad(node)){
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 27dcbf1..f08fa13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
@@ -1559,4 +1560,31 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
}
assertTrue(found);
}
+
+ @Test
+ public void testMaxLoad() {
+ FSClusterStats statistics = mock(FSClusterStats.class);
+ DatanodeDescriptor node = mock(DatanodeDescriptor.class);
+
+ when(statistics.getInServiceXceiverAverage()).thenReturn(0.0);
+ when(node.getXceiverCount()).thenReturn(1);
+
+ final Configuration conf = new Configuration();
+ final Class<? extends BlockPlacementPolicy> replicatorClass = conf
+ .getClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
+ BlockPlacementPolicy.class);
+ BlockPlacementPolicy bpp = ReflectionUtils.
+ newInstance(replicatorClass, conf);
+ assertTrue(bpp instanceof BlockPlacementPolicyDefault);
+
+ BlockPlacementPolicyDefault bppd = (BlockPlacementPolicyDefault) bpp;
+ bppd.initialize(conf, statistics, null, null);
+ assertFalse(bppd.excludeNodeByLoad(node));
+
+ when(statistics.getInServiceXceiverAverage()).thenReturn(1.0);
+ when(node.getXceiverCount()).thenReturn(10);
+ assertTrue(bppd.excludeNodeByLoad(node));
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org