You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2017/08/28 23:22:59 UTC
hadoop git commit: HDFS-11998. Enable DFSNetworkTopology as default.
Contributed by Chen Liang.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 98ba68b05 -> e19f004f6
HDFS-11998. Enable DFSNetworkTopology as default. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e19f004f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e19f004f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e19f004f
Branch: refs/heads/branch-2
Commit: e19f004f653d0d3d985ec8e3215a611753416415
Parents: 98ba68b
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Aug 28 16:22:42 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Aug 28 16:22:42 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../AvailableSpaceBlockPlacementPolicy.java | 21 ++++++++++++++++++++
.../src/main/resources/hdfs-default.xml | 2 +-
.../TestReplicationPolicyWithNodeGroup.java | 4 ++++
4 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e19f004f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c5231d8..d5192c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -987,7 +987,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_USE_DFS_NETWORK_TOPOLOGY_KEY =
"dfs.use.dfs.network.topology";
- public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = false;
+ public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true;
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e19f004f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
index 706768c..8435b46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
@@ -24,10 +24,13 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_
import java.util.Collection;
import java.util.Random;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
@@ -71,11 +74,29 @@ public class AvailableSpaceBlockPlacementPolicy extends
@Override
protected DatanodeDescriptor chooseDataNode(final String scope,
+ final Collection<Node> excludedNode, StorageType type) {
+ // only the code that uses DFSNetworkTopology should trigger this code path.
+ Preconditions.checkArgument(clusterMap instanceof DFSNetworkTopology);
+ DFSNetworkTopology dfsClusterMap = (DFSNetworkTopology)clusterMap;
+ DatanodeDescriptor a = (DatanodeDescriptor) dfsClusterMap
+ .chooseRandomWithStorageType(scope, excludedNode, type);
+ DatanodeDescriptor b = (DatanodeDescriptor) dfsClusterMap
+ .chooseRandomWithStorageType(scope, excludedNode, type);
+ return select(a, b);
+ }
+
+ @Override
+ protected DatanodeDescriptor chooseDataNode(final String scope,
final Collection<Node> excludedNode) {
DatanodeDescriptor a =
(DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode);
DatanodeDescriptor b =
(DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode);
+ return select(a, b);
+ }
+
+ private DatanodeDescriptor select(
+ DatanodeDescriptor a, DatanodeDescriptor b) {
if (a != null && b != null){
int ret = compareDataNode(a, b);
if (ret == 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e19f004f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 6c383f3..4091c6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4286,7 +4286,7 @@
<property>
<name>dfs.use.dfs.network.topology</name>
- <value>false</value>
+ <value>true</value>
<description>
Enables DFSNetworkTopology to choose nodes for placing replicas.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e19f004f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
index e5ffd56..81ecdcd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -51,6 +52,9 @@ public class TestReplicationPolicyWithNodeGroup extends BaseReplicationPolicyTes
@Override
DatanodeDescriptor[] getDatanodeDescriptors(Configuration conf) {
+ // default is true, in this case this test will against DFSNetworkTopology
+ // but it run on NetworkTopologyWithNodeGroup, so set to false.
+ conf.setBoolean(DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false);
conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
NetworkTopologyWithNodeGroup.class.getName());
final String[] racks = {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org