You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2017/06/22 05:02:17 UTC
hadoop git commit: HDFS-11998. Enable DFSNetworkTopology as default.
Contributed by Chen Liang.
Repository: hadoop
Updated Branches:
refs/heads/trunk c22cf0044 -> 9ae9467f9
HDFS-11998. Enable DFSNetworkTopology as default. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ae9467f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ae9467f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ae9467f
Branch: refs/heads/trunk
Commit: 9ae9467f920e95ca989d7d51775b39e1b9fee300
Parents: c22cf00
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Jun 21 22:01:37 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Jun 21 22:01:41 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../AvailableSpaceBlockPlacementPolicy.java | 21 ++++++++++++++++++++
.../src/main/resources/hdfs-default.xml | 2 +-
.../TestReplicationPolicyWithNodeGroup.java | 4 ++++
4 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae9467f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e7aba4e..1f60f32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1085,7 +1085,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_USE_DFS_NETWORK_TOPOLOGY_KEY =
"dfs.use.dfs.network.topology";
- public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = false;
+ public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true;
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae9467f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
index 706768c..8435b46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
@@ -24,10 +24,13 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_
import java.util.Collection;
import java.util.Random;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
@@ -71,11 +74,29 @@ public class AvailableSpaceBlockPlacementPolicy extends
@Override
protected DatanodeDescriptor chooseDataNode(final String scope,
+ final Collection<Node> excludedNode, StorageType type) {
+ // only the code that uses DFSNetworkTopology should trigger this code path.
+ Preconditions.checkArgument(clusterMap instanceof DFSNetworkTopology);
+ DFSNetworkTopology dfsClusterMap = (DFSNetworkTopology)clusterMap;
+ DatanodeDescriptor a = (DatanodeDescriptor) dfsClusterMap
+ .chooseRandomWithStorageType(scope, excludedNode, type);
+ DatanodeDescriptor b = (DatanodeDescriptor) dfsClusterMap
+ .chooseRandomWithStorageType(scope, excludedNode, type);
+ return select(a, b);
+ }
+
+ @Override
+ protected DatanodeDescriptor chooseDataNode(final String scope,
final Collection<Node> excludedNode) {
DatanodeDescriptor a =
(DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode);
DatanodeDescriptor b =
(DatanodeDescriptor) clusterMap.chooseRandom(scope, excludedNode);
+ return select(a, b);
+ }
+
+ private DatanodeDescriptor select(
+ DatanodeDescriptor a, DatanodeDescriptor b) {
if (a != null && b != null){
int ret = compareDataNode(a, b);
if (ret == 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae9467f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0f72041..be345af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4534,7 +4534,7 @@
<property>
<name>dfs.use.dfs.network.topology</name>
- <value>false</value>
+ <value>true</value>
<description>
Enables DFSNetworkTopology to choose nodes for placing replicas.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae9467f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
index ebd4b81..6572f78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -51,6 +52,9 @@ public class TestReplicationPolicyWithNodeGroup extends BaseReplicationPolicyTes
@Override
DatanodeDescriptor[] getDatanodeDescriptors(Configuration conf) {
+ // default is true, in this case this test will against DFSNetworkTopology
+ // but it run on NetworkTopologyWithNodeGroup, so set to false.
+ conf.setBoolean(DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false);
conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
NetworkTopologyWithNodeGroup.class.getName());
final String[] racks = {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org