You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2017/05/22 00:54:22 UTC
[1/2] hadoop git commit: HDFS-8131. Implement a space balanced block
placement policy. Contributed by Liu Shaohui. (cherry picked from commit
de30d66b2673d0344346fb985e786247ca682317)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 321f730ca -> 8a0e5e3b7
HDFS-8131. Implement a space balanced block placement policy. Contributed by Liu Shaohui.
(cherry picked from commit de30d66b2673d0344346fb985e786247ca682317)
# Conflicts:
# hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9440ec27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9440ec27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9440ec27
Branch: refs/heads/branch-2.7
Commit: 9440ec2771fac429b5c73ade35f95aeafe7bd94a
Parents: 321f730
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue May 19 08:06:07 2015 -0500
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Sun May 21 17:19:54 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
.../AvailableSpaceBlockPlacementPolicy.java | 95 +++++++++++
.../BlockPlacementPolicyDefault.java | 11 +-
.../TestAvailableSpaceBlockPlacementPolicy.java | 167 +++++++++++++++++++
5 files changed, 278 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9440ec27/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c37d997..7de65c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -12,6 +12,9 @@ Release 2.7.4 - UNRELEASED
HDFS-9804. Allow long-running Balancer to login with keytab.
(Xiao Chen via Zhe Zhang)
+ HDFS-8131. Implement a space balanced block placement policy (Liu Shaohui
+ via kihwal)
+
IMPROVEMENTS
HDFS-8721. Add a metric for number of encryption zones.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9440ec27/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index fb41ee1..d5eb9e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -666,6 +666,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY = "dfs.web.authentication.kerberos.keytab";
public static final String DFS_NAMENODE_MAX_OP_SIZE_KEY = "dfs.namenode.max.op.size";
public static final int DFS_NAMENODE_MAX_OP_SIZE_DEFAULT = 50 * 1024 * 1024;
+ public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY =
+ "dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction";
+ public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT =
+ 0.6f;
public static final String DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_KEY =
"dfs.namenode.block-placement-policy.default.prefer-local-node";
public static final boolean DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_DEFAULT = true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9440ec27/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
new file mode 100644
index 0000000..74c1c78
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT;
+
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.net.NetworkTopology;
+
+/**
+ * Space balanced block placement policy.
+ */
+public class AvailableSpaceBlockPlacementPolicy extends
+ BlockPlacementPolicyDefault {
+ private static final Log LOG = LogFactory
+ .getLog(AvailableSpaceBlockPlacementPolicy.class);
+ private static final Random RAND = new Random();
+ private int balancedPreference =
+ (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
+
+ @Override
+ public void initialize(Configuration conf, FSClusterStats stats,
+ NetworkTopology clusterMap, Host2NodesMap host2datanodeMap) {
+ super.initialize(conf, stats, clusterMap, host2datanodeMap);
+ float balancedPreferencePercent =
+ conf.getFloat(
+ DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY,
+ DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
+
+ LOG.info("Available space block placement policy initialized: "
+ + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
+ + " = " + balancedPreferencePercent);
+
+ if (balancedPreferencePercent > 1.0) {
+ LOG.warn("The value of "
+ + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
+ + " is greater than 1.0 but should be in the range 0.0 - 1.0");
+ }
+ if (balancedPreferencePercent < 0.5) {
+ LOG.warn("The value of "
+ + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
+ + " is less than 0.5 so datanodes with more used percent will"
+ + " receive more block allocations.");
+ }
+ balancedPreference = (int) (100 * balancedPreferencePercent);
+ }
+
+ @Override
+ protected DatanodeDescriptor chooseDataNode(String scope) {
+ DatanodeDescriptor a = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
+ DatanodeDescriptor b = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
+ int ret = compareDataNode(a, b);
+ if (ret == 0) {
+ return a;
+ } else if (ret < 0) {
+ return (RAND.nextInt(100) < balancedPreference) ? a : b;
+ } else {
+ return (RAND.nextInt(100) < balancedPreference) ? b : a;
+ }
+ }
+
+ /**
+ * Compare the two data nodes.
+ */
+ protected int compareDataNode(final DatanodeDescriptor a,
+ final DatanodeDescriptor b) {
+ if (a.equals(b)
+ || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < 5) {
+ return 0;
+ }
+ return a.getDfsUsedPercent() < b.getDfsUsedPercent() ? -1 : 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9440ec27/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d39c68e..b7ec320 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -635,8 +635,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
boolean badTarget = false;
DatanodeStorageInfo firstChosen = null;
while(numOfReplicas > 0 && numOfAvailableNodes > 0) {
- DatanodeDescriptor chosenNode =
- (DatanodeDescriptor)clusterMap.chooseRandom(scope);
+ DatanodeDescriptor chosenNode = chooseDataNode(scope);
if (excludedNodes.add(chosenNode)) { //was not in the excluded list
if (LOG.isDebugEnabled()) {
builder.append("\nNode ").append(NodeBase.getPath(chosenNode)).append(" [");
@@ -706,6 +705,14 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
/**
+ * Choose a datanode from the given <i>scope</i>.
+ * @return the chosen node, if there is any.
+ */
+ protected DatanodeDescriptor chooseDataNode(final String scope) {
+ return (DatanodeDescriptor) clusterMap.chooseRandom(scope);
+ }
+
+ /**
* If the given storage is a good target, add it to the result list and
* update the set of excluded nodes.
* @return -1 if the given is not a good target;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9440ec27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
new file mode 100644
index 0000000..59791b0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.io.File;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.TestBlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.test.PathUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestAvailableSpaceBlockPlacementPolicy {
+ private final static int numRacks = 4;
+ private final static int nodesPerRack = 5;
+ private final static int blockSize = 1024;
+ private final static int chooseTimes = 10000;
+ private final static String file = "/tobers/test";
+ private final static int replica = 3;
+
+ private static DatanodeStorageInfo[] storages;
+ private static DatanodeDescriptor[] dataNodes;
+ private static Configuration conf;
+ private static NameNode namenode;
+ private static BlockPlacementPolicy placementPolicy;
+ private static NetworkTopology cluster;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ conf = new HdfsConfiguration();
+ conf.setFloat(
+ DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY,
+ 0.6f);
+ String[] racks = new String[numRacks];
+ for (int i = 0; i < numRacks; i++) {
+ racks[i] = "/rack" + i;
+ }
+
+ String[] owerRackOfNodes = new String[numRacks * nodesPerRack];
+ for (int i = 0; i < nodesPerRack; i++) {
+ for (int j = 0; j < numRacks; j++) {
+ owerRackOfNodes[i * numRacks + j] = racks[j];
+ }
+ }
+
+ storages = DFSTestUtil.createDatanodeStorageInfos(owerRackOfNodes);
+ dataNodes = DFSTestUtil.toDatanodeDescriptor(storages);
+
+ FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+ File baseDir = PathUtils.getTestDir(AvailableSpaceBlockPlacementPolicy.class);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, new File(baseDir, "name").getPath());
+ conf.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ AvailableSpaceBlockPlacementPolicy.class.getName());
+
+ DFSTestUtil.formatNameNode(conf);
+ namenode = new NameNode(conf);
+
+ final BlockManager bm = namenode.getNamesystem().getBlockManager();
+ placementPolicy = bm.getBlockPlacementPolicy();
+ cluster = bm.getDatanodeManager().getNetworkTopology();
+ for (int i = 0; i < nodesPerRack * numRacks; i++) {
+ cluster.add(dataNodes[i]);
+ }
+
+ setupDataNodeCapacity();
+ }
+
+ private static void updateHeartbeatWithUsage(DatanodeDescriptor dn,
+ long capacity, long dfsUsed, long remaining, long blockPoolUsed,
+ long dnCacheCapacity, long dnCacheUsed, int xceiverCount,
+ int volFailures) {
+ dn.getStorageInfos()[0].setUtilizationForTesting(
+ capacity, dfsUsed, remaining, blockPoolUsed);
+ dn.updateHeartbeat(
+ BlockManagerTestUtil.getStorageReportsForDatanode(dn),
+ dnCacheCapacity, dnCacheUsed, xceiverCount, volFailures, null);
+ }
+
+ private static void setupDataNodeCapacity() {
+ for (int i = 0; i < nodesPerRack * numRacks; i++) {
+ if ((i % 2) == 0) {
+ // remaining 100%
+ updateHeartbeatWithUsage(dataNodes[i], 2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+ 0L, 2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE * blockSize, 0L, 0L, 0L, 0, 0);
+ } else {
+ // remaining 50%
+ updateHeartbeatWithUsage(dataNodes[i], 2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+ HdfsConstants.MIN_BLOCKS_FOR_WRITE * blockSize, HdfsConstants.MIN_BLOCKS_FOR_WRITE
+ * blockSize, 0L, 0L, 0L, 0, 0);
+ }
+ }
+ }
+
+ /*
+ * To verify that the BlockPlacementPolicy can be replaced by AvailableSpaceBlockPlacementPolicy via
+ * changing the configuration.
+ */
+ @Test
+ public void testPolicyReplacement() {
+ Assert.assertTrue((placementPolicy instanceof AvailableSpaceBlockPlacementPolicy));
+ }
+
+ /*
+ * Call choose target many times and verify that nodes with more remaining percent will be chosen
+ * with high possibility.
+ */
+ @Test
+ public void testChooseTarget() {
+ int total = 0;
+ int moreRemainingNode = 0;
+ for (int i = 0; i < chooseTimes; i++) {
+ DatanodeStorageInfo[] targets =
+ namenode
+ .getNamesystem()
+ .getBlockManager()
+ .getBlockPlacementPolicy()
+ .chooseTarget(file, replica, null, new ArrayList<DatanodeStorageInfo>(), false, null,
+ blockSize, TestBlockStoragePolicy.DEFAULT_STORAGE_POLICY);
+
+ Assert.assertTrue(targets.length == replica);
+ for (int j = 0; j < replica; j++) {
+ total++;
+ if (targets[j].getDatanodeDescriptor().getRemainingPercent() > 60) {
+ moreRemainingNode++;
+ }
+ }
+ }
+ Assert.assertTrue(total == replica * chooseTimes);
+ double possibility = 1.0 * moreRemainingNode / total;
+ Assert.assertTrue(possibility > 0.52);
+ Assert.assertTrue(possibility < 0.55);
+ }
+
+ @AfterClass
+ public static void teardownCluster() {
+ if (namenode != null) {
+ namenode.stop();
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HDFS-10715. NPE when applying
AvailableSpaceBlockPlacementPolicy. Contributed by Guangbin Zhu.
Posted by sh...@apache.org.
HDFS-10715. NPE when applying AvailableSpaceBlockPlacementPolicy. Contributed by Guangbin Zhu.
(cherry picked from commit ef432579a7763cc0e482fe049027c6e5325eb034)
(cherry picked from commit 68638661271e6e69c220b19fec7cbc7ae5e6208e)
# Conflicts:
# hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a0e5e3b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a0e5e3b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a0e5e3b
Branch: refs/heads/branch-2.7
Commit: 8a0e5e3b70552fbb8f956d380c28c6a3cd05220a
Parents: 9440ec2
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Aug 5 17:31:11 2016 +0900
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Sun May 21 17:47:51 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../AvailableSpaceBlockPlacementPolicy.java | 16 ++++++++++------
.../TestAvailableSpaceBlockPlacementPolicy.java | 18 ++++++++++++++++++
3 files changed, 31 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a0e5e3b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7de65c8..5063cd3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -303,6 +303,9 @@ Release 2.7.4 - UNRELEASED
HDFS-8498. Blocks can be committed with wrong size. (Jing Zhao)
Backport HDFS-11732 by Zhe Zhang.
+ HDFS-10715. NPE when applying AvailableSpaceBlockPlacementPolicy.
+ (Guangbin Zhu)
+
Release 2.7.3 - 2016-08-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a0e5e3b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
index 74c1c78..43cd51d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
@@ -71,13 +71,17 @@ public class AvailableSpaceBlockPlacementPolicy extends
protected DatanodeDescriptor chooseDataNode(String scope) {
DatanodeDescriptor a = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
DatanodeDescriptor b = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
- int ret = compareDataNode(a, b);
- if (ret == 0) {
- return a;
- } else if (ret < 0) {
- return (RAND.nextInt(100) < balancedPreference) ? a : b;
+ if (a != null && b != null){
+ int ret = compareDataNode(a, b);
+ if (ret == 0) {
+ return a;
+ } else if (ret < 0) {
+ return (RAND.nextInt(100) < balancedPreference) ? a : b;
+ } else {
+ return (RAND.nextInt(100) < balancedPreference) ? b : a;
+ }
} else {
- return (RAND.nextInt(100) < balancedPreference) ? b : a;
+ return a == null ? b : a;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a0e5e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
index 59791b0..cc9f06d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.File;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -30,6 +32,7 @@ import org.apache.hadoop.hdfs.TestBlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.Node;
import org.apache.hadoop.test.PathUtils;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -158,6 +161,21 @@ public class TestAvailableSpaceBlockPlacementPolicy {
Assert.assertTrue(possibility < 0.55);
}
+ @Test
+ public void testChooseDataNode() {
+ try {
+ Collection<Node> allNodes = new ArrayList<>(dataNodes.length);
+ Collections.addAll(allNodes, dataNodes);
+ if (placementPolicy instanceof AvailableSpaceBlockPlacementPolicy){
+ // exclude all datanodes when chooseDataNode, no NPE should be thrown
+ ((AvailableSpaceBlockPlacementPolicy)placementPolicy)
+ .chooseDataNode("~");
+ }
+ }catch (NullPointerException npe){
+ Assert.fail("NPE should not be thrown");
+ }
+ }
+
@AfterClass
public static void teardownCluster() {
if (namenode != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org