You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2017/05/22 00:54:23 UTC
[2/2] hadoop git commit: HDFS-10715. NPE when applying
AvailableSpaceBlockPlacementPolicy. Contributed by Guangbin Zhu.
HDFS-10715. NPE when applying AvailableSpaceBlockPlacementPolicy. Contributed by Guangbin Zhu.
(cherry picked from commit ef432579a7763cc0e482fe049027c6e5325eb034)
(cherry picked from commit 68638661271e6e69c220b19fec7cbc7ae5e6208e)
# Conflicts:
# hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a0e5e3b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a0e5e3b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a0e5e3b
Branch: refs/heads/branch-2.7
Commit: 8a0e5e3b70552fbb8f956d380c28c6a3cd05220a
Parents: 9440ec2
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Aug 5 17:31:11 2016 +0900
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Sun May 21 17:47:51 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../AvailableSpaceBlockPlacementPolicy.java | 16 ++++++++++------
.../TestAvailableSpaceBlockPlacementPolicy.java | 18 ++++++++++++++++++
3 files changed, 31 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a0e5e3b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7de65c8..5063cd3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -303,6 +303,9 @@ Release 2.7.4 - UNRELEASED
HDFS-8498. Blocks can be committed with wrong size. (Jing Zhao)
Backport HDFS-11732 by Zhe Zhang.
+ HDFS-10715. NPE when applying AvailableSpaceBlockPlacementPolicy.
+ (Guangbin Zhu)
+
Release 2.7.3 - 2016-08-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a0e5e3b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
index 74c1c78..43cd51d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
@@ -71,13 +71,17 @@ public class AvailableSpaceBlockPlacementPolicy extends
protected DatanodeDescriptor chooseDataNode(String scope) {
DatanodeDescriptor a = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
DatanodeDescriptor b = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
- int ret = compareDataNode(a, b);
- if (ret == 0) {
- return a;
- } else if (ret < 0) {
- return (RAND.nextInt(100) < balancedPreference) ? a : b;
+ if (a != null && b != null){
+ int ret = compareDataNode(a, b);
+ if (ret == 0) {
+ return a;
+ } else if (ret < 0) {
+ return (RAND.nextInt(100) < balancedPreference) ? a : b;
+ } else {
+ return (RAND.nextInt(100) < balancedPreference) ? b : a;
+ }
} else {
- return (RAND.nextInt(100) < balancedPreference) ? b : a;
+ return a == null ? b : a;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a0e5e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
index 59791b0..cc9f06d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.File;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -30,6 +32,7 @@ import org.apache.hadoop.hdfs.TestBlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.Node;
import org.apache.hadoop.test.PathUtils;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -158,6 +161,21 @@ public class TestAvailableSpaceBlockPlacementPolicy {
Assert.assertTrue(possibility < 0.55);
}
+ @Test
+ public void testChooseDataNode() {
+ try {
+ Collection<Node> allNodes = new ArrayList<>(dataNodes.length);
+ Collections.addAll(allNodes, dataNodes);
+ if (placementPolicy instanceof AvailableSpaceBlockPlacementPolicy){
+ // exclude all datanodes when chooseDataNode, no NPE should be thrown
+ ((AvailableSpaceBlockPlacementPolicy)placementPolicy)
+ .chooseDataNode("~");
+ }
+ }catch (NullPointerException npe){
+ Assert.fail("NPE should not be thrown");
+ }
+ }
+
@AfterClass
public static void teardownCluster() {
if (namenode != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org