You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/12 14:11:22 UTC
[1/2] hadoop git commit: HDFS-9036. In
BlockPlacementPolicyWithNodeGroup#chooseLocalStorage ,
random node is selected eventhough fallbackToLocalRack is true. (Contributed
by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 185762167 -> 7545488b1
refs/heads/trunk d8455479b -> c71565038
HDFS-9036. In BlockPlacementPolicyWithNodeGroup#chooseLocalStorage , random node is selected eventhough fallbackToLocalRack is true. (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7156503
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7156503
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7156503
Branch: refs/heads/trunk
Commit: c7156503856e24faf844c5c647157b310d8b537f
Parents: d845547
Author: Vinayakumar B <vi...@apache.org>
Authored: Sat Sep 12 17:40:16 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Sat Sep 12 17:40:16 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++
.../BlockPlacementPolicyWithNodeGroup.java | 14 ++++++--------
.../TestReplicationPolicyWithNodeGroup.java | 19 +++++++++++++++++++
3 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7156503/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5a42499..0c891ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1315,6 +1315,10 @@ Release 2.8.0 - UNRELEASED
HDFS-8581. ContentSummary on / skips further counts on yielding lock
(J.Andreina via vinayakumarb)
+ HDFS-9036. In BlockPlacementPolicyWithNodeGroup#chooseLocalStorage , random
+ node is selected eventhough fallbackToLocalRack is true.
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7156503/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
index b1c4b78..89f47ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.net.NodeBase;
* for placing block replicas on environment with node-group layer.
* The replica placement strategy is adjusted to:
* If the writer is on a datanode, the 1st replica is placed on the local
- * node (or local node-group), otherwise a random datanode.
+ * node(or local node-group or on local rack), otherwise a random datanode.
* The 2nd replica is placed on a datanode that is on a different rack with 1st
* replica node.
* The 3rd replica is placed on a datanode which is on a different node-group
@@ -165,7 +165,7 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
/* choose one node from the nodegroup that <i>localMachine</i> is on.
* if no such node is available, choose one node from the nodegroup where
* a second replica is on.
- * if still no such node is available, choose a random node in the cluster.
+ * if still no such node is available, return null.
* @return the chosen node
*/
private DatanodeStorageInfo chooseLocalNodeGroup(
@@ -195,14 +195,12 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
excludedNodes, blocksize, maxNodesPerRack, results,
avoidStaleNodes, storageTypes);
} catch(NotEnoughReplicasException e2) {
- //otherwise randomly choose one from the network
- return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
- maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ //otherwise return null
+ return null;
}
} else {
- //otherwise randomly choose one from the network
- return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
- maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ //otherwise return null
+ return null;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7156503/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
index ed54aeb..e973925 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
@@ -487,6 +487,25 @@ public class TestReplicationPolicyWithNodeGroup {
}
/**
+ * In this testcase, client is dataNodes[7], but it is not qualified
+ * to be chosen. And there is no other node available on client Node group.
+ * So the 1st replica should be placed on client local rack dataNodes[6]
+ * @throws Exception
+ */
+ @Test
+ public void testChooseTargetForLocalStorage() throws Exception {
+ updateHeartbeatWithUsage(dataNodes[7],
+ 2* HdfsServerConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
+ (HdfsServerConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L,
+ 0L, 0L, 0, 0); // no space
+
+ DatanodeStorageInfo[] targets;
+ targets = chooseTarget(1, dataNodes[7]);
+ assertEquals(targets.length, 1);
+ assertTrue(targets[0].getDatanodeDescriptor().equals(dataNodes[6]));
+ }
+
+ /**
* This testcase tests re-replication, when dataNodes[0] is already chosen.
* So the 1st replica can be placed on random rack.
* the 2nd replica should be placed on different node and nodegroup by same rack as
[2/2] hadoop git commit: HDFS-9036. In
BlockPlacementPolicyWithNodeGroup#chooseLocalStorage ,
random node is selected eventhough fallbackToLocalRack is true. (Contributed
by J.Andreina)
Posted by vi...@apache.org.
HDFS-9036. In BlockPlacementPolicyWithNodeGroup#chooseLocalStorage , random node is selected eventhough fallbackToLocalRack is true. (Contributed by J.Andreina)
(cherry picked from commit c7156503856e24faf844c5c647157b310d8b537f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7545488b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7545488b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7545488b
Branch: refs/heads/branch-2
Commit: 7545488b12437609f0681e78527af138c9af4aeb
Parents: 1857621
Author: Vinayakumar B <vi...@apache.org>
Authored: Sat Sep 12 17:40:16 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Sat Sep 12 17:40:41 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++
.../BlockPlacementPolicyWithNodeGroup.java | 14 ++++++--------
.../TestReplicationPolicyWithNodeGroup.java | 19 +++++++++++++++++++
3 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7545488b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 17e882d..19dbbcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -968,6 +968,10 @@ Release 2.8.0 - UNRELEASED
HDFS-8581. ContentSummary on / skips further counts on yielding lock
(J.Andreina via vinayakumarb)
+ HDFS-9036. In BlockPlacementPolicyWithNodeGroup#chooseLocalStorage , random
+ node is selected eventhough fallbackToLocalRack is true.
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7545488b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
index 8ec1778..cc8b227 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.net.NodeBase;
* for placing block replicas on environment with node-group layer.
* The replica placement strategy is adjusted to:
* If the writer is on a datanode, the 1st replica is placed on the local
- * node (or local node-group), otherwise a random datanode.
+ * node(or local node-group or on local rack), otherwise a random datanode.
* The 2nd replica is placed on a datanode that is on a different rack with 1st
* replica node.
* The 3rd replica is placed on a datanode which is on a different node-group
@@ -167,7 +167,7 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
/* choose one node from the nodegroup that <i>localMachine</i> is on.
* if no such node is available, choose one node from the nodegroup where
* a second replica is on.
- * if still no such node is available, choose a random node in the cluster.
+ * if still no such node is available, return null.
* @return the chosen node
*/
private DatanodeStorageInfo chooseLocalNodeGroup(
@@ -197,14 +197,12 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
excludedNodes, blocksize, maxNodesPerRack, results,
avoidStaleNodes, storageTypes);
} catch(NotEnoughReplicasException e2) {
- //otherwise randomly choose one from the network
- return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
- maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ //otherwise return null
+ return null;
}
} else {
- //otherwise randomly choose one from the network
- return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
- maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ //otherwise return null
+ return null;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7545488b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
index ed54aeb..e973925 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
@@ -487,6 +487,25 @@ public class TestReplicationPolicyWithNodeGroup {
}
/**
+ * In this testcase, client is dataNodes[7], but it is not qualified
+ * to be chosen. And there is no other node available on client Node group.
+ * So the 1st replica should be placed on client local rack dataNodes[6]
+ * @throws Exception
+ */
+ @Test
+ public void testChooseTargetForLocalStorage() throws Exception {
+ updateHeartbeatWithUsage(dataNodes[7],
+ 2* HdfsServerConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
+ (HdfsServerConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L,
+ 0L, 0L, 0, 0); // no space
+
+ DatanodeStorageInfo[] targets;
+ targets = chooseTarget(1, dataNodes[7]);
+ assertEquals(targets.length, 1);
+ assertTrue(targets[0].getDatanodeDescriptor().equals(dataNodes[6]));
+ }
+
+ /**
* This testcase tests re-replication, when dataNodes[0] is already chosen.
* So the 1st replica can be placed on random rack.
* the 2nd replica should be placed on different node and nodegroup by same rack as