You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/04/10 06:25:10 UTC
[14/47] hadoop git commit: HDFS-8073. Split
BlockPlacementPolicyDefault.chooseTarget(..) so it can be easily overrided.
(Contributed by Walter Su)
HDFS-8073. Split BlockPlacementPolicyDefault.chooseTarget(..) so it can be easily overrided. (Contributed by Walter Su)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49373365
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49373365
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49373365
Branch: refs/heads/YARN-2928
Commit: 493733658ed1be073638afa13b9a67c5b4306d10
Parents: 41b7a26
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Apr 8 09:56:37 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Thu Apr 9 20:55:58 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../BlockPlacementPolicyDefault.java | 87 ++++++++++++--------
2 files changed, 54 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49373365/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f194bd7..ac508cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -385,6 +385,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8079. Move dfs.client.retry.* confs from DFSConfigKeys to
HdfsClientConfigKeys.Retry. (szetszwo)
+ HDFS-8073. Split BlockPlacementPolicyDefault.chooseTarget(..) so it
+ can be easily overrided. (Walter Su via vinayakumarb)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49373365/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 3262772..09db986 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -333,41 +333,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
+ " unavailableStorages=" + unavailableStorages
+ ", storagePolicy=" + storagePolicy);
}
-
- if (numOfResults == 0) {
- writer = chooseLocalStorage(writer, excludedNodes, blocksize,
- maxNodesPerRack, results, avoidStaleNodes, storageTypes, true)
- .getDatanodeDescriptor();
- if (--numOfReplicas == 0) {
- return writer;
- }
- }
- final DatanodeDescriptor dn0 = results.get(0).getDatanodeDescriptor();
- if (numOfResults <= 1) {
- chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
- if (--numOfReplicas == 0) {
- return writer;
- }
- }
- if (numOfResults <= 2) {
- final DatanodeDescriptor dn1 = results.get(1).getDatanodeDescriptor();
- if (clusterMap.isOnSameRack(dn0, dn1)) {
- chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
- } else if (newBlock){
- chooseLocalRack(dn1, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
- } else {
- chooseLocalRack(writer, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
- }
- if (--numOfReplicas == 0) {
- return writer;
- }
- }
- chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, blocksize,
- maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ writer = chooseTargetInOrder(numOfReplicas, writer, excludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, newBlock, storageTypes);
} catch (NotEnoughReplicasException e) {
final String message = "Failed to place enough replicas, still in need of "
+ (totalReplicasExpected - results.size()) + " to reach "
@@ -422,7 +389,55 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
return writer;
}
-
+
+ protected Node chooseTargetInOrder(int numOfReplicas,
+ Node writer,
+ final Set<Node> excludedNodes,
+ final long blocksize,
+ final int maxNodesPerRack,
+ final List<DatanodeStorageInfo> results,
+ final boolean avoidStaleNodes,
+ final boolean newBlock,
+ EnumMap<StorageType, Integer> storageTypes)
+ throws NotEnoughReplicasException {
+ final int numOfResults = results.size();
+ if (numOfResults == 0) {
+ writer = chooseLocalStorage(writer, excludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, storageTypes, true)
+ .getDatanodeDescriptor();
+ if (--numOfReplicas == 0) {
+ return writer;
+ }
+ }
+ final DatanodeDescriptor dn0 = results.get(0).getDatanodeDescriptor();
+ if (numOfResults <= 1) {
+ chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack,
+ results, avoidStaleNodes, storageTypes);
+ if (--numOfReplicas == 0) {
+ return writer;
+ }
+ }
+ if (numOfResults <= 2) {
+ final DatanodeDescriptor dn1 = results.get(1).getDatanodeDescriptor();
+ if (clusterMap.isOnSameRack(dn0, dn1)) {
+ chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack,
+ results, avoidStaleNodes, storageTypes);
+ } else if (newBlock){
+ chooseLocalRack(dn1, excludedNodes, blocksize, maxNodesPerRack,
+ results, avoidStaleNodes, storageTypes);
+ } else {
+ chooseLocalRack(writer, excludedNodes, blocksize, maxNodesPerRack,
+ results, avoidStaleNodes, storageTypes);
+ }
+ if (--numOfReplicas == 0) {
+ return writer;
+ }
+ }
+ chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ return writer;
+ }
+
/**
* Choose <i>localMachine</i> as the target.
* if <i>localMachine</i> is not available,