You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by gi...@apache.org on 2018/11/28 19:33:50 UTC
hadoop git commit: HDFS-14102. Performance improvement in
BlockPlacementPolicyDefault. Contributed by Beluga Behr.
Repository: hadoop
Updated Branches:
refs/heads/trunk 4ca3a6b21 -> 300f560fc
HDFS-14102. Performance improvement in BlockPlacementPolicyDefault. Contributed by Beluga Behr.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/300f560f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/300f560f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/300f560f
Branch: refs/heads/trunk
Commit: 300f560fcca879d212eadffaa59fbbb7017c9a3f
Parents: 4ca3a6b
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Wed Nov 28 11:33:22 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Wed Nov 28 11:33:22 2018 -0800
----------------------------------------------------------------------
.../BlockPlacementPolicyDefault.java | 22 ++++++++++++--------
1 file changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/300f560f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d396845..5b8a8b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -71,6 +71,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
CHOOSE_RANDOM_REASONS = ThreadLocal
.withInitial(() -> new HashMap<NodeNotChosenReason, Integer>());
+ private static final BlockPlacementStatus ONE_RACK_PLACEMENT =
+ new BlockPlacementStatusDefault(1, 1, 1);
+
private enum NodeNotChosenReason {
NOT_IN_SERVICE("the node is not in service"),
NODE_STALE("the node is stale"),
@@ -1029,22 +1032,23 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
@Override
public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs,
int numberOfReplicas) {
- if (locs == null)
+ if (locs == null) {
locs = DatanodeDescriptor.EMPTY_ARRAY;
+ }
if (!clusterMap.hasClusterEverBeenMultiRack()) {
// only one rack
- return new BlockPlacementStatusDefault(1, 1, 1);
+ return ONE_RACK_PLACEMENT;
}
- int minRacks = 2;
- minRacks = Math.min(minRacks, numberOfReplicas);
+ final int minRacks = Math.min(2, numberOfReplicas);
// 1. Check that all locations are different.
// 2. Count locations on different racks.
- Set<String> racks = new TreeSet<>();
- for (DatanodeInfo dn : locs)
- racks.add(dn.getNetworkLocation());
- return new BlockPlacementStatusDefault(racks.size(), minRacks,
- clusterMap.getNumOfRacks());
+ final long rackCount = Arrays.asList(locs).stream()
+ .map(dn -> dn.getNetworkLocation()).distinct().count();
+
+ return new BlockPlacementStatusDefault(Math.toIntExact(rackCount),
+ minRacks, clusterMap.getNumOfRacks());
}
+
/**
* Decide whether deleting the specified replica of the block still makes
* the block conform to the configured block placement policy.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org