You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by gi...@apache.org on 2018/11/28 19:26:17 UTC
hadoop git commit: HDFS-14108. Performance improvement in
BlockManager Data Structures. Contributed by Beluga Behr.
Repository: hadoop
Updated Branches:
refs/heads/trunk 64a4b6b08 -> 4ca3a6b21
HDFS-14108. Performance improvement in BlockManager Data Structures. Contributed by Beluga Behr.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ca3a6b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ca3a6b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ca3a6b2
Branch: refs/heads/trunk
Commit: 4ca3a6b21a3a25acf16d026c699154047b1f686b
Parents: 64a4b6b
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Wed Nov 28 11:25:47 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Wed Nov 28 11:25:47 2018 -0800
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 30 ++++++++------------
1 file changed, 12 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ca3a6b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 36bbeb1..b326a75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -34,12 +34,10 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
-import java.util.TreeSet;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
@@ -844,7 +842,7 @@ public class BlockManager implements BlockStatsMXBean {
// source node returned is not used
chooseSourceDatanodes(getStoredBlock(block), containingNodes,
containingLiveReplicasNodes, numReplicas,
- new LinkedList<Byte>(), LowRedundancyBlocks.LEVEL);
+ new ArrayList<Byte>(), LowRedundancyBlocks.LEVEL);
// containingLiveReplicasNodes can include READ_ONLY_SHARED replicas which are
// not included in the numReplicas.liveReplicas() count
@@ -1842,7 +1840,7 @@ public class BlockManager implements BlockStatsMXBean {
int computeReconstructionWorkForBlocks(
List<List<BlockInfo>> blocksToReconstruct) {
int scheduledWork = 0;
- List<BlockReconstructionWork> reconWork = new LinkedList<>();
+ List<BlockReconstructionWork> reconWork = new ArrayList<>();
// Step 1: categorize at-risk blocks into replication and EC tasks
namesystem.writeLock();
@@ -1864,14 +1862,10 @@ public class BlockManager implements BlockStatsMXBean {
}
// Step 2: choose target nodes for each reconstruction task
- final Set<Node> excludedNodes = new HashSet<>();
- for(BlockReconstructionWork rw : reconWork){
+ for (BlockReconstructionWork rw : reconWork) {
// Exclude all of the containing nodes from being targets.
// This list includes decommissioning or corrupt nodes.
- excludedNodes.clear();
- for (DatanodeDescriptor dn : rw.getContainingNodes()) {
- excludedNodes.add(dn);
- }
+ final Set<Node> excludedNodes = new HashSet<>(rw.getContainingNodes());
// choose replication targets: NOT HOLDING THE GLOBAL LOCK
final BlockPlacementPolicy placementPolicy =
@@ -1882,9 +1876,9 @@ public class BlockManager implements BlockStatsMXBean {
// Step 3: add tasks to the DN
namesystem.writeLock();
try {
- for(BlockReconstructionWork rw : reconWork){
+ for (BlockReconstructionWork rw : reconWork) {
final DatanodeStorageInfo[] targets = rw.getTargets();
- if(targets == null || targets.length == 0){
+ if (targets == null || targets.length == 0) {
rw.resetTargets();
continue;
}
@@ -1901,7 +1895,7 @@ public class BlockManager implements BlockStatsMXBean {
if (blockLog.isDebugEnabled()) {
// log which blocks have been scheduled for reconstruction
- for(BlockReconstructionWork rw : reconWork){
+ for (BlockReconstructionWork rw : reconWork) {
DatanodeStorageInfo[] targets = rw.getTargets();
if (targets != null && targets.length != 0) {
StringBuilder targetList = new StringBuilder("datanode(s)");
@@ -2666,11 +2660,11 @@ public class BlockManager implements BlockStatsMXBean {
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
//
- Collection<BlockInfoToAdd> toAdd = new LinkedList<>();
- Collection<BlockInfo> toRemove = new TreeSet<>();
- Collection<Block> toInvalidate = new LinkedList<>();
- Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<>();
- Collection<StatefulBlockInfo> toUC = new LinkedList<>();
+ Collection<BlockInfoToAdd> toAdd = new ArrayList<>();
+ Collection<BlockInfo> toRemove = new HashSet<>();
+ Collection<Block> toInvalidate = new ArrayList<>();
+ Collection<BlockToMarkCorrupt> toCorrupt = new ArrayList<>();
+ Collection<StatefulBlockInfo> toUC = new ArrayList<>();
boolean sorted = false;
String strBlockReportId = "";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org