You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yl...@apache.org on 2015/10/12 09:11:56 UTC
hadoop git commit: HDFS-8988. Use LightWeightHashSet instead of
LightWeightLinkedSet in BlockManager#excessReplicateMap. (yliu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0ff121610 -> 73b86a504
HDFS-8988. Use LightWeightHashSet instead of LightWeightLinkedSet in BlockManager#excessReplicateMap. (yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73b86a50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73b86a50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73b86a50
Branch: refs/heads/trunk
Commit: 73b86a5046fe3262dde7b05be46b18575e35fd5f
Parents: 0ff1216
Author: yliu <yl...@apache.org>
Authored: Mon Oct 12 14:40:14 2015 +0800
Committer: yliu <yl...@apache.org>
Committed: Mon Oct 12 14:40:14 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/blockmanagement/BlockManager.java | 23 ++++++++------------
.../hdfs/server/namenode/NamenodeFsck.java | 4 ++--
3 files changed, 14 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b86a50/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4171433..938546c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1512,6 +1512,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9221. HdfsServerConstants#ReplicaState#getState should avoid calling
values() since it creates a temporary array. (Staffan Friberg via yliu)
+ HDFS-8988. Use LightWeightHashSet instead of LightWeightLinkedSet in
+ BlockManager#excessReplicateMap. (yliu)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b86a50/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 18bfc41..8a64b74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -89,7 +89,6 @@ import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.util.LightWeightHashSet;
-import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import static org.apache.hadoop.hdfs.util.StripedBlockUtil.getInternalBlockLength;
@@ -219,7 +218,7 @@ public class BlockManager implements BlockStatsMXBean {
* Maps a StorageID to the set of blocks that are "extra" for this
* DataNode. We'll eventually remove these extras.
*/
- public final Map<String, LightWeightLinkedSet<BlockInfo>> excessReplicateMap =
+ public final Map<String, LightWeightHashSet<BlockInfo>> excessReplicateMap =
new HashMap<>();
/**
@@ -1421,11 +1420,6 @@ public class BlockManager implements BlockStatsMXBean {
*/
@VisibleForTesting
int computeRecoveryWorkForBlocks(List<List<BlockInfo>> blocksToRecover) {
- int requiredReplication, numEffectiveReplicas;
- List<DatanodeDescriptor> containingNodes;
- BlockCollection bc;
- int additionalReplRequired;
-
int scheduledWork = 0;
List<BlockRecoveryWork> recovWork = new LinkedList<>();
@@ -1786,7 +1780,7 @@ public class BlockManager implements BlockStatsMXBean {
Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(block);
for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
- LightWeightLinkedSet<BlockInfo> excessBlocks =
+ LightWeightHashSet<BlockInfo> excessBlocks =
excessReplicateMap.get(node.getDatanodeUuid());
int countableReplica = storage.getState() == State.NORMAL ? 1 : 0;
if ((nodesCorrupt != null) && (nodesCorrupt.contains(node)))
@@ -3090,7 +3084,7 @@ public class BlockManager implements BlockStatsMXBean {
postponeBlock(block);
return;
}
- LightWeightLinkedSet<BlockInfo> excessBlocks = excessReplicateMap.get(
+ LightWeightHashSet<BlockInfo> excessBlocks = excessReplicateMap.get(
cur.getDatanodeUuid());
if (excessBlocks == null || !excessBlocks.contains(block)) {
if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) {
@@ -3297,10 +3291,10 @@ public class BlockManager implements BlockStatsMXBean {
private void addToExcessReplicate(DatanodeInfo dn, BlockInfo storedBlock) {
assert namesystem.hasWriteLock();
- LightWeightLinkedSet<BlockInfo> excessBlocks = excessReplicateMap.get(
+ LightWeightHashSet<BlockInfo> excessBlocks = excessReplicateMap.get(
dn.getDatanodeUuid());
if (excessBlocks == null) {
- excessBlocks = new LightWeightLinkedSet<>();
+ excessBlocks = new LightWeightHashSet<>();
excessReplicateMap.put(dn.getDatanodeUuid(), excessBlocks);
}
if (excessBlocks.add(storedBlock)) {
@@ -3364,7 +3358,7 @@ public class BlockManager implements BlockStatsMXBean {
// We've removed a block from a node, so it's definitely no longer
// in "excess" there.
//
- LightWeightLinkedSet<BlockInfo> excessBlocks = excessReplicateMap.get(
+ LightWeightHashSet<BlockInfo> excessBlocks = excessReplicateMap.get(
node.getDatanodeUuid());
if (excessBlocks != null) {
if (excessBlocks.remove(storedBlock)) {
@@ -3581,7 +3575,7 @@ public class BlockManager implements BlockStatsMXBean {
} else if (node.isDecommissioned()) {
decommissioned++;
} else {
- LightWeightLinkedSet<BlockInfo> blocksExcess = excessReplicateMap.get(
+ LightWeightHashSet<BlockInfo> blocksExcess = excessReplicateMap.get(
node.getDatanodeUuid());
if (blocksExcess != null && blocksExcess.contains(b)) {
excess++;
@@ -3988,7 +3982,8 @@ public class BlockManager implements BlockStatsMXBean {
private void removeFromExcessReplicateMap(Block block) {
for (DatanodeStorageInfo info : blocksMap.getStorages(block)) {
String uuid = info.getDatanodeDescriptor().getDatanodeUuid();
- LightWeightLinkedSet<BlockInfo> excessReplicas = excessReplicateMap.get(uuid);
+ LightWeightHashSet<BlockInfo> excessReplicas =
+ excessReplicateMap.get(uuid);
if (excessReplicas != null) {
if (excessReplicas.remove(block)) {
excessBlocksCount.decrementAndGet();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b86a50/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 404a71e..2166639 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -74,7 +74,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
-import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
+import org.apache.hadoop.hdfs.util.LightWeightHashSet;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.NodeBase;
@@ -687,7 +687,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
.getStorageType()));
}
if (showReplicaDetails) {
- LightWeightLinkedSet<BlockInfo> blocksExcess =
+ LightWeightHashSet<BlockInfo> blocksExcess =
bm.excessReplicateMap.get(dnDesc.getDatanodeUuid());
Collection<DatanodeDescriptor> corruptReplicas =
bm.getCorruptReplicas(block.getLocalBlock());