You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/04/06 21:22:32 UTC
[02/50] [abbrv] hadoop git commit: HDFS-6945. BlockManager should
remove a block from excessReplicateMap and decrement ExcessBlocks metric when
the block is removed. (aajisaka)
HDFS-6945. BlockManager should remove a block from excessReplicateMap and decrement ExcessBlocks metric when the block is removed. (aajisaka)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71b43741
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71b43741
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71b43741
Branch: refs/heads/YARN-2928
Commit: 71b43741dde686725ac188ae91b14185989944a1
Parents: 40a1282
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Apr 1 09:07:28 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:09 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/blockmanagement/BlockManager.java | 22 ++++++++++++++++++--
.../namenode/metrics/TestNameNodeMetrics.java | 9 ++++++--
3 files changed, 30 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71b43741/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f3537b0..4247ea6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -398,6 +398,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7997. The first non-existing xattr should also throw IOException.
(zhouyingchao via yliu)
+ HDFS-6945. BlockManager should remove a block from excessReplicateMap and
+ decrement ExcessBlocks metric when the block is removed. (aajisaka)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71b43741/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index f6e15a3..d9aee62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3351,8 +3351,7 @@ public class BlockManager {
// file already removes them from the block map below.
block.setNumBytes(BlockCommand.NO_ACK);
addToInvalidates(block);
- corruptReplicas.removeFromCorruptReplicasMap(block);
- blocksMap.removeBlock(block);
+ removeBlockFromMap(block);
// Remove the block from pendingReplications and neededReplications
pendingReplications.remove(block);
neededReplications.remove(block, UnderReplicatedBlocks.LEVEL);
@@ -3528,11 +3527,30 @@ public class BlockManager {
}
public void removeBlockFromMap(Block block) {
+ removeFromExcessReplicateMap(block);
blocksMap.removeBlock(block);
// If block is removed from blocksMap remove it from corruptReplicasMap
corruptReplicas.removeFromCorruptReplicasMap(block);
}
+ /**
+ * If a block is removed from blocksMap, remove it from excessReplicateMap.
+ */
+ private void removeFromExcessReplicateMap(Block block) {
+ for (DatanodeStorageInfo info : blocksMap.getStorages(block)) {
+ String uuid = info.getDatanodeDescriptor().getDatanodeUuid();
+ LightWeightLinkedSet<Block> excessReplicas = excessReplicateMap.get(uuid);
+ if (excessReplicas != null) {
+ if (excessReplicas.remove(block)) {
+ excessBlocksCount.decrementAndGet();
+ if (excessReplicas.isEmpty()) {
+ excessReplicateMap.remove(uuid);
+ }
+ }
+ }
+ }
+ }
+
public int getCapacity() {
return blocksMap.getCapacity();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71b43741/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index 64ea1e4..b390391 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -280,11 +280,16 @@ public class TestNameNodeMetrics {
public void testExcessBlocks() throws Exception {
Path file = getTestPath("testExcessBlocks");
createFile(file, 100, (short)2);
- long totalBlocks = 1;
NameNodeAdapter.setReplication(namesystem, file.toString(), (short)1);
MetricsRecordBuilder rb = getMetrics(NS_METRICS);
- assertGauge("ExcessBlocks", totalBlocks, rb);
+ assertGauge("ExcessBlocks", 1L, rb);
+
+ // verify ExcessBlocks metric is decremented and
+ // excessReplicateMap is cleared after deleting a file
fs.delete(file, true);
+ rb = getMetrics(NS_METRICS);
+ assertGauge("ExcessBlocks", 0L, rb);
+ assertTrue(bm.excessReplicateMap.isEmpty());
}
/** Test to ensure metrics reflects missing blocks */