You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/12/11 08:43:57 UTC
[2/2] hadoop git commit: HDFS-7503. Namenode restart after large
deletions can cause slow processReport (Arpit Agarwal)
HDFS-7503. Namenode restart after large deletions can cause slow processReport (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91042c95
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91042c95
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91042c95
Branch: refs/heads/branch-2.6
Commit: 91042c950f146e46a7ec6acf7c710cfa53624c70
Parents: d93bfba
Author: arp <ar...@apache.org>
Authored: Wed Dec 10 23:37:26 2014 -0800
Committer: arp <ar...@apache.org>
Committed: Wed Dec 10 23:38:55 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/blockmanagement/BlockManager.java | 22 ++++++++++++++------
2 files changed, 19 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91042c95/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a542c66..e1ab3e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -21,6 +21,9 @@ Release 2.6.1 - UNRELEASED
HDFS-7489. Incorrect locking in FsVolumeList#checkDirs can hang datanodes
(Noah Lorang via Colin P. McCabe)
+ HDFS-7503. Namenode restart after large deletions can cause slow
+ processReport. (Arpit Agarwal)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91042c95/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index b97eeb2..37df223 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1776,6 +1776,8 @@ public class BlockManager {
final long startTime = Time.now(); //after acquiring write lock
final long endTime;
DatanodeDescriptor node;
+ Collection<Block> invalidatedBlocks = null;
+
try {
node = datanodeManager.getDatanode(nodeID);
if (node == null || !node.isAlive) {
@@ -1804,7 +1806,7 @@ public class BlockManager {
// ordinary block reports. This shortens restart times.
processFirstBlockReport(storageInfo, newReport);
} else {
- processReport(storageInfo, newReport);
+ invalidatedBlocks = processReport(storageInfo, newReport);
}
// Now that we have an up-to-date block report, we know that any
@@ -1823,6 +1825,14 @@ public class BlockManager {
namesystem.writeUnlock();
}
+ if (invalidatedBlocks != null) {
+ for (Block b : invalidatedBlocks) {
+ blockLog.info("BLOCK* processReport: " + b + " on " + node
+ + " size " + b.getNumBytes()
+ + " does not belong to any file");
+ }
+ }
+
// Log the block report processing stats from Namenode perspective
final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
if (metrics != null) {
@@ -1866,8 +1876,9 @@ public class BlockManager {
}
}
- private void processReport(final DatanodeStorageInfo storageInfo,
- final BlockListAsLongs report) throws IOException {
+ private Collection<Block> processReport(
+ final DatanodeStorageInfo storageInfo,
+ final BlockListAsLongs report) throws IOException {
// Normal case:
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
@@ -1898,14 +1909,13 @@ public class BlockManager {
+ " of " + numBlocksLogged + " reported.");
}
for (Block b : toInvalidate) {
- blockLog.info("BLOCK* processReport: "
- + b + " on " + node + " size " + b.getNumBytes()
- + " does not belong to any file");
addToInvalidates(b, node);
}
for (BlockToMarkCorrupt b : toCorrupt) {
markBlockAsCorrupt(b, storageInfo, node);
}
+
+ return toInvalidate;
}
/**