You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/12/11 08:52:53 UTC
hadoop git commit: HDFS-7503. Namenode restart after large deletions
can cause slow processReport. (Arpit Agarwal)
Repository: hadoop
Updated Branches:
refs/heads/branch-1 2dff54ebc -> a8eee1fd0
HDFS-7503. Namenode restart after large deletions can cause slow processReport. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8eee1fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8eee1fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8eee1fd
Branch: refs/heads/branch-1
Commit: a8eee1fd01ddad153388e269b0d1381433864a82
Parents: 2dff54e
Author: arp <ar...@apache.org>
Authored: Wed Dec 10 23:51:45 2014 -0800
Committer: arp <ar...@apache.org>
Committed: Wed Dec 10 23:51:45 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 12 +++++-------
.../apache/hadoop/hdfs/server/namenode/NameNode.java | 12 +++++++++++-
3 files changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8eee1fd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0bc2773..59bef6c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -256,6 +256,9 @@ Release 1.3.0 - unreleased
MAPREDUCE-6170. TestUlimit failure on JDK8. (bc Wong via kasha)
+ HDFS-7503. Namenode restart after large deletions can cause slow
+ processReport (Arpit Agarwal)
+
Release 1.2.2 - unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8eee1fd/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 8a95a75..f9b9d7d 100644
--- a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3934,9 +3934,9 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
* The given node is reporting all its blocks. Use this info to
* update the (machine-->blocklist) and (block-->machinelist) tables.
*/
- public synchronized void processReport(DatanodeID nodeID,
- BlockListAsLongs newReport
- ) throws IOException {
+ public synchronized Collection<Block> processReport(
+ DatanodeID nodeID,
+ BlockListAsLongs newReport) throws IOException {
long startTime = now();
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("BLOCK* processReport: "
@@ -3961,7 +3961,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
NameNode.stateChangeLog.info("BLOCK* processReport: "
+ "discarded non-initial block report from " + nodeID.getName()
+ " because namenode still in startup phase");
- return;
+ return null;
}
//
@@ -3980,9 +3980,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
addStoredBlock(b, node, null);
}
for (Block b : toInvalidate) {
- NameNode.stateChangeLog.info("BLOCK* processReport: "
- + b + " on " + node.getName() + " size " + b.getNumBytes()
- + " does not belong to any file");
addToInvalidates(b, node);
}
long endTime = now();
@@ -3991,6 +3988,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
+ nodeID.getName() + ", blocks: " + newReport.getNumberOfBlocks()
+ ", processing time: " + (endTime - startTime) + " msecs");
node.processedBlockReport();
+ return toInvalidate;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8eee1fd/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 268d2e3..be0089c 100644
--- a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -1130,7 +1130,17 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
stateChangeLog.debug("*BLOCK* NameNode.blockReport: "
+"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks");
- namesystem.processReport(nodeReg, blist);
+ Collection<Block> blocksInvalidated =
+ namesystem.processReport(nodeReg, blist);
+
+ final String node = nodeReg.toString();
+ if (blocksInvalidated != null) {
+ for (Block b : blocksInvalidated) {
+ stateChangeLog.info("BLOCK* processReport: "
+ + b + " on " + node + " size " + b.getNumBytes()
+ + " does not belong to any file");
+ }
+ }
if (getFSImage().isUpgradeFinalized())
return DatanodeCommand.FINALIZE;
return null;