You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/10/23 06:34:50 UTC
svn commit: r1401158 - in /hadoop/common/branches/branch-1: CHANGES.txt
src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Author: suresh
Date: Tue Oct 23 04:34:49 2012
New Revision: 1401158
URL: http://svn.apache.org/viewvc?rev=1401158&view=rev
Log:
HDFS-3062. Print logs outside the namesystem lock invalidateWorkForOneNode and computeReplicationWorkForBlock. Contributed by Jing Zhao.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1401158&r1=1401157&r2=1401158&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Oct 23 04:34:49 2012
@@ -101,6 +101,9 @@ Release 1.2.0 - unreleased
from -1 to 0, i.e. enable the end of edit log check but do not tolerate
any corruption. (szetszwo)
+ HDFS-3062. Print logs outside the namesystem lock invalidateWorkForOneNode
+ and computeReplicationWorkForBlock. (Jing Zhao via suresh)
+
OPTIMIZATIONS
HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1401158&r1=1401157&r2=1401158&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Oct 23 04:34:49 2012
@@ -3286,23 +3286,24 @@ public class FSNamesystem implements FSC
neededReplications.remove(block, priority); // remove from neededReplications
replIndex--;
}
- if (NameNode.stateChangeLog.isInfoEnabled()) {
- StringBuffer targetList = new StringBuffer("datanode(s)");
- for (int k = 0; k < targets.length; k++) {
- targetList.append(' ');
- targetList.append(targets[k].getName());
- }
- NameNode.stateChangeLog.info(
- "BLOCK* ask "
- + srcNode.getName() + " to replicate "
- + block + " to " + targetList);
- NameNode.stateChangeLog.debug(
- "BLOCK* neededReplications = " + neededReplications.size()
- + " pendingReplications = " + pendingReplications.size());
- }
}
}
+ if (NameNode.stateChangeLog.isInfoEnabled()) {
+ StringBuilder targetList = new StringBuilder("datanode(s)");
+ for (int k = 0; k < targets.length; k++) {
+ targetList.append(' ');
+ targetList.append(targets[k].getName());
+ }
+ NameNode.stateChangeLog.info(
+ "BLOCK* ask "
+ + srcNode.getName() + " to replicate "
+ + block + " to " + targetList);
+ NameNode.stateChangeLog.debug(
+ "BLOCK* neededReplications = " + neededReplications.size()
+ + " pendingReplications = " + pendingReplications.size());
+ }
+
return true;
}
@@ -3396,42 +3397,46 @@ public class FSNamesystem implements FSC
*
* @return number of blocks scheduled for removal during this iteration.
*/
- private synchronized int invalidateWorkForOneNode(String nodeId) {
- // blocks should not be replicated or removed if safe mode is on
- if (isInSafeMode())
- return 0;
- // get blocks to invalidate for the nodeId
- assert nodeId != null;
- DatanodeDescriptor dn = datanodeMap.get(nodeId);
- if (dn == null) {
- recentInvalidateSets.remove(nodeId);
- return 0;
- }
- Collection<Block> invalidateSet = recentInvalidateSets.get(nodeId);
- if (invalidateSet == null) {
- return 0;
- }
-
+ private int invalidateWorkForOneNode(String nodeId) {
ArrayList<Block> blocksToInvalidate =
- new ArrayList<Block>(blockInvalidateLimit);
-
- // # blocks that can be sent in one message is limited
- Iterator<Block> it = invalidateSet.iterator();
- for(int blkCount = 0; blkCount < blockInvalidateLimit && it.hasNext();
- blkCount++) {
- blocksToInvalidate.add(it.next());
- it.remove();
- }
-
- // If we send everything in this message, remove this node entry
- if (!it.hasNext()) {
- recentInvalidateSets.remove(nodeId);
+ new ArrayList<Block>(blockInvalidateLimit);
+ DatanodeDescriptor dn = null;
+
+ synchronized (this) {
+ // blocks should not be replicated or removed if safe mode is on
+ if (isInSafeMode())
+ return 0;
+ // get blocks to invalidate for the nodeId
+ assert nodeId != null;
+ dn = datanodeMap.get(nodeId);
+ if (dn == null) {
+ recentInvalidateSets.remove(nodeId);
+ return 0;
+ }
+ Collection<Block> invalidateSet = recentInvalidateSets.get(nodeId);
+ if (invalidateSet == null) {
+ return 0;
+ }
+
+ // # blocks that can be sent in one message is limited
+ Iterator<Block> it = invalidateSet.iterator();
+ for(int blkCount = 0; blkCount < blockInvalidateLimit && it.hasNext();
+ blkCount++) {
+ blocksToInvalidate.add(it.next());
+ it.remove();
+ }
+
+ // If we send everything in this message, remove this node entry
+ if (!it.hasNext()) {
+ recentInvalidateSets.remove(nodeId);
+ }
+
+ dn.addBlocksToBeInvalidated(blocksToInvalidate);
+ pendingDeletionBlocksCount -= blocksToInvalidate.size();
}
- dn.addBlocksToBeInvalidated(blocksToInvalidate);
-
if(NameNode.stateChangeLog.isInfoEnabled()) {
- StringBuffer blockList = new StringBuffer();
+ StringBuilder blockList = new StringBuilder();
for(Block blk : blocksToInvalidate) {
blockList.append(' ');
blockList.append(blk);
@@ -3439,7 +3444,6 @@ public class FSNamesystem implements FSC
NameNode.stateChangeLog.info("BLOCK* ask "
+ dn.getName() + " to delete " + blockList);
}
- pendingDeletionBlocksCount -= blocksToInvalidate.size();
return blocksToInvalidate.size();
}