You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2012/02/15 23:00:08 UTC
svn commit: r1244752 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt
src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
Author: suresh
Date: Wed Feb 15 22:00:08 2012
New Revision: 1244752
URL: http://svn.apache.org/viewvc?rev=1244752&view=rev
Log:
HDFS-2938. Recursive delete of a large directory make namenode unresponsive. Contributed by Hari Mankude.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1244752&r1=1244751&r2=1244752&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Feb 15 22:00:08 2012
@@ -251,6 +251,9 @@ Release 0.23.2 - UNRELEASED
HDFS-2950. Secondary NN HTTPS address should be listed as a
NAMESERVICE_SPECIFIC_KEY. (todd)
+ HDFS-2938. Recursive delete of a large directory make namenode
+ unresponsive. (Hari Mankude via suresh)
+
Release 0.23.1 - 2012-02-08
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1244752&r1=1244751&r2=1244752&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Feb 15 22:00:08 2012
@@ -1980,15 +1980,8 @@ public class FSNamesystem implements Nam
} finally {
writeUnlock();
}
-
- getEditLog().logSync();
-
- writeLock();
- try {
- removeBlocks(collectedBlocks); // Incremental deletion of blocks
- } finally {
- writeUnlock();
- }
+ getEditLog().logSync();
+ removeBlocks(collectedBlocks); // Incremental deletion of blocks
collectedBlocks.clear();
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* Namesystem.delete: "
@@ -1997,16 +1990,24 @@ public class FSNamesystem implements Nam
return true;
}
- /** From the given list, incrementally remove the blocks from blockManager */
+ /**
+ * From the given list, incrementally remove the blocks from blockManager
+ * Writelock is dropped and reacquired every BLOCK_DELETION_INCREMENT to
+ * ensure that other waiters on the lock can get in. See HDFS-2938
+ */
private void removeBlocks(List<Block> blocks) {
- assert hasWriteLock();
int start = 0;
int end = 0;
while (start < blocks.size()) {
end = BLOCK_DELETION_INCREMENT + start;
end = end > blocks.size() ? blocks.size() : end;
- for (int i=start; i<end; i++) {
- blockManager.removeBlock(blocks.get(i));
+ writeLock();
+ try {
+ for (int i = start; i < end; i++) {
+ blockManager.removeBlock(blocks.get(i));
+ }
+ } finally {
+ writeUnlock();
}
start = end;
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java?rev=1244752&r1=1244751&r2=1244752&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java Wed Feb 15 22:00:08 2012
@@ -117,8 +117,11 @@ public class TestLargeDirectoryDelete {
try {
int blockcount = getBlockCount();
if (blockcount < TOTAL_BLOCKS && blockcount > 0) {
- synchronized(mc.getNamesystem()) {
+ mc.getNamesystem().writeLock();
+ try {
lockOps++;
+ } finally {
+ mc.getNamesystem().writeUnlock();
}
Thread.sleep(1);
}