You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2013/07/05 05:20:33 UTC
svn commit: r1499904 - in /hadoop/common/branches/branch-1/src:
mapred/org/apache/hadoop/mapred/CleanupQueue.java
test/org/apache/hadoop/mapred/TestCleanupQueue.java
Author: acmurthy
Date: Fri Jul 5 03:20:32 2013
New Revision: 1499904
URL: http://svn.apache.org/r1499904
Log:
MAPREDUCE-5351. Addedndum patch to ensure we don't incorrectly close wrong filesystems. Contributed by Sandy Ryza.
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java?rev=1499904&r1=1499903&r2=1499904&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java Fri Jul 5 03:20:32 2013
@@ -112,8 +112,10 @@ public class CleanupQueue {
return null;
} finally {
// So that we don't leave an entry in the FileSystem cache for
- // every job.
- fs.close();
+ // every UGI that a job is submitted with.
+ if (ugi != null) {
+ fs.close();
+ }
}
}
});
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java?rev=1499904&r1=1499903&r2=1499904&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java Fri Jul 5 03:20:32 2013
@@ -33,20 +33,35 @@ public class TestCleanupQueue {
@Test (timeout = 2000)
public void testCleanupQueueClosesFilesystem() throws IOException,
InterruptedException {
+ Configuration conf = new Configuration();
File file = new File("afile.txt");
file.createNewFile();
Path path = new Path(file.getAbsoluteFile().toURI());
- FileSystem.get(new Configuration());
+ FileSystem.get(conf);
Assert.assertEquals(1, FileSystem.getCacheSize());
+ // With UGI, should close FileSystem
CleanupQueue cleanupQueue = new CleanupQueue();
- PathDeletionContext context = new PathDeletionContext(path,
- new Configuration(), UserGroupInformation.getLoginUser());
+ PathDeletionContext context = new PathDeletionContext(path, conf,
+ UserGroupInformation.getLoginUser());
cleanupQueue.addToQueue(context);
while (FileSystem.getCacheSize() > 0) {
Thread.sleep(100);
}
+
+ file.createNewFile();
+ FileSystem.get(conf);
+ Assert.assertEquals(1, FileSystem.getCacheSize());
+
+ // Without UGI, should not close FileSystem
+ context = new PathDeletionContext(path, conf);
+ cleanupQueue.addToQueue(context);
+
+ while (file.exists()) {
+ Thread.sleep(100);
+ }
+ Assert.assertEquals(1, FileSystem.getCacheSize());
}
}