You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/09/24 07:34:38 UTC
svn commit: r1525790 - in /hadoop/common/branches/branch-1-win:
CHANGES.branch-1-win.txt
src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
src/mapred/org/apache/hadoop/mapred/JobInProgress.java
src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
Author: cnauroth
Date: Tue Sep 24 05:34:38 2013
New Revision: 1525790
URL: http://svn.apache.org/r1525790
Log:
MAPREDUCE-5508. JobTracker memory leak caused by unreleased FileSystem objects in JobInProgress#cleanupJob. Contributed by Xi Fang.
Modified:
hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1525790&r1=1525789&r2=1525790&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Tue Sep 24 05:34:38 2013
@@ -501,3 +501,6 @@ Branch-hadoop-1-win (branched from branc
MAPREDUCE-5351. Fixed a memory leak in JobTracker due to stable FS objects in
FSCache. (Sandy Ryza via acmurthy)
+
+ MAPREDUCE-5508. JobTracker memory leak caused by unreleased FileSystem
+ objects in JobInProgress#cleanupJob. (Xi Fang via cnauroth)
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java?rev=1525790&r1=1525789&r2=1525790&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java Tue Sep 24 05:34:38 2013
@@ -60,14 +60,10 @@ public class CleanupQueue {
final Configuration conf;
final UserGroupInformation ugi;
final JobID jobIdTokenRenewalToCancel;
+ FileSystem fs;
public PathDeletionContext(Path fullPath, Configuration conf) {
- this(fullPath, conf, null, null);
- }
-
- public PathDeletionContext(Path fullPath, Configuration conf,
- UserGroupInformation ugi) {
- this(fullPath, conf, ugi, null);
+ this(fullPath, conf, null, null, null);
}
/**
@@ -86,11 +82,12 @@ public class CleanupQueue {
* <code>null</code>
*/
public PathDeletionContext(Path fullPath, Configuration conf,
- UserGroupInformation ugi, JobID jobIdTokenRenewalToCancel) {
+ UserGroupInformation ugi, JobID jobIdTokenRenewalToCancel, FileSystem fs) {
this.fullPath = fullPath;
this.conf = conf;
this.ugi = ugi;
this.jobIdTokenRenewalToCancel = jobIdTokenRenewalToCancel;
+ this.fs = fs;
}
protected Path getPathForCleanup() {
@@ -106,7 +103,7 @@ public class CleanupQueue {
(ugi == null ? UserGroupInformation.getLoginUser() : ugi).doAs(
new PrivilegedExceptionAction<Object>() {
public Object run() throws IOException {
- FileSystem fs = p.getFileSystem(conf);
+ fs = (fs == null ? p.getFileSystem(conf) : fs);
try {
fs.delete(p, true);
return null;
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1525790&r1=1525789&r2=1525790&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Tue Sep 24 05:34:38 2013
@@ -3325,10 +3325,17 @@ public class JobInProgress {
String jobTempDir = conf.get("mapreduce.job.dir");
if (jobTempDir != null && conf.getKeepTaskFilesPattern() == null &&
!conf.getKeepFailedTaskFiles()) {
- Path jobTempDirPath = new Path(jobTempDir);
- tempDirFs = jobTempDirPath.getFileSystem(conf);
+ final Path jobTempDirPath = new Path(jobTempDir);
+ try {
+ tempDirFs = userUGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ public FileSystem run() throws IOException {
+ return jobTempDirPath.getFileSystem(conf);
+ }});
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
CleanupQueue.getInstance().addToQueue(
- new PathDeletionContext(jobTempDirPath, conf, userUGI, jobId));
+ new PathDeletionContext(jobTempDirPath, conf, userUGI, jobId, tempDirFs));
}
} catch (IOException e) {
Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java?rev=1525790&r1=1525789&r2=1525790&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java (original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestCleanupQueue.java Tue Sep 24 05:34:38 2013
@@ -44,7 +44,7 @@ public class TestCleanupQueue {
// With UGI, should close FileSystem
CleanupQueue cleanupQueue = new CleanupQueue();
PathDeletionContext context = new PathDeletionContext(path, conf,
- UserGroupInformation.getLoginUser());
+ UserGroupInformation.getLoginUser(), null, null);
cleanupQueue.addToQueue(context);
while (FileSystem.getCacheSize() > 0) {