You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2015/03/23 18:56:48 UTC
[1/3] hadoop git commit: YARN-3336. FileSystem memory leak in
DelegationTokenRenewer.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 943d9ee60 -> 342c525ea
refs/heads/branch-2.7 6b9f2d9f3 -> 57e297208
refs/heads/trunk 7e6f384dd -> 6ca1f1202
YARN-3336. FileSystem memory leak in DelegationTokenRenewer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ca1f120
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ca1f120
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ca1f120
Branch: refs/heads/trunk
Commit: 6ca1f12024fd7cec7b01df0f039ca59f3f365dc1
Parents: 7e6f384
Author: cnauroth <cn...@apache.org>
Authored: Mon Mar 23 10:45:50 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Mon Mar 23 10:45:50 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../security/DelegationTokenRenewer.java | 13 +++++++--
.../security/TestDelegationTokenRenewer.java | 30 ++++++++++++++++++--
3 files changed, 41 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ca1f120/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e04624e..b90109c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -819,6 +819,9 @@ Release 2.7.0 - UNRELEASED
YARN-3384. TestLogAggregationService.verifyContainerLogs fails after
YARN-2777. (Naganarasimha G R via ozawa)
+ YARN-3336. FileSystem memory leak in DelegationTokenRenewer.
+ (Zhihai Xu via cnauroth)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ca1f120/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index cb456d8..2619971 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -605,6 +605,7 @@ public class DelegationTokenRenewer extends AbstractService {
rmContext.getSystemCredentialsForApps().put(applicationId, byteBuffer);
}
+ @VisibleForTesting
protected Token<?>[] obtainSystemTokensForUser(String user,
final Credentials credentials) throws IOException, InterruptedException {
// Get new hdfs tokens on behalf of this user
@@ -615,8 +616,16 @@ public class DelegationTokenRenewer extends AbstractService {
proxyUser.doAs(new PrivilegedExceptionAction<Token<?>[]>() {
@Override
public Token<?>[] run() throws Exception {
- return FileSystem.get(getConfig()).addDelegationTokens(
- UserGroupInformation.getLoginUser().getUserName(), credentials);
+ FileSystem fs = FileSystem.get(getConfig());
+ try {
+ return fs.addDelegationTokens(
+ UserGroupInformation.getLoginUser().getUserName(),
+ credentials);
+ } finally {
+ // Close the FileSystem created by the new proxy user,
+ // So that we don't leave an entry in the FileSystem cache
+ fs.close();
+ }
}
});
return newTokens;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ca1f120/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index 5d31404..99a506a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -287,9 +287,16 @@ public class TestDelegationTokenRenewer {
* exception
*/
static class MyFS extends DistributedFileSystem {
-
- public MyFS() {}
- public void close() {}
+ private static AtomicInteger instanceCounter = new AtomicInteger();
+ public MyFS() {
+ instanceCounter.incrementAndGet();
+ }
+ public void close() {
+ instanceCounter.decrementAndGet();
+ }
+ public static int getInstanceCounter() {
+ return instanceCounter.get();
+ }
@Override
public void initialize(URI uri, Configuration conf) throws IOException {}
@@ -299,6 +306,11 @@ public class TestDelegationTokenRenewer {
LOG.info("Called MYDFS.getdelegationtoken " + result);
return result;
}
+
+ public Token<?>[] addDelegationTokens(
+ final String renewer, Credentials credentials) throws IOException {
+ return new Token<?>[0];
+ }
}
/**
@@ -1022,4 +1034,16 @@ public class TestDelegationTokenRenewer {
// app2 completes, app1 is still running, check the token is not cancelled
Assert.assertFalse(Renewer.cancelled);
}
+
+ // Test FileSystem memory leak in obtainSystemTokensForUser.
+ @Test
+ public void testFSLeakInObtainSystemTokensForUser() throws Exception{
+ Credentials credentials = new Credentials();
+ String user = "test";
+ int oldCounter = MyFS.getInstanceCounter();
+ delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
+ delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
+ delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
+ Assert.assertEquals(oldCounter, MyFS.getInstanceCounter());
+ }
}
[2/3] hadoop git commit: YARN-3336. FileSystem memory leak in
DelegationTokenRenewer.
Posted by cn...@apache.org.
YARN-3336. FileSystem memory leak in DelegationTokenRenewer.
(cherry picked from commit 6ca1f12024fd7cec7b01df0f039ca59f3f365dc1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/342c525e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/342c525e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/342c525e
Branch: refs/heads/branch-2
Commit: 342c525eaa0749175f0e3827d245642776d043a5
Parents: 943d9ee
Author: cnauroth <cn...@apache.org>
Authored: Mon Mar 23 10:45:50 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Mon Mar 23 10:46:06 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../security/DelegationTokenRenewer.java | 13 +++++++--
.../security/TestDelegationTokenRenewer.java | 30 ++++++++++++++++++--
3 files changed, 41 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/342c525e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1b3ed2c..f5b04d3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -774,6 +774,9 @@ Release 2.7.0 - UNRELEASED
YARN-3384. TestLogAggregationService.verifyContainerLogs fails after
YARN-2777. (Naganarasimha G R via ozawa)
+ YARN-3336. FileSystem memory leak in DelegationTokenRenewer.
+ (Zhihai Xu via cnauroth)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/342c525e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index cb456d8..2619971 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -605,6 +605,7 @@ public class DelegationTokenRenewer extends AbstractService {
rmContext.getSystemCredentialsForApps().put(applicationId, byteBuffer);
}
+ @VisibleForTesting
protected Token<?>[] obtainSystemTokensForUser(String user,
final Credentials credentials) throws IOException, InterruptedException {
// Get new hdfs tokens on behalf of this user
@@ -615,8 +616,16 @@ public class DelegationTokenRenewer extends AbstractService {
proxyUser.doAs(new PrivilegedExceptionAction<Token<?>[]>() {
@Override
public Token<?>[] run() throws Exception {
- return FileSystem.get(getConfig()).addDelegationTokens(
- UserGroupInformation.getLoginUser().getUserName(), credentials);
+ FileSystem fs = FileSystem.get(getConfig());
+ try {
+ return fs.addDelegationTokens(
+ UserGroupInformation.getLoginUser().getUserName(),
+ credentials);
+ } finally {
+ // Close the FileSystem created by the new proxy user,
+ // So that we don't leave an entry in the FileSystem cache
+ fs.close();
+ }
}
});
return newTokens;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/342c525e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index 5d31404..99a506a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -287,9 +287,16 @@ public class TestDelegationTokenRenewer {
* exception
*/
static class MyFS extends DistributedFileSystem {
-
- public MyFS() {}
- public void close() {}
+ private static AtomicInteger instanceCounter = new AtomicInteger();
+ public MyFS() {
+ instanceCounter.incrementAndGet();
+ }
+ public void close() {
+ instanceCounter.decrementAndGet();
+ }
+ public static int getInstanceCounter() {
+ return instanceCounter.get();
+ }
@Override
public void initialize(URI uri, Configuration conf) throws IOException {}
@@ -299,6 +306,11 @@ public class TestDelegationTokenRenewer {
LOG.info("Called MYDFS.getdelegationtoken " + result);
return result;
}
+
+ public Token<?>[] addDelegationTokens(
+ final String renewer, Credentials credentials) throws IOException {
+ return new Token<?>[0];
+ }
}
/**
@@ -1022,4 +1034,16 @@ public class TestDelegationTokenRenewer {
// app2 completes, app1 is still running, check the token is not cancelled
Assert.assertFalse(Renewer.cancelled);
}
+
+ // Test FileSystem memory leak in obtainSystemTokensForUser.
+ @Test
+ public void testFSLeakInObtainSystemTokensForUser() throws Exception{
+ Credentials credentials = new Credentials();
+ String user = "test";
+ int oldCounter = MyFS.getInstanceCounter();
+ delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
+ delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
+ delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
+ Assert.assertEquals(oldCounter, MyFS.getInstanceCounter());
+ }
}
[3/3] hadoop git commit: YARN-3336. FileSystem memory leak in
DelegationTokenRenewer.
Posted by cn...@apache.org.
YARN-3336. FileSystem memory leak in DelegationTokenRenewer.
(cherry picked from commit 6ca1f12024fd7cec7b01df0f039ca59f3f365dc1)
(cherry picked from commit 342c525eaa0749175f0e3827d245642776d043a5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57e29720
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57e29720
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57e29720
Branch: refs/heads/branch-2.7
Commit: 57e297208d1b6f5b12b1d9251dc0081b4b603ed0
Parents: 6b9f2d9
Author: cnauroth <cn...@apache.org>
Authored: Mon Mar 23 10:45:50 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Mon Mar 23 10:46:22 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../security/DelegationTokenRenewer.java | 13 +++++++--
.../security/TestDelegationTokenRenewer.java | 30 ++++++++++++++++++--
3 files changed, 41 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57e29720/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ef816fc..1cb95ae 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -732,6 +732,9 @@ Release 2.7.0 - UNRELEASED
YARN-3384. TestLogAggregationService.verifyContainerLogs fails after
YARN-2777. (Naganarasimha G R via ozawa)
+ YARN-3336. FileSystem memory leak in DelegationTokenRenewer.
+ (Zhihai Xu via cnauroth)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57e29720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index cb456d8..2619971 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -605,6 +605,7 @@ public class DelegationTokenRenewer extends AbstractService {
rmContext.getSystemCredentialsForApps().put(applicationId, byteBuffer);
}
+ @VisibleForTesting
protected Token<?>[] obtainSystemTokensForUser(String user,
final Credentials credentials) throws IOException, InterruptedException {
// Get new hdfs tokens on behalf of this user
@@ -615,8 +616,16 @@ public class DelegationTokenRenewer extends AbstractService {
proxyUser.doAs(new PrivilegedExceptionAction<Token<?>[]>() {
@Override
public Token<?>[] run() throws Exception {
- return FileSystem.get(getConfig()).addDelegationTokens(
- UserGroupInformation.getLoginUser().getUserName(), credentials);
+ FileSystem fs = FileSystem.get(getConfig());
+ try {
+ return fs.addDelegationTokens(
+ UserGroupInformation.getLoginUser().getUserName(),
+ credentials);
+ } finally {
+ // Close the FileSystem created by the new proxy user,
+ // So that we don't leave an entry in the FileSystem cache
+ fs.close();
+ }
}
});
return newTokens;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57e29720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index 5d31404..99a506a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -287,9 +287,16 @@ public class TestDelegationTokenRenewer {
* exception
*/
static class MyFS extends DistributedFileSystem {
-
- public MyFS() {}
- public void close() {}
+ private static AtomicInteger instanceCounter = new AtomicInteger();
+ public MyFS() {
+ instanceCounter.incrementAndGet();
+ }
+ public void close() {
+ instanceCounter.decrementAndGet();
+ }
+ public static int getInstanceCounter() {
+ return instanceCounter.get();
+ }
@Override
public void initialize(URI uri, Configuration conf) throws IOException {}
@@ -299,6 +306,11 @@ public class TestDelegationTokenRenewer {
LOG.info("Called MYDFS.getdelegationtoken " + result);
return result;
}
+
+ public Token<?>[] addDelegationTokens(
+ final String renewer, Credentials credentials) throws IOException {
+ return new Token<?>[0];
+ }
}
/**
@@ -1022,4 +1034,16 @@ public class TestDelegationTokenRenewer {
// app2 completes, app1 is still running, check the token is not cancelled
Assert.assertFalse(Renewer.cancelled);
}
+
+ // Test FileSystem memory leak in obtainSystemTokensForUser.
+ @Test
+ public void testFSLeakInObtainSystemTokensForUser() throws Exception{
+ Credentials credentials = new Credentials();
+ String user = "test";
+ int oldCounter = MyFS.getInstanceCounter();
+ delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
+ delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
+ delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
+ Assert.assertEquals(oldCounter, MyFS.getInstanceCounter());
+ }
}