You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zx...@apache.org on 2015/12/15 07:18:46 UTC
hadoop git commit: YARN-3857: Memory leak in ResourceManager with
SIMPLE mode. Contributed by mujunchao.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 843dac535 -> 70289432f
YARN-3857: Memory leak in ResourceManager with SIMPLE mode. Contributed by mujunchao.
(cherry picked from commit 3a76a010b85176f2bcb85ed6f74c25dcb8acfe4d)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/70289432
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/70289432
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/70289432
Branch: refs/heads/branch-2.6
Commit: 70289432f7aee2793ad75f1b018a1681b20be8ee
Parents: 843dac5
Author: Zhihai Xu <zx...@apache.org>
Authored: Tue Aug 18 10:36:40 2015 -0700
Committer: Zhihai Xu <zx...@apache.org>
Committed: Mon Dec 14 22:17:23 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../rmapp/attempt/RMAppAttemptImpl.java | 8 +++--
.../ClientToAMTokenSecretManagerInRM.java | 7 +++++
.../attempt/TestRMAppAttemptTransitions.java | 32 ++++++++++++++++++++
4 files changed, 46 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70289432/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 19fa0c2..7200e20 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -14,6 +14,8 @@ Release 2.6.4 - UNRELEASED
YARN-2975. FSLeafQueue app lists are accessed without required locks. (kasha)
+ YARN-3857. Memory leak in ResourceManager with SIMPLE mode. (mujunchao via zxu)
+
Release 2.6.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70289432/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 13e620a..a19f5ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -1262,9 +1262,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
// register the ClientTokenMasterKey after it is saved in the store,
// otherwise client may hold an invalid ClientToken after RM restarts.
- appAttempt.rmContext.getClientToAMTokenSecretManager()
- .registerApplication(appAttempt.getAppAttemptId(),
- appAttempt.getClientTokenMasterKey());
+ if (UserGroupInformation.isSecurityEnabled()) {
+ appAttempt.rmContext.getClientToAMTokenSecretManager()
+ .registerApplication(appAttempt.getAppAttemptId(),
+ appAttempt.getClientTokenMasterKey());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70289432/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java
index 4fbe2ce..4047bd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import javax.crypto.SecretKey;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager;
@@ -61,4 +62,10 @@ public class ClientToAMTokenSecretManagerInRM extends
ApplicationAttemptId applicationAttemptID) {
return this.masterKeys.get(applicationAttemptID);
}
+
+ @VisibleForTesting
+ public synchronized boolean hasMasterKey(
+ ApplicationAttemptId applicationAttemptID) {
+ return this.masterKeys.containsKey(applicationAttemptID);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70289432/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 8018f41..ccd2613 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -1363,6 +1363,38 @@ public class TestRMAppAttemptTransitions {
Assert.assertNull(token);
}
+ // this is to test master key is saved in the secret manager only after
+ // attempt is launched and in secure-mode
+ @Test
+ public void testApplicationAttemptMasterKey() throws Exception {
+ Container amContainer = allocateApplicationAttempt();
+ ApplicationAttemptId appid = applicationAttempt.getAppAttemptId();
+ boolean isMasterKeyExisted = false;
+
+ // before attempt is launched, can not get MasterKey
+ isMasterKeyExisted = clientToAMTokenManager.hasMasterKey(appid);
+ Assert.assertFalse(isMasterKeyExisted);
+
+ launchApplicationAttempt(amContainer);
+ // after attempt is launched and in secure mode, can get MasterKey
+ isMasterKeyExisted = clientToAMTokenManager.hasMasterKey(appid);
+ if (isSecurityEnabled) {
+ Assert.assertTrue(isMasterKeyExisted);
+ Assert.assertNotNull(clientToAMTokenManager.getMasterKey(appid));
+ } else {
+ Assert.assertFalse(isMasterKeyExisted);
+ }
+
+ applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
+ .getAppAttemptId(), RMAppAttemptEventType.KILL));
+ assertEquals(YarnApplicationAttemptState.LAUNCHED,
+ applicationAttempt.createApplicationAttemptState());
+ sendAttemptUpdateSavedEvent(applicationAttempt);
+ // after attempt is killed, can not get MasterKey
+ isMasterKeyExisted = clientToAMTokenManager.hasMasterKey(appid);
+ Assert.assertFalse(isMasterKeyExisted);
+ }
+
@Test
public void testFailedToFailed() {
// create a failed attempt.