You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ji...@apache.org on 2014/08/09 00:21:43 UTC
svn commit: r1616897 [1/2] - in
/hadoop/common/branches/HDFS-6584/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/proto/ hadoop-...
Author: jing9
Date: Fri Aug 8 22:21:35 2014
New Revision: 1616897
URL: http://svn.apache.org/r1616897
Log:
Merging r1616428 through r1616893 from trunk.
Added:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java
- copied unchanged from r1616893, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
- copied unchanged from r1616893, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt Fri Aug 8 22:21:35 2014
@@ -91,6 +91,15 @@ Release 2.6.0 - UNRELEASED
YARN-2298. Move TimelineClient to yarn-common project (Zhijie Shen via
junping_du)
+ YARN-2288. Made persisted data in LevelDB timeline store be versioned. (Junping Du
+ via zjshen)
+
+ YARN-2352. FairScheduler: Collect metrics on duration of critical methods that
+ affect performance. (kasha)
+
+ YARN-2212. ApplicationMaster needs to find a way to update the AMRMToken
+ periodically. (xgong)
+
OPTIMIZATIONS
BUG FIXES
@@ -142,6 +151,9 @@ Release 2.6.0 - UNRELEASED
YARN-2388. Fixed TestTimelineWebServices failure due to HADOOP-10791. (zjshen)
+ YARN-2008. Fixed CapacityScheduler to calculate headroom based on max available
+ capacity instead of configured max capacity. (Craig Welch via jianhe)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Fri Aug 8 22:21:35 2014
@@ -200,6 +200,13 @@
<Field name="updateInterval" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!-- Inconsistent sync warning - callDurationMetrics is only initialized once and never changed -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+ <Field name="fsOpDurations" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+
<!-- Inconsistent sync warning - numRetries is only initialized once and never changed -->
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore" />
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java Fri Aug 8 22:21:35 2014
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.util.Records;
/**
@@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.util.Recor
* <li>A list of nodes whose status has been updated.</li>
* <li>The number of available nodes in a cluster.</li>
* <li>A description of resources requested back by the cluster</li>
+ * <li>AMRMToken, if AMRMToken has been rolled over</li>
* </ul>
* </p>
*
@@ -102,6 +104,23 @@ public abstract class AllocateResponse {
return response;
}
+ @Private
+ @Unstable
+ public static AllocateResponse newInstance(int responseId,
+ List<ContainerStatus> completedContainers,
+ List<Container> allocatedContainers, List<NodeReport> updatedNodes,
+ Resource availResources, AMCommand command, int numClusterNodes,
+ PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken,
+ List<ContainerResourceIncrease> increasedContainers,
+ List<ContainerResourceDecrease> decreasedContainers) {
+ AllocateResponse response =
+ newInstance(responseId, completedContainers, allocatedContainers,
+ updatedNodes, availResources, command, numClusterNodes, preempt,
+ nmTokens, increasedContainers, decreasedContainers);
+ response.setAMRMToken(amRMToken);
+ return response;
+ }
+
/**
* If the <code>ResourceManager</code> needs the
* <code>ApplicationMaster</code> to take some action then it will send an
@@ -270,4 +289,17 @@ public abstract class AllocateResponse {
@Unstable
public abstract void setDecreasedContainers(
List<ContainerResourceDecrease> decreasedContainers);
+
+ /**
+ * The AMRMToken that belong to this attempt
+ *
+ * @return The AMRMToken that belong to this attempt
+ */
+ @Public
+ @Unstable
+ public abstract Token getAMRMToken();
+
+ @Private
+ @Unstable
+ public abstract void setAMRMToken(Token amRMToken);
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Fri Aug 8 22:21:35 2014
@@ -85,6 +85,7 @@ message AllocateResponseProto {
repeated NMTokenProto nm_tokens = 9;
repeated ContainerResourceIncreaseProto increased_containers = 10;
repeated ContainerResourceDecreaseProto decreased_containers = 11;
+ optional hadoop.common.TokenProto am_rm_token = 12;
}
//////////////////////////////////////////////////////
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java Fri Aug 8 22:21:35 2014
@@ -39,7 +39,9 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -56,6 +58,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
@@ -64,6 +67,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.RackResolver;
import com.google.common.annotations.VisibleForTesting;
@@ -300,6 +304,9 @@ public class AMRMClientImpl<T extends Co
if (!allocateResponse.getNMTokens().isEmpty()) {
populateNMTokens(allocateResponse.getNMTokens());
}
+ if (allocateResponse.getAMRMToken() != null) {
+ updateAMRMToken(allocateResponse.getAMRMToken());
+ }
if (!pendingRelease.isEmpty()
&& !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
removePendingReleaseRequests(allocateResponse
@@ -743,4 +750,16 @@ public class AMRMClientImpl<T extends Co
"blacklistRemovals in updateBlacklist.");
}
}
+
+ private void updateAMRMToken(Token token) throws IOException {
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
+ new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
+ .getIdentifier().array(), token.getPassword().array(), new Text(
+ token.getKind()), new Text(token.getService()));
+ UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
+ if (UserGroupInformation.isSecurityEnabled()) {
+ currentUGI = UserGroupInformation.getLoginUser();
+ }
+ currentUGI.addToken(amrmToken);
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Fri Aug 8 22:21:35 2014
@@ -27,19 +27,23 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -71,9 +75,12 @@ import org.apache.hadoop.yarn.client.api
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
@@ -93,6 +100,9 @@ public class TestAMRMClient {
static ApplicationAttemptId attemptId = null;
static int nodeCount = 3;
+ static final int rolling_interval_sec = 13;
+ static final long am_expire_ms = 4000;
+
static Resource capability;
static Priority priority;
static Priority priority2;
@@ -106,6 +116,10 @@ public class TestAMRMClient {
public static void setup() throws Exception {
// start minicluster
conf = new YarnConfiguration();
+ conf.setLong(
+ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
+ rolling_interval_sec);
+ conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
@@ -809,4 +823,123 @@ public class TestAMRMClient {
}
}
+ @Test(timeout = 60000)
+ public void testAMRMClientOnAMRMTokenRollOver() throws YarnException,
+ IOException {
+ AMRMClient<ContainerRequest> amClient = null;
+ try {
+ AMRMTokenSecretManager amrmTokenSecretManager =
+ yarnCluster.getResourceManager().getRMContext()
+ .getAMRMTokenSecretManager();
+
+ // start am rm client
+ amClient = AMRMClient.<ContainerRequest> createAMRMClient();
+
+ amClient.init(conf);
+ amClient.start();
+
+ Long startTime = System.currentTimeMillis();
+ amClient.registerApplicationMaster("Host", 10000, "");
+
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_1 =
+ getAMRMToken();
+ Assert.assertNotNull(amrmToken_1);
+ Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(),
+ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
+
+ // Wait for enough time and make sure the roll_over happens
+ // At mean time, the old AMRMToken should continue to work
+ while (System.currentTimeMillis() - startTime <
+ rolling_interval_sec * 1000) {
+ amClient.allocate(0.1f);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ amClient.allocate(0.1f);
+
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_2 =
+ getAMRMToken();
+ Assert.assertNotNull(amrmToken_2);
+ Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(),
+ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
+
+ Assert.assertNotEquals(amrmToken_1, amrmToken_2);
+
+ // can do the allocate call with latest AMRMToken
+ amClient.allocate(0.1f);
+
+ // Make sure previous token has been rolled-over
+ // and can not use this rolled-over token to make a allocate all.
+ while (true) {
+ if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager
+ .getCurrnetMasterKeyData().getMasterKey().getKeyId()) {
+ if (amrmTokenSecretManager.getNextMasterKeyData() == null) {
+ break;
+ } else if (amrmToken_2.decodeIdentifier().getKeyId() !=
+ amrmTokenSecretManager.getNextMasterKeyData().getMasterKey()
+ .getKeyId()) {
+ break;
+ }
+ }
+ amClient.allocate(0.1f);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // DO NOTHING
+ }
+ }
+
+ try {
+ UserGroupInformation testUser =
+ UserGroupInformation.createRemoteUser("testUser");
+ SecurityUtil.setTokenService(amrmToken_2, yarnCluster
+ .getResourceManager().getApplicationMasterService().getBindAddress());
+ testUser.addToken(amrmToken_2);
+ testUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
+ @Override
+ public ApplicationMasterProtocol run() {
+ return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(
+ ApplicationMasterProtocol.class,
+ yarnCluster.getResourceManager().getApplicationMasterService()
+ .getBindAddress(), conf);
+ }
+ }).allocate(Records.newRecord(AllocateRequest.class));
+ Assert.fail("The old Token should not work");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof InvalidToken);
+ Assert.assertTrue(ex.getMessage().contains(
+ "Invalid AMRMToken from "
+ + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
+ }
+
+ amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
+ null, null);
+
+ } finally {
+ if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
+ amClient.stop();
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>
+ getAMRMToken() throws IOException {
+ Credentials credentials =
+ UserGroupInformation.getCurrentUser().getCredentials();
+ Iterator<org.apache.hadoop.security.token.Token<?>> iter =
+ credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ org.apache.hadoop.security.token.Token<?> token = iter.next();
+ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
+ return (org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>)
+ token;
+ }
+ }
+ return null;
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java Fri Aug 8 22:21:35 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client.api.impl;
import java.io.IOException;
+import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -26,7 +27,11 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -42,10 +47,12 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
+import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -58,12 +65,16 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
+import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
public class TestAMRMClientOnRMRestart {
static Configuration conf = null;
+ static final int rolling_interval_sec = 13;
+ static final long am_expire_ms = 4000;
@BeforeClass
public static void setup() throws Exception {
@@ -362,6 +373,134 @@ public class TestAMRMClientOnRMRestart {
}
+
+ // Test verify for AM issued with rolled-over AMRMToken
+ // is still able to communicate with restarted RM.
+ @Test(timeout = 30000)
+ public void testAMRMClientOnAMRMTokenRollOverOnRMRestart() throws Exception {
+ conf.setLong(
+ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
+ rolling_interval_sec);
+ conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+
+ // start first RM
+ MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore);
+ rm1.start();
+ DrainDispatcher dispatcher =
+ (DrainDispatcher) rm1.getRMContext().getDispatcher();
+ Long startTime = System.currentTimeMillis();
+ // Submit the application
+ RMApp app = rm1.submitApp(1024);
+ dispatcher.await();
+
+ MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+ nm1.nodeHeartbeat(true); // Node heartbeat
+ dispatcher.await();
+
+ ApplicationAttemptId appAttemptId =
+ app.getCurrentAppAttempt().getAppAttemptId();
+ rm1.sendAMLaunched(appAttemptId);
+ dispatcher.await();
+
+ AMRMTokenSecretManager amrmTokenSecretManagerForRM1 =
+ rm1.getRMContext().getAMRMTokenSecretManager();
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token =
+ amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId);
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ ugi.addTokenIdentifier(token.decodeIdentifier());
+
+ AMRMClient<ContainerRequest> amClient = new MyAMRMClientImpl(rm1);
+ amClient.init(conf);
+ amClient.start();
+
+ amClient.registerApplicationMaster("h1", 10000, "");
+ amClient.allocate(0.1f);
+
+ // Wait for enough time and make sure the roll_over happens
+ // At mean time, the old AMRMToken should continue to work
+ while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
+ amClient.allocate(0.1f);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // DO NOTHING
+ }
+ }
+ Assert.assertTrue(amrmTokenSecretManagerForRM1.getMasterKey()
+ .getMasterKey().getKeyId() != token.decodeIdentifier().getKeyId());
+
+ amClient.allocate(0.1f);
+
+ // active the nextMasterKey, and replace the currentMasterKey
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newToken =
+ amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId);
+ int waitCount = 0;
+ while (waitCount++ <= 50) {
+ if (amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData().getMasterKey()
+ .getKeyId() != token.decodeIdentifier().getKeyId()) {
+ break;
+ }
+ try {
+ amClient.allocate(0.1f);
+ } catch (Exception ex) {
+ break;
+ }
+ Thread.sleep(500);
+ }
+ Assert
+ .assertTrue(amrmTokenSecretManagerForRM1.getNextMasterKeyData() == null);
+ Assert.assertTrue(amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData()
+ .getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId());
+
+ // start 2nd RM
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:9030");
+ final MyResourceManager2 rm2 = new MyResourceManager2(conf, memStore);
+ rm2.start();
+ nm1.setResourceTrackerService(rm2.getResourceTrackerService());
+ ((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
+ dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
+
+ AMRMTokenSecretManager amrmTokenSecretManagerForRM2 =
+ rm2.getRMContext().getAMRMTokenSecretManager();
+ Assert.assertTrue(amrmTokenSecretManagerForRM2.getCurrnetMasterKeyData()
+ .getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId());
+ Assert
+ .assertTrue(amrmTokenSecretManagerForRM2.getNextMasterKeyData() == null);
+
+ try {
+ UserGroupInformation testUser =
+ UserGroupInformation.createRemoteUser("testUser");
+ SecurityUtil.setTokenService(token, rm2.getApplicationMasterService()
+ .getBindAddress());
+ testUser.addToken(token);
+ testUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
+ @Override
+ public ApplicationMasterProtocol run() {
+ return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(
+ ApplicationMasterProtocol.class,
+ rm2.getApplicationMasterService().getBindAddress(), conf);
+ }
+ }).allocate(Records.newRecord(AllocateRequest.class));
+ Assert.fail("The old Token should not work");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof InvalidToken);
+ Assert.assertTrue(ex.getMessage().contains(
+ "Invalid AMRMToken from "
+ + token.decodeIdentifier().getApplicationAttemptId()));
+ }
+
+ // make sure the recovered AMRMToken works for new RM
+ amClient.allocate(0.1f);
+ amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
+ null, null);
+ amClient.stop();
+ rm1.stop();
+ rm2.stop();
+ }
+
private static class MyFifoScheduler extends FifoScheduler {
public MyFifoScheduler(RMContext rmContext) {
@@ -445,6 +584,18 @@ public class TestAMRMClientOnRMRestart {
}
}
+ private static class MyResourceManager2 extends MyResourceManager {
+
+ public MyResourceManager2(Configuration conf, RMStateStore store) {
+ super(conf, store);
+ }
+
+ @Override
+ protected ApplicationMasterService createApplicationMasterService() {
+ return new ApplicationMasterService(getRMContext(), scheduler);
+ }
+ }
+
private static class MyAMRMClientImpl extends
AMRMClientImpl<ContainerRequest> {
private MyResourceManager rm;
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java Fri Aug 8 22:21:35 2014
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.AMCommand;
import org.apache.hadoop.yarn.api.records.Container;
@@ -35,6 +36,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreasePBImpl;
@@ -44,6 +46,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto;
@@ -74,7 +77,7 @@ public class AllocateResponsePBImpl exte
private List<NodeReport> updatedNodes = null;
private PreemptionMessage preempt;
-
+ private Token amrmToken = null;
public AllocateResponsePBImpl() {
builder = AllocateResponseProto.newBuilder();
@@ -154,6 +157,9 @@ public class AllocateResponsePBImpl exte
getChangeProtoIterable(this.decreasedContainers);
builder.addAllDecreasedContainers(iterable);
}
+ if (this.amrmToken != null) {
+ builder.setAmRmToken(convertToProtoFormat(this.amrmToken));
+ }
}
private synchronized void mergeLocalToProto() {
@@ -357,6 +363,28 @@ public class AllocateResponsePBImpl exte
this.decreasedContainers.addAll(decreasedContainers);
}
+ @Override
+ public synchronized Token getAMRMToken() {
+ AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (amrmToken != null) {
+ return amrmToken;
+ }
+ if (!p.hasAmRmToken()) {
+ return null;
+ }
+ this.amrmToken = convertFromProtoFormat(p.getAmRmToken());
+ return amrmToken;
+ }
+
+ @Override
+ public synchronized void setAMRMToken(Token amRMToken) {
+ maybeInitBuilder();
+ if (amRMToken == null) {
+ builder.clearAmRmToken();
+ }
+ this.amrmToken = amRMToken;
+ }
+
private synchronized void initLocalIncreasedContainerList() {
if (this.increasedContainers != null) {
return;
@@ -699,4 +727,12 @@ public class AllocateResponsePBImpl exte
private synchronized NMToken convertFromProtoFormat(NMTokenProto proto) {
return new NMTokenPBImpl(proto);
}
+
+ private TokenPBImpl convertFromProtoFormat(TokenProto p) {
+ return new TokenPBImpl(p);
+ }
+
+ private TokenProto convertToProtoFormat(Token t) {
+ return ((TokenPBImpl)t).getProto();
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java Fri Aug 8 22:21:35 2014
@@ -42,6 +42,13 @@ public class DefaultResourceCalculator e
Resource numerator, Resource denominator) {
return ratio(numerator, denominator);
}
+
+ public boolean isInvalidDivisor(Resource r) {
+ if (r.getMemory() == 0.0f) {
+ return true;
+ }
+ return false;
+ }
@Override
public float ratio(Resource a, Resource b) {
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java Fri Aug 8 22:21:35 2014
@@ -109,6 +109,14 @@ public class DominantResourceCalculator
getResourceAsValue(clusterResource, numerator, true) /
getResourceAsValue(clusterResource, denominator, true);
}
+
+ @Override
+ public boolean isInvalidDivisor(Resource r) {
+ if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f) {
+ return true;
+ }
+ return false;
+ }
@Override
public float ratio(Resource a, Resource b) {
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java Fri Aug 8 22:21:35 2014
@@ -150,6 +150,15 @@ public abstract class ResourceCalculator
Resource clusterResource, Resource numerator, Resource denominator);
/**
+ * Determine if a resource is not suitable for use as a divisor
+ * (will result in divide by 0, etc)
+ *
+ * @param r resource
+ * @return true if divisor is invalid (should not be used), false else
+ */
+ public abstract boolean isInvalidDivisor(Resource r);
+
+ /**
* Ratio of resource <code>a</code> to resource <code>b</code>.
*
* @param a resource
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java Fri Aug 8 22:21:35 2014
@@ -184,6 +184,11 @@ public class Resources {
return calculator.roundDown(lhs, factor);
}
+ public static boolean isInvalidDivisor(
+ ResourceCalculator resourceCalculator, Resource divisor) {
+ return resourceCalculator.isInvalidDivisor(divisor);
+ }
+
public static float ratio(
ResourceCalculator resourceCalculator, Resource lhs, Resource rhs) {
return resourceCalculator.ratio(lhs, rhs);
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java Fri Aug 8 22:21:35 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.ti
import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.readReverseOrderedLong;
import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong;
+import static org.fusesource.leveldbjni.JniDBFactory.bytes;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -60,8 +61,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.DB;
+import org.iq80.leveldb.DBException;
import org.iq80.leveldb.DBIterator;
import org.iq80.leveldb.Options;
import org.iq80.leveldb.ReadOptions;
@@ -141,6 +146,11 @@ public class LeveldbTimelineStore extend
"z".getBytes();
private static final byte[] EMPTY_BYTES = new byte[0];
+
+ private static final String TIMELINE_STORE_VERSION_KEY = "timeline-store-version";
+
+ private static final Version CURRENT_VERSION_INFO = Version
+ .newInstance(1, 0);
@Private
@VisibleForTesting
@@ -193,6 +203,7 @@ public class LeveldbTimelineStore extend
}
LOG.info("Using leveldb path " + dbPath);
db = factory.open(new File(dbPath.toString()), options);
+ checkVersion();
startTimeWriteCache =
Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize(
conf)));
@@ -1270,8 +1281,6 @@ public class LeveldbTimelineStore extend
DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE);
}
- // warning is suppressed to prevent eclipse from noting unclosed resource
- @SuppressWarnings("resource")
@VisibleForTesting
List<String> getEntityTypes() throws IOException {
DBIterator iterator = null;
@@ -1489,4 +1498,65 @@ public class LeveldbTimelineStore extend
readOptions.fillCache(fillCache);
return db.iterator(readOptions);
}
+
+ Version loadVersion() throws IOException {
+ byte[] data = db.get(bytes(TIMELINE_STORE_VERSION_KEY));
+ // if version is not stored previously, treat it as 1.0.
+ if (data == null || data.length == 0) {
+ return Version.newInstance(1, 0);
+ }
+ Version version =
+ new VersionPBImpl(VersionProto.parseFrom(data));
+ return version;
+ }
+
+ // Only used for test
+ @VisibleForTesting
+ void storeVersion(Version state) throws IOException {
+ dbStoreVersion(state);
+ }
+
+ private void dbStoreVersion(Version state) throws IOException {
+ String key = TIMELINE_STORE_VERSION_KEY;
+ byte[] data =
+ ((VersionPBImpl) state).getProto().toByteArray();
+ try {
+ db.put(bytes(key), data);
+ } catch (DBException e) {
+ throw new IOException(e);
+ }
+ }
+
+ Version getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
+
+ /**
+ * 1) Versioning timeline store: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
+ * 2) Any incompatible change of TS-store is a major upgrade, and any
+ * compatible change of TS-store is a minor upgrade.
+ * 3) Within a minor upgrade, say 1.1 to 1.2:
+ * overwrite the version info and proceed as normal.
+ * 4) Within a major upgrade, say 1.2 to 2.0:
+ * throw exception and indicate user to use a separate upgrade tool to
+ * upgrade timeline store or remove incompatible old state.
+ */
+ private void checkVersion() throws IOException {
+ Version loadedVersion = loadVersion();
+ LOG.info("Loaded timeline store version info " + loadedVersion);
+ if (loadedVersion.equals(getCurrentVersion())) {
+ return;
+ }
+ if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
+ LOG.info("Storing timeline store version info " + getCurrentVersion());
+ dbStoreVersion(CURRENT_VERSION_INFO);
+ } else {
+ String incompatibleMessage =
+ "Incompatible version for timeline store: expecting version "
+ + getCurrentVersion() + ", but loading version " + loadedVersion;
+ LOG.fatal(incompatibleMessage);
+ throw new IOException(incompatibleMessage);
+ }
+ }
+
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java Fri Aug 8 22:21:35 2014
@@ -36,14 +36,17 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.iq80.leveldb.DBIterator;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -52,19 +55,19 @@ import org.junit.Test;
public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
private FileContext fsContext;
private File fsPath;
+ private Configuration config = new YarnConfiguration();
@Before
public void setup() throws Exception {
fsContext = FileContext.getLocalFSFileContext();
- Configuration conf = new YarnConfiguration();
fsPath = new File("target", this.getClass().getSimpleName() +
"-tmpDir").getAbsoluteFile();
fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
- conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH,
+ config.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH,
fsPath.getAbsolutePath());
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_TTL_ENABLE, false);
+ config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_TTL_ENABLE, false);
store = new LeveldbTimelineStore();
- store.init(conf);
+ store.init(config);
store.start();
loadTestData();
loadVerificationData();
@@ -263,5 +266,47 @@ public class TestLeveldbTimelineStore ex
assertEquals(1, getEntities("type_2").size());
assertEquals(2, getEntitiesWithPrimaryFilter("type_1", userFilter).size());
}
+
+ @Test
+ public void testCheckVersion() throws IOException {
+ LeveldbTimelineStore dbStore = (LeveldbTimelineStore) store;
+ // default version
+ Version defaultVersion = dbStore.getCurrentVersion();
+ Assert.assertEquals(defaultVersion, dbStore.loadVersion());
+
+ // compatible version
+ Version compatibleVersion =
+ Version.newInstance(defaultVersion.getMajorVersion(),
+ defaultVersion.getMinorVersion() + 2);
+ dbStore.storeVersion(compatibleVersion);
+ Assert.assertEquals(compatibleVersion, dbStore.loadVersion());
+ restartTimelineStore();
+ dbStore = (LeveldbTimelineStore) store;
+ // overwrite the compatible version
+ Assert.assertEquals(defaultVersion, dbStore.loadVersion());
+
+ // incompatible version
+ Version incompatibleVersion =
+ Version.newInstance(defaultVersion.getMajorVersion() + 1,
+ defaultVersion.getMinorVersion());
+ dbStore.storeVersion(incompatibleVersion);
+ try {
+ restartTimelineStore();
+ Assert.fail("Incompatible version, should expect fail here.");
+ } catch (ServiceStateException e) {
+ Assert.assertTrue("Exception message mismatch",
+ e.getMessage().contains("Incompatible version for timeline store"));
+ }
+ }
+
+ private void restartTimelineStore() throws IOException {
+ // need to close so leveldb releases database lock
+ if (store != null) {
+ store.close();
+ }
+ store = new LeveldbTimelineStore();
+ store.init(config);
+ store.start();
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Fri Aug 8 22:21:35 2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
@@ -80,6 +81,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStatusupdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
@@ -89,6 +91,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
+import org.apache.hadoop.yarn.server.security.MasterKeyData;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -189,7 +192,7 @@ public class ApplicationMasterService ex
return result;
}
- private ApplicationAttemptId authorizeRequest()
+ private AMRMTokenIdentifier authorizeRequest()
throws YarnException {
UserGroupInformation remoteUgi;
@@ -226,7 +229,7 @@ public class ApplicationMasterService ex
throw RPCUtil.getRemoteException(message);
}
- return appTokenIdentifier.getApplicationAttemptId();
+ return appTokenIdentifier;
}
@Override
@@ -234,7 +237,9 @@ public class ApplicationMasterService ex
RegisterApplicationMasterRequest request) throws YarnException,
IOException {
- ApplicationAttemptId applicationAttemptId = authorizeRequest();
+ AMRMTokenIdentifier amrmTokenIdentifier = authorizeRequest();
+ ApplicationAttemptId applicationAttemptId =
+ amrmTokenIdentifier.getApplicationAttemptId();
ApplicationId appID = applicationAttemptId.getApplicationId();
AllocateResponseLock lock = responseMap.get(applicationAttemptId);
@@ -333,7 +338,8 @@ public class ApplicationMasterService ex
FinishApplicationMasterRequest request) throws YarnException,
IOException {
- ApplicationAttemptId applicationAttemptId = authorizeRequest();
+ ApplicationAttemptId applicationAttemptId =
+ authorizeRequest().getApplicationAttemptId();
AllocateResponseLock lock = responseMap.get(applicationAttemptId);
if (lock == null) {
@@ -408,7 +414,10 @@ public class ApplicationMasterService ex
public AllocateResponse allocate(AllocateRequest request)
throws YarnException, IOException {
- ApplicationAttemptId appAttemptId = authorizeRequest();
+ AMRMTokenIdentifier amrmTokenIdentifier = authorizeRequest();
+
+ ApplicationAttemptId appAttemptId =
+ amrmTokenIdentifier.getApplicationAttemptId();
this.amLivelinessMonitor.receivedPing(appAttemptId);
@@ -557,6 +566,23 @@ public class ApplicationMasterService ex
allocateResponse
.setPreemptionMessage(generatePreemptionMessage(allocation));
+ // update AMRMToken if the token is rolled-up
+ MasterKeyData nextMasterKey =
+ this.rmContext.getAMRMTokenSecretManager().getNextMasterKeyData();
+
+ if (nextMasterKey != null
+ && nextMasterKey.getMasterKey().getKeyId() != amrmTokenIdentifier
+ .getKeyId()) {
+ Token<AMRMTokenIdentifier> amrmToken =
+ rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
+ appAttemptId);
+ ((RMAppAttemptImpl)appAttempt).setAMRMToken(amrmToken);
+ allocateResponse.setAMRMToken(org.apache.hadoop.yarn.api.records.Token
+ .newInstance(amrmToken.getIdentifier(), amrmToken.getKind()
+ .toString(), amrmToken.getPassword(), amrmToken.getService()
+ .toString()));
+ }
+
/*
* As we are updating the response inside the lock object so we don't
* need to worry about unregister call occurring in between (which
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Fri Aug 8 22:21:35 2014
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -226,7 +227,7 @@ public class AMLauncher implements Runna
}
// Add AMRMToken
- Token<AMRMTokenIdentifier> amrmToken = getAMRMToken();
+ Token<AMRMTokenIdentifier> amrmToken = createAndSetAMRMToken();
if (amrmToken != null) {
credentials.addToken(amrmToken.getService(), amrmToken);
}
@@ -236,8 +237,12 @@ public class AMLauncher implements Runna
}
@VisibleForTesting
- protected Token<AMRMTokenIdentifier> getAMRMToken() {
- return application.getAMRMToken();
+ protected Token<AMRMTokenIdentifier> createAndSetAMRMToken() {
+ Token<AMRMTokenIdentifier> amrmToken =
+ this.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
+ application.getAppAttemptId());
+ ((RMAppAttemptImpl)application).setAMRMToken(amrmToken);
+ return amrmToken;
}
@SuppressWarnings("unchecked")
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java Fri Aug 8 22:21:35 2014
@@ -71,6 +71,10 @@ import com.google.common.annotations.Vis
* FileSystem interface. Does not use directories so that simple key-value
* stores can be used. The retry policy for the real filesystem client must be
* configured separately to enable retry of filesystem operations when needed.
+ *
+ * Changes from 1.1 to 1.2, AMRMTokenSecretManager state has been saved
+ * separately. The currentMasterkey and nextMasterkey have been stored.
+ * Also, AMRMToken has been removed from ApplicationAttemptState.
*/
public class FileSystemRMStateStore extends RMStateStore {
@@ -78,7 +82,7 @@ public class FileSystemRMStateStore exte
protected static final String ROOT_DIR_NAME = "FSRMStateRoot";
protected static final Version CURRENT_VERSION_INFO = Version
- .newInstance(1, 1);
+ .newInstance(1, 2);
protected static final String AMRMTOKEN_SECRET_MANAGER_NODE =
"AMRMTokenSecretManagerNode";
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java Fri Aug 8 22:21:35 2014
@@ -32,7 +32,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -45,7 +44,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEvent;
@@ -769,10 +767,7 @@ public abstract class RMStateStore exten
public Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
Credentials credentials = new Credentials();
- Token<AMRMTokenIdentifier> appToken = appAttempt.getAMRMToken();
- if(appToken != null){
- credentials.addToken(AM_RM_TOKEN_SERVICE, appToken);
- }
+
SecretKey clientTokenMasterKey =
appAttempt.getClientTokenMasterKey();
if(clientTokenMasterKey != null){
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java Fri Aug 8 22:21:35 2014
@@ -78,6 +78,11 @@ import org.apache.zookeeper.server.auth.
import com.google.common.annotations.VisibleForTesting;
+/**
+ * Changes from 1.1 to 1.2, AMRMTokenSecretManager state has been saved
+ * separately. The currentMasterkey and nextMasterkey have been stored.
+ * Also, AMRMToken has been removed from ApplicationAttemptState.
+ */
@Private
@Unstable
public class ZKRMStateStore extends RMStateStore {
@@ -87,7 +92,7 @@ public class ZKRMStateStore extends RMSt
protected static final String ROOT_ZNODE_NAME = "ZKRMStateRoot";
protected static final Version CURRENT_VERSION_INFO = Version
- .newInstance(1, 1);
+ .newInstance(1, 2);
private static final String RM_DELEGATION_TOKENS_ROOT_ZNODE_NAME =
"RMDelegationTokensRoot";
private static final String RM_DT_SEQUENTIAL_NUMBER_ZNODE_NAME =
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Fri Aug 8 22:21:35 2014
@@ -37,6 +37,7 @@ import org.apache.commons.lang.StringUti
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -559,7 +560,22 @@ public class RMAppAttemptImpl implements
@Override
public Token<AMRMTokenIdentifier> getAMRMToken() {
- return this.amrmToken;
+ this.readLock.lock();
+ try {
+ return this.amrmToken;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Private
+ public void setAMRMToken(Token<AMRMTokenIdentifier> lastToken) {
+ this.writeLock.lock();
+ try {
+ this.amrmToken = lastToken;
+ } finally {
+ this.writeLock.unlock();
+ }
}
@Override
@@ -713,7 +729,8 @@ public class RMAppAttemptImpl implements
this.attemptMetrics.setIsPreempted();
}
setMasterContainer(attemptState.getMasterContainer());
- recoverAppAttemptCredentials(attemptState.getAppAttemptCredentials());
+ recoverAppAttemptCredentials(attemptState.getAppAttemptCredentials(),
+ attemptState.getState());
this.recoveredFinalState = attemptState.getState();
this.originalTrackingUrl = attemptState.getFinalTrackingUrl();
this.proxiedTrackingUrl = generateProxyUriWithScheme(originalTrackingUrl);
@@ -725,9 +742,11 @@ public class RMAppAttemptImpl implements
this.justFinishedContainers = attempt.getJustFinishedContainers();
}
- private void recoverAppAttemptCredentials(Credentials appAttemptTokens)
- throws IOException {
- if (appAttemptTokens == null) {
+ private void recoverAppAttemptCredentials(Credentials appAttemptTokens,
+ RMAppAttemptState state) throws IOException {
+ if (appAttemptTokens == null || state == RMAppAttemptState.FAILED
+ || state == RMAppAttemptState.FINISHED
+ || state == RMAppAttemptState.KILLED) {
return;
}
@@ -738,12 +757,9 @@ public class RMAppAttemptImpl implements
.registerMasterKey(applicationAttemptId, clientTokenMasterKeyBytes);
}
- // Only one AMRMToken is stored per-attempt, so this should be fine. Can't
- // use TokenSelector as service may change - think fail-over.
this.amrmToken =
- (Token<AMRMTokenIdentifier>) appAttemptTokens
- .getToken(RMStateStore.AM_RM_TOKEN_SERVICE);
- rmContext.getAMRMTokenSecretManager().addPersistedPassword(this.amrmToken);
+ rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
+ applicationAttemptId);
}
private static class BaseTransition implements
@@ -779,11 +795,6 @@ public class RMAppAttemptImpl implements
.createMasterKey(appAttempt.applicationAttemptId);
}
- // create AMRMToken
- appAttempt.amrmToken =
- appAttempt.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
- appAttempt.applicationAttemptId);
-
// Add the applicationAttempt to the scheduler and inform the scheduler
// whether to transfer the state from previous attempt.
appAttempt.eventHandler.handle(new AppAttemptAddedSchedulerEvent(
@@ -896,6 +907,7 @@ public class RMAppAttemptImpl implements
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
appAttempt.checkAttemptStoreError(event);
+
appAttempt.launchAttempt();
}
}
@@ -1185,11 +1197,12 @@ public class RMAppAttemptImpl implements
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
appAttempt.checkAttemptStoreError(event);
- // TODO Today unmanaged AM client is waiting for app state to be Accepted to
- // launch the AM. This is broken since we changed to start the attempt
- // after the application is Accepted. We may need to introduce an attempt
- // report that client can rely on to query the attempt state and choose to
- // launch the unmanaged AM.
+
+ // create AMRMToken
+ appAttempt.amrmToken =
+ appAttempt.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
+ appAttempt.applicationAttemptId);
+
super.transition(appAttempt, event);
}
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java Fri Aug 8 22:21:35 2014
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -24,6 +27,8 @@ import org.apache.hadoop.yarn.util.resou
class CSQueueUtils {
+ private static final Log LOG = LogFactory.getLog(CSQueueUtils.class);
+
final static float EPSILON = 0.0001f;
public static void checkMaxCapacity(String queueName,
@@ -113,4 +118,52 @@ class CSQueueUtils {
)
);
}
+
+ public static float getAbsoluteMaxAvailCapacity(
+ ResourceCalculator resourceCalculator, Resource clusterResource, CSQueue queue) {
+ CSQueue parent = queue.getParent();
+ if (parent == null) {
+ return queue.getAbsoluteMaximumCapacity();
+ }
+
+ //Get my parent's max avail, needed to determine my own
+ float parentMaxAvail = getAbsoluteMaxAvailCapacity(
+ resourceCalculator, clusterResource, parent);
+ //...and as a resource
+ Resource parentResource = Resources.multiply(clusterResource, parentMaxAvail);
+
+ //check for no resources parent before dividing, if so, max avail is none
+ if (Resources.isInvalidDivisor(resourceCalculator, parentResource)) {
+ return 0.0f;
+ }
+ //sibling used is parent used - my used...
+ float siblingUsedCapacity = Resources.ratio(
+ resourceCalculator,
+ Resources.subtract(parent.getUsedResources(), queue.getUsedResources()),
+ parentResource);
+ //my max avail is the lesser of my max capacity and what is unused from my parent
+ //by my siblings (if they are beyond their base capacity)
+ float maxAvail = Math.min(
+ queue.getMaximumCapacity(),
+ 1.0f - siblingUsedCapacity);
+ //and, mutiply by parent to get absolute (cluster relative) value
+ float absoluteMaxAvail = maxAvail * parentMaxAvail;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("qpath " + queue.getQueuePath());
+ LOG.debug("parentMaxAvail " + parentMaxAvail);
+ LOG.debug("siblingUsedCapacity " + siblingUsedCapacity);
+ LOG.debug("getAbsoluteMaximumCapacity " + queue.getAbsoluteMaximumCapacity());
+ LOG.debug("maxAvail " + maxAvail);
+ LOG.debug("absoluteMaxAvail " + absoluteMaxAvail);
+ }
+
+ if (absoluteMaxAvail < 0.0f) {
+ absoluteMaxAvail = 0.0f;
+ } else if (absoluteMaxAvail > 1.0f) {
+ absoluteMaxAvail = 1.0f;
+ }
+
+ return absoluteMaxAvail;
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Fri Aug 8 22:21:35 2014
@@ -976,13 +976,18 @@ public class LeafQueue implements CSQueu
Resource userLimit = // User limit
computeUserLimit(application, clusterResource, required);
-
+
+ //Max avail capacity needs to take into account usage by ancestor-siblings
+ //which are greater than their base capacity, so we are interested in "max avail"
+ //capacity
+ float absoluteMaxAvailCapacity = CSQueueUtils.getAbsoluteMaxAvailCapacity(
+ resourceCalculator, clusterResource, this);
Resource queueMaxCap = // Queue Max-Capacity
Resources.multiplyAndNormalizeDown(
resourceCalculator,
clusterResource,
- absoluteMaxCapacity,
+ absoluteMaxAvailCapacity,
minimumAllocation);
Resource userConsumed = getUser(user).getConsumedResources();
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Fri Aug 8 22:21:35 2014
@@ -149,6 +149,7 @@ public class FairScheduler extends
// Aggregate metrics
FSQueueMetrics rootMetrics;
+ FSOpDurations fsOpDurations;
// Time when we last updated preemption vars
protected long lastPreemptionUpdateTime;
@@ -256,8 +257,11 @@ public class FairScheduler extends
while (!Thread.currentThread().isInterrupted()) {
try {
Thread.sleep(updateInterval);
+ long start = getClock().getTime();
update();
preemptTasksIfNecessary();
+ long duration = getClock().getTime() - start;
+ fsOpDurations.addUpdateThreadRunDuration(duration);
} catch (InterruptedException ie) {
LOG.warn("Update thread interrupted. Exiting.");
return;
@@ -294,6 +298,7 @@ public class FairScheduler extends
* required resources per job.
*/
protected synchronized void update() {
+ long start = getClock().getTime();
updatePreemptionVariables(); // Determine if any queues merit preemption
FSQueue rootQueue = queueMgr.getRootQueue();
@@ -317,6 +322,9 @@ public class FairScheduler extends
" Demand: " + rootQueue.getDemand());
}
}
+
+ long duration = getClock().getTime() - start;
+ fsOpDurations.addUpdateCallDuration(duration);
}
/**
@@ -325,7 +333,7 @@ public class FairScheduler extends
* for each type of task.
*/
private void updatePreemptionVariables() {
- long now = clock.getTime();
+ long now = getClock().getTime();
lastPreemptionUpdateTime = now;
for (FSLeafQueue sched : queueMgr.getLeafQueues()) {
if (!isStarvedForMinShare(sched)) {
@@ -352,7 +360,8 @@ public class FairScheduler extends
* defined as being below half its fair share.
*/
boolean isStarvedForFairShare(FSLeafQueue sched) {
- Resource desiredFairShare = Resources.min(RESOURCE_CALCULATOR, clusterResource,
+ Resource desiredFairShare = Resources.min(RESOURCE_CALCULATOR,
+ clusterResource,
Resources.multiply(sched.getFairShare(), .5), sched.getDemand());
return Resources.lessThan(RESOURCE_CALCULATOR, clusterResource,
sched.getResourceUsage(), desiredFairShare);
@@ -370,7 +379,7 @@ public class FairScheduler extends
return;
}
- long curTime = clock.getTime();
+ long curTime = getClock().getTime();
if (curTime - lastPreemptCheckTime < preemptionInterval) {
return;
}
@@ -398,6 +407,7 @@ public class FairScheduler extends
* We make sure that no queue is placed below its fair share in the process.
*/
protected void preemptResources(Resource toPreempt) {
+ long start = getClock().getTime();
if (Resources.equals(toPreempt, Resources.none())) {
return;
}
@@ -448,6 +458,9 @@ public class FairScheduler extends
}
}
}
+
+ long duration = getClock().getTime() - start;
+ fsOpDurations.addPreemptCallDuration(duration);
}
protected void warnOrKillContainer(RMContainer container) {
@@ -463,7 +476,7 @@ public class FairScheduler extends
if (time != null) {
// if we asked for preemption more than maxWaitTimeBeforeKill ms ago,
// proceed with kill
- if (time + waitTimeBeforeKill < clock.getTime()) {
+ if (time + waitTimeBeforeKill < getClock().getTime()) {
ContainerStatus status =
SchedulerUtils.createPreemptedContainerStatus(
container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER);
@@ -474,11 +487,11 @@ public class FairScheduler extends
completedContainer(container, status, RMContainerEventType.KILL);
LOG.info("Killing container" + container +
" (after waiting for premption for " +
- (clock.getTime() - time) + "ms)");
+ (getClock().getTime() - time) + "ms)");
}
} else {
// track the request in the FSSchedulerApp itself
- app.addPreemption(container, clock.getTime());
+ app.addPreemption(container, getClock().getTime());
}
}
@@ -659,7 +672,7 @@ public class FairScheduler extends
rmContext);
if (transferStateFromPreviousAttempt) {
attempt.transferStateFromPreviousAttempt(application
- .getCurrentAppAttempt());
+ .getCurrentAppAttempt());
}
application.setCurrentAppAttempt(attempt);
@@ -960,6 +973,7 @@ public class FairScheduler extends
* Process a heartbeat update from a node.
*/
private synchronized void nodeUpdate(RMNode nm) {
+ long start = getClock().getTime();
if (LOG.isDebugEnabled()) {
LOG.debug("nodeUpdate: " + nm + " cluster capacity: " + clusterResource);
}
@@ -996,9 +1010,13 @@ public class FairScheduler extends
} else {
attemptScheduling(node);
}
+
+ long duration = getClock().getTime() - start;
+ fsOpDurations.addNodeUpdateDuration(duration);
}
void continuousSchedulingAttempt() throws InterruptedException {
+ long start = getClock().getTime();
List<NodeId> nodeIdList = new ArrayList<NodeId>(nodes.keySet());
// Sort the nodes by space available on them, so that we offer
// containers on emptier nodes first, facilitating an even spread. This
@@ -1021,6 +1039,9 @@ public class FairScheduler extends
": " + ex.toString(), ex);
}
}
+
+ long duration = getClock().getTime() - start;
+ fsOpDurations.addContinuousSchedulingRunDuration(duration);
}
/** Sort nodes by available resource */
@@ -1244,6 +1265,8 @@ public class FairScheduler extends
}
rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf);
+ fsOpDurations = FSOpDurations.getInstance(true);
+
// This stores per-application scheduling information
this.applications =
new ConcurrentHashMap<ApplicationId,SchedulerApplication<FSSchedulerApp>>();
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java Fri Aug 8 22:21:35 2014
@@ -167,6 +167,11 @@ public class AMRMTokenSecretManager exte
+ this.nextMasterKey.getMasterKey().getKeyId());
this.currentMasterKey = this.nextMasterKey;
this.nextMasterKey = null;
+ AMRMTokenSecretManagerState state =
+ AMRMTokenSecretManagerState.newInstance(
+ this.currentMasterKey.getMasterKey(), null);
+ rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state,
+ true);
} finally {
this.writeLock.unlock();
}