You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/10/25 07:49:00 UTC
git commit: YARN-1915. Fixed a race condition that client could use
the ClientToAMToken to contact with AM before AM actually receives the
ClientToAMTokenMasterKey. Contributed by Jason Lowe (cherry picked from
commit 5864dd99a419b8a0fb973a4984c0f1d1e02c
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9af3cfc7e -> 96a6e02d1
YARN-1915. Fixed a race condition that client could use the ClientToAMToken to contact with AM before AM actually receives the ClientToAMTokenMasterKey. Contributed by Jason Lowe
(cherry picked from commit 5864dd99a419b8a0fb973a4984c0f1d1e02ccf16)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96a6e02d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96a6e02d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96a6e02d
Branch: refs/heads/branch-2
Commit: 96a6e02d16ca0334775b2a878124ae3c54eec624
Parents: 9af3cfc
Author: Jian He <ji...@apache.org>
Authored: Fri Oct 24 22:47:16 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Fri Oct 24 22:48:48 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 +
.../client/ClientToAMTokenSecretManager.java | 25 +++-
.../security/TestClientToAMTokens.java | 122 ++++++++++++++++++-
3 files changed, 148 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96a6e02d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3d4cdfd..9cb4efc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -712,6 +712,10 @@ Release 2.6.0 - UNRELEASED
YARN-2724. Skipped uploading a local log file to HDFS if exception is raised
when opening it. (Xuan Gong via zjshen)
+ YARN-1915. Fixed a race condition that client could use the ClientToAMToken
+ to contact with AM before AM actually receives the ClientToAMTokenMasterKey.
+ (Jason Lowe via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96a6e02d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
index 541f7a8..29fe979 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
@@ -36,9 +36,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@Evolving
public class ClientToAMTokenSecretManager extends
BaseClientToAMTokenSecretManager {
+ private static final int MASTER_KEY_WAIT_MSEC = 10 * 1000;
// Only one master-key for AM
- private SecretKey masterKey;
+ private volatile SecretKey masterKey;
public ClientToAMTokenSecretManager(
ApplicationAttemptId applicationAttemptID, byte[] key) {
@@ -52,12 +53,32 @@ public class ClientToAMTokenSecretManager extends
}
@Override
+ public byte[] retrievePassword(ClientToAMTokenIdentifier identifier)
+ throws InvalidToken {
+ if (this.masterKey == null) {
+ synchronized (this) {
+ while (masterKey == null) {
+ try {
+ wait(MASTER_KEY_WAIT_MSEC);
+ break;
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+ }
+ return super.retrievePassword(identifier);
+ }
+
+ @Override
public SecretKey getMasterKey(ApplicationAttemptId applicationAttemptID) {
// Only one master-key for AM, just return that.
return this.masterKey;
}
public void setMasterKey(byte[] key) {
- this.masterKey = SecretManager.createSecretKey(key);
+ synchronized (this) {
+ this.masterKey = SecretManager.createSecretKey(key);
+ notifyAll();
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96a6e02d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
index 9b205f6..78bc728 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
@@ -30,9 +30,12 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
+import java.util.Timer;
+import java.util.TimerTask;
import javax.security.sasl.SaslException;
@@ -169,6 +172,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
this.address = NetUtils.getConnectAddress(server);
super.serviceStart();
}
+
+ public void setClientSecretKey(byte[] key) {
+ secretMgr.setMasterKey(key);
+ }
}
@Test
@@ -291,7 +298,7 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
// Verify for a new version token
verifyNewVersionToken(conf, am, token, rm);
-
+ am.stop();
rm.stop();
}
@@ -410,4 +417,117 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
}
});
}
+
+ @Test(timeout=20000)
+ public void testClientTokenRace() throws Exception {
+
+ final Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+
+ ContainerManagementProtocol containerManager =
+ mock(ContainerManagementProtocol.class);
+ StartContainersResponse mockResponse = mock(StartContainersResponse.class);
+ when(containerManager.startContainers((StartContainersRequest) any()))
+ .thenReturn(mockResponse);
+ final DrainDispatcher dispatcher = new DrainDispatcher();
+
+ MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
+ protected ClientRMService createClientRMService() {
+ return new ClientRMService(this.rmContext, scheduler,
+ this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
+ getRMContext().getRMDelegationTokenSecretManager());
+ };
+
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
+
+ @Override
+ protected void doSecureLogin() throws IOException {
+ }
+ };
+ rm.start();
+
+ // Submit an app
+ RMApp app = rm.submitApp(1024);
+
+ // Set up a node.
+ MockNM nm1 = rm.registerNode("localhost:1234", 3072);
+ nm1.nodeHeartbeat(true);
+ dispatcher.await();
+
+ nm1.nodeHeartbeat(true);
+ dispatcher.await();
+
+ ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
+ final MockAM mockAM =
+ new MockAM(rm.getRMContext(), rm.getApplicationMasterService(),
+ app.getCurrentAppAttempt().getAppAttemptId());
+ UserGroupInformation appUgi =
+ UserGroupInformation.createRemoteUser(appAttempt.toString());
+ RegisterApplicationMasterResponse response =
+ appUgi.doAs(new PrivilegedAction<RegisterApplicationMasterResponse>() {
+
+ @Override
+ public RegisterApplicationMasterResponse run() {
+ RegisterApplicationMasterResponse response = null;
+ try {
+ response = mockAM.registerAppAttempt();
+ } catch (Exception e) {
+ Assert.fail("Exception was not expected");
+ }
+ return response;
+ }
+ });
+
+ // Get the app-report.
+ GetApplicationReportRequest request =
+ Records.newRecord(GetApplicationReportRequest.class);
+ request.setApplicationId(app.getApplicationId());
+ GetApplicationReportResponse reportResponse =
+ rm.getClientRMService().getApplicationReport(request);
+ ApplicationReport appReport = reportResponse.getApplicationReport();
+ org.apache.hadoop.yarn.api.records.Token originalClientToAMToken =
+ appReport.getClientToAMToken();
+
+ // ClientToAMToken master key should have been received on register
+ // application master response.
+ final ByteBuffer clientMasterKey = response.getClientToAMTokenMasterKey();
+ Assert.assertNotNull(clientMasterKey);
+ Assert.assertTrue(clientMasterKey.array().length > 0);
+
+ // Start the AM with the correct shared-secret.
+ ApplicationAttemptId appAttemptId =
+ app.getAppAttempts().keySet().iterator().next();
+ Assert.assertNotNull(appAttemptId);
+ final CustomAM am = new CustomAM(appAttemptId, null);
+ am.init(conf);
+ am.start();
+
+ // Now the real test!
+ // Set up clients to be able to pick up correct tokens.
+ SecurityUtil.setSecurityInfoProviders(new CustomSecurityInfo());
+
+ Token<ClientToAMTokenIdentifier> token =
+ ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
+
+ // Schedule the key to be set after a significant delay
+ Timer timer = new Timer();
+ TimerTask timerTask = new TimerTask() {
+ @Override
+ public void run() {
+ am.setClientSecretKey(clientMasterKey.array());
+ }
+ };
+ timer.schedule(timerTask, 250);
+
+ // connect should pause waiting for the master key to arrive
+ verifyValidToken(conf, am, token);
+
+ am.stop();
+ rm.stop();
+ }
}