You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/06/18 01:13:18 UTC
svn commit: r1493968 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-a...
Author: vinodkv
Date: Mon Jun 17 23:13:17 2013
New Revision: 1493968
URL: http://svn.apache.org/r1493968
Log:
YARN-610. ClientToken is no longer set in the environment of the Containers. Contributed by Omkar Vinit Joshi.
Added:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSelector.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
Removed:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTokenIdentifier.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTokenSelector.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Mon Jun 17 23:13:17 2013
@@ -183,6 +183,9 @@ Release 2.1.0-beta - UNRELEASED
YARN-822. Renamed ApplicationToken to be AMRMToken, and similarly the
corresponding TokenSelector and SecretManager. (Omkar Vinit Joshi via vinodkv)
+ YARN-610. ClientToken is no longer set in the environment of the Containers.
+ (Omkar Vinit Joshi via vinodkv)
+
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Mon Jun 17 23:13:17 2013
@@ -28,10 +28,6 @@ import org.apache.hadoop.util.Shell;
* TODO: Investigate the semantics and security of each cross-boundary refs.
*/
public interface ApplicationConstants {
-
- // TODO: They say tokens via env isn't good.
- public static final String APPLICATION_CLIENT_SECRET_ENV_NAME =
- "AppClientSecretEnv";
/**
* The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java Mon Jun 17 23:13:17 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.api.protocolrecords;
+import java.nio.ByteBuffer;
import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -84,4 +85,18 @@ public abstract class RegisterApplicatio
@Private
@Unstable
public abstract void setApplicationACLs(Map<ApplicationAccessType, String> acls);
+
+ /**
+ * Set ClientToAMToken master key.
+ */
+ @Public
+ @Stable
+ public abstract void setClientToAMTokenMasterKey(ByteBuffer key);
+
+ /**
+ * Get ClientToAMToken master key.
+ */
+ @Public
+ @Stable
+ public abstract ByteBuffer getClientToAMTokenMasterKey();
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java Mon Jun 17 23:13:17 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -34,6 +35,8 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder;
import org.apache.hadoop.yarn.util.ProtoUtils;
+import com.google.protobuf.ByteString;
+
public class RegisterApplicationMasterResponsePBImpl extends
RegisterApplicationMasterResponse {
@@ -200,7 +203,23 @@ public class RegisterApplicationMasterRe
this.applicationACLS.clear();
this.applicationACLS.putAll(appACLs);
}
-
+
+ @Override
+ public void setClientToAMTokenMasterKey(ByteBuffer key) {
+ if (key == null) {
+ return;
+ }
+ maybeInitBuilder();
+ builder.setClientToAmTokenMasterKey(ByteString.copyFrom(key));
+ }
+
+ @Override
+ public ByteBuffer getClientToAMTokenMasterKey() {
+ ByteBuffer key =
+ ByteBuffer.wrap(builder.getClientToAmTokenMasterKey().toByteArray());
+ return key;
+ }
+
private Resource convertFromProtoFormat(ResourceProto resource) {
return new ResourcePBImpl(resource);
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java Mon Jun 17 23:13:17 2013
@@ -54,7 +54,7 @@ public abstract class ApplicationReport
@Stable
public static ApplicationReport newInstance(ApplicationId applicationId,
ApplicationAttemptId applicationAttemptId, String user, String queue,
- String name, String host, int rpcPort, Token clientToken,
+ String name, String host, int rpcPort, Token clientToAMToken,
YarnApplicationState state, String diagnostics, String url,
long startTime, long finishTime, FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
@@ -67,7 +67,7 @@ public abstract class ApplicationReport
report.setName(name);
report.setHost(host);
report.setRpcPort(rpcPort);
- report.setClientToken(clientToken);
+ report.setClientToAMToken(clientToAMToken);
report.setYarnApplicationState(state);
report.setDiagnostics(diagnostics);
report.setTrackingUrl(url);
@@ -172,13 +172,13 @@ public abstract class ApplicationReport
* Get the <em>client token</em> for communicating with the
* <code>ApplicationMaster</code>.
* <p>
- * <code>ClientToken</code> is the security token used by the AMs to verify
+ * <em>ClientToAMToken</em> is the security token used by the AMs to verify
* authenticity of any <code>client</code>.
* </p>
*
* <p>
* The <code>ResourceManager</code>, provides a secure token (via
- * {@link ApplicationReport#getClientToken()}) which is verified by the
+ * {@link ApplicationReport#getClientToAMToken()}) which is verified by the
* ApplicationMaster when the client directly talks to an AM.
* </p>
* @return <em>client token</em> for communicating with the
@@ -186,11 +186,11 @@ public abstract class ApplicationReport
*/
@Public
@Stable
- public abstract Token getClientToken();
+ public abstract Token getClientToAMToken();
@Private
@Unstable
- public abstract void setClientToken(Token clientToken);
+ public abstract void setClientToAMToken(Token clientToAMToken);
/**
* Get the <code>YarnApplicationState</code> of the application.
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Mon Jun 17 23:13:17 2013
@@ -42,7 +42,7 @@ public class ApplicationReportPBImpl ext
private ApplicationId applicationId;
private ApplicationAttemptId currentApplicationAttemptId;
- private Token clientToken = null;
+ private Token clientToAMToken = null;
public ApplicationReportPBImpl() {
builder = ApplicationReportProto.newBuilder();
@@ -160,16 +160,16 @@ public class ApplicationReportPBImpl ext
}
@Override
- public Token getClientToken() {
+ public Token getClientToAMToken() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
- if (this.clientToken != null) {
- return this.clientToken;
+ if (this.clientToAMToken != null) {
+ return this.clientToAMToken;
}
- if (!p.hasClientToken()) {
+ if (!p.hasClientToAmToken()) {
return null;
}
- this.clientToken = convertFromProtoFormat(p.getClientToken());
- return this.clientToken;
+ this.clientToAMToken = convertFromProtoFormat(p.getClientToAmToken());
+ return this.clientToAMToken;
}
@Override
@@ -309,11 +309,11 @@ public class ApplicationReportPBImpl ext
}
@Override
- public void setClientToken(Token clientToken) {
+ public void setClientToAMToken(Token clientToAMToken) {
maybeInitBuilder();
- if (clientToken == null)
- builder.clearClientToken();
- this.clientToken = clientToken;
+ if (clientToAMToken == null)
+ builder.clearClientToAmToken();
+ this.clientToAMToken = clientToAMToken;
}
@Override
@@ -412,10 +412,10 @@ public class ApplicationReportPBImpl ext
builder.getCurrentApplicationAttemptId())) {
builder.setCurrentApplicationAttemptId(convertToProtoFormat(this.currentApplicationAttemptId));
}
- if (this.clientToken != null
- && !((TokenPBImpl) this.clientToken).getProto().equals(
- builder.getClientToken())) {
- builder.setClientToken(convertToProtoFormat(this.clientToken));
+ if (this.clientToAMToken != null
+ && !((TokenPBImpl) this.clientToAMToken).getProto().equals(
+ builder.getClientToAmToken())) {
+ builder.setClientToAmToken(convertToProtoFormat(this.clientToAMToken));
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Mon Jun 17 23:13:17 2013
@@ -131,7 +131,7 @@ message ApplicationReportProto {
optional string name = 4;
optional string host = 5;
optional int32 rpc_port = 6;
- optional hadoop.common.TokenProto client_token = 7;
+ optional hadoop.common.TokenProto client_to_am_token = 7;
optional YarnApplicationStateProto yarn_application_state = 8;
optional string trackingUrl = 9;
optional string diagnostics = 10 [default = "N/A"];
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Mon Jun 17 23:13:17 2013
@@ -36,7 +36,8 @@ message RegisterApplicationMasterRequest
message RegisterApplicationMasterResponseProto {
optional ResourceProto maximumCapability = 1;
- repeated ApplicationACLMapProto application_ACLs = 2;
+ optional bytes client_to_am_token_master_key = 2;
+ repeated ApplicationACLMapProto application_ACLs = 3;
}
message FinishApplicationMasterRequestProto {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Mon Jun 17 23:13:17 2013
@@ -601,7 +601,7 @@ public class Client {
LOG.info("Got application report from ASM for"
+ ", appId=" + appId.getId()
- + ", clientToken=" + report.getClientToken()
+ + ", clientToAMToken=" + report.getClientToAMToken()
+ ", appDiagnostics=" + report.getDiagnostics()
+ ", appMasterHost=" + report.getHost()
+ ", appQueue=" + report.getQueue()
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java Mon Jun 17 23:13:17 2013
@@ -388,8 +388,8 @@ public class UnmanagedAMLauncher {
LOG.info("Got application report from ASM for" + ", appId="
+ appId.getId() + ", appAttemptId="
- + report.getCurrentApplicationAttemptId() + ", clientToken="
- + report.getClientToken() + ", appDiagnostics="
+ + report.getCurrentApplicationAttemptId() + ", clientToAMToken="
+ + report.getClientToAMToken() + ", appDiagnostics="
+ report.getDiagnostics() + ", appMasterHost=" + report.getHost()
+ ", appQueue=" + report.getQueue() + ", appMasterRpcPort="
+ report.getRpcPort() + ", appStartTime=" + report.getStartTime()
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java Mon Jun 17 23:13:17 2013
@@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.api.record
@Public
@Evolving
public abstract class BaseClientToAMTokenSecretManager extends
- SecretManager<ClientTokenIdentifier> {
+ SecretManager<ClientToAMTokenIdentifier> {
@Private
public abstract SecretKey getMasterKey(
@@ -45,14 +45,14 @@ public abstract class BaseClientToAMToke
@Private
@Override
public synchronized byte[] createPassword(
- ClientTokenIdentifier identifier) {
+ ClientToAMTokenIdentifier identifier) {
return createPassword(identifier.getBytes(),
getMasterKey(identifier.getApplicationAttemptID()));
}
@Private
@Override
- public byte[] retrievePassword(ClientTokenIdentifier identifier)
+ public byte[] retrievePassword(ClientToAMTokenIdentifier identifier)
throws SecretManager.InvalidToken {
SecretKey masterKey = getMasterKey(identifier.getApplicationAttemptID());
if (masterKey == null) {
@@ -63,8 +63,8 @@ public abstract class BaseClientToAMToke
@Private
@Override
- public ClientTokenIdentifier createIdentifier() {
- return new ClientTokenIdentifier();
+ public ClientToAMTokenIdentifier createIdentifier() {
+ return new ClientToAMTokenIdentifier();
}
}
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java?rev=1493968&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java Mon Jun 17 23:13:17 2013
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.security.client;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+@Public
+@Evolving
+public class ClientToAMTokenIdentifier extends TokenIdentifier {
+
+ public static final Text KIND_NAME = new Text("YARN_CLIENT_TOKEN");
+
+ private ApplicationAttemptId applicationAttemptId;
+
+ // TODO: Add more information in the tokenID such that it is not
+ // transferrable, more secure etc.
+
+ public ClientToAMTokenIdentifier() {
+ }
+
+ public ClientToAMTokenIdentifier(ApplicationAttemptId id) {
+ this();
+ this.applicationAttemptId = id;
+ }
+
+ public ApplicationAttemptId getApplicationAttemptID() {
+ return this.applicationAttemptId;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeLong(this.applicationAttemptId.getApplicationId()
+ .getClusterTimestamp());
+ out.writeInt(this.applicationAttemptId.getApplicationId().getId());
+ out.writeInt(this.applicationAttemptId.getAttemptId());
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ this.applicationAttemptId =
+ ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(in.readLong(), in.readInt()), in.readInt());
+ }
+
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ @Override
+ public UserGroupInformation getUser() {
+ if (this.applicationAttemptId == null) {
+ return null;
+ }
+ return UserGroupInformation.createRemoteUser(this.applicationAttemptId.toString());
+ }
+
+ @InterfaceAudience.Private
+ public static class Renewer extends Token.TrivialRenewer {
+ @Override
+ protected Text getKind() {
+ return KIND_NAME;
+ }
+ }
+}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java Mon Jun 17 23:13:17 2013
@@ -38,12 +38,17 @@ public class ClientToAMTokenSecretManage
BaseClientToAMTokenSecretManager {
// Only one client-token and one master-key for AM
- private final SecretKey masterKey;
+ private SecretKey masterKey;
public ClientToAMTokenSecretManager(
- ApplicationAttemptId applicationAttemptID, byte[] secretKeyBytes) {
+ ApplicationAttemptId applicationAttemptID, byte[] key) {
super();
- this.masterKey = SecretManager.createSecretKey(secretKeyBytes);
+ if (key != null) {
+ this.masterKey = SecretManager.createSecretKey(key);
+ } else {
+ this.masterKey = null;
+ }
+
}
@Override
@@ -52,4 +57,7 @@ public class ClientToAMTokenSecretManage
return this.masterKey;
}
+ public void setMasterKey(byte[] key) {
+ this.masterKey = SecretManager.createSecretKey(key);
+ }
}
\ No newline at end of file
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSelector.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSelector.java?rev=1493968&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSelector.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSelector.java Mon Jun 17 23:13:17 2013
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.security.client;
+
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+
+public class ClientToAMTokenSelector implements
+ TokenSelector<ClientToAMTokenIdentifier> {
+
+ private static final Log LOG = LogFactory
+ .getLog(ClientToAMTokenSelector.class);
+
+ @SuppressWarnings("unchecked")
+ public Token<ClientToAMTokenIdentifier> selectToken(Text service,
+ Collection<Token<? extends TokenIdentifier>> tokens) {
+ if (service == null) {
+ return null;
+ }
+ LOG.debug("Looking for a token with service " + service.toString());
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ LOG.debug("Token kind is " + token.getKind().toString()
+ + " and the token's service name is " + token.getService());
+ if (ClientToAMTokenIdentifier.KIND_NAME.equals(token.getKind())
+ && service.equals(token.getService())) {
+ return (Token<ClientToAMTokenIdentifier>) token;
+ }
+ }
+ return null;
+ }
+
+}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier Mon Jun 17 23:13:17 2013
@@ -13,5 +13,5 @@
#
org.apache.hadoop.yarn.security.ContainerTokenIdentifier
org.apache.hadoop.yarn.security.AMRMTokenIdentifier
-org.apache.hadoop.yarn.security.client.ClientTokenIdentifier
+org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier
org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java Mon Jun 17 23:13:17 2013
@@ -61,7 +61,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.util.ConverterUtils;
/**
@@ -224,7 +223,7 @@ public class BuilderUtils {
return newToken(Token.class, identifier, kind, password, service);
}
- public static Token newClientToken(byte[] identifier, String kind,
+ public static Token newClientToAMToken(byte[] identifier, String kind,
byte[] password, String service) {
return newToken(Token.class, identifier, kind, password, service);
}
@@ -299,7 +298,7 @@ public class BuilderUtils {
public static ApplicationReport newApplicationReport(
ApplicationId applicationId, ApplicationAttemptId applicationAttemptId,
String user, String queue, String name, String host, int rpcPort,
- Token clientToken, YarnApplicationState state, String diagnostics,
+ Token clientToAMToken, YarnApplicationState state, String diagnostics,
String url, long startTime, long finishTime,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
@@ -313,7 +312,7 @@ public class BuilderUtils {
report.setName(name);
report.setHost(host);
report.setRpcPort(rpcPort);
- report.setClientToken(clientToken);
+ report.setClientToAMToken(clientToAMToken);
report.setYarnApplicationState(state);
report.setDiagnostics(diagnostics);
report.setTrackingUrl(url);
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Mon Jun 17 23:13:17 2013
@@ -214,6 +214,12 @@ public class ApplicationMasterService ex
.getMaximumResourceCapability());
response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId)
.getSubmissionContext().getAMContainerSpec().getApplicationACLs());
+ if (UserGroupInformation.isSecurityEnabled()) {
+ LOG.info("Setting client token master key");
+ response.setClientToAMTokenMasterKey(java.nio.ByteBuffer.wrap(rmContext
+ .getClientToAMTokenSecretManager()
+ .getMasterKey(applicationAttemptId).getEncoded()));
+ }
return response;
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java Mon Jun 17 23:13:17 2013
@@ -71,7 +71,7 @@ public class RMContextImpl implements RM
AMRMTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
- ClientToAMTokenSecretManagerInRM clientTokenSecretManager) {
+ ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager) {
this.rmDispatcher = rmDispatcher;
this.stateStore = store;
this.containerAllocationExpirer = containerAllocationExpirer;
@@ -81,7 +81,7 @@ public class RMContextImpl implements RM
this.appTokenSecretManager = appTokenSecretManager;
this.containerTokenSecretManager = containerTokenSecretManager;
this.nmTokenSecretManager = nmTokenSecretManager;
- this.clientToAMTokenSecretManager = clientTokenSecretManager;
+ this.clientToAMTokenSecretManager = clientToAMTokenSecretManager;
}
@VisibleForTesting
@@ -94,11 +94,11 @@ public class RMContextImpl implements RM
AMRMTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
- ClientToAMTokenSecretManagerInRM clientTokenSecretManager) {
+ ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager) {
this(rmDispatcher, null, containerAllocationExpirer, amLivelinessMonitor,
amFinishingMonitor, tokenRenewer, appTokenSecretManager,
containerTokenSecretManager, nmTokenSecretManager,
- clientTokenSecretManager);
+ clientToAMTokenSecretManager);
RMStateStore nullStore = new NullRMStateStore();
nullStore.setDispatcher(rmDispatcher);
try {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Mon Jun 17 23:13:17 2013
@@ -165,12 +165,12 @@ public class AMLauncher implements Runna
new String[0])));
// Finalize the container
- setupTokensAndEnv(container, containerID);
+ setupTokens(container, containerID);
return container;
}
- private void setupTokensAndEnv(
+ private void setupTokens(
ContainerLaunchContext container, ContainerId containerID)
throws IOException {
Map<String, String> environment = container.getEnvironment();
@@ -210,15 +210,6 @@ public class AMLauncher implements Runna
credentials.writeTokenStorageToStream(dob);
container.setTokens(ByteBuffer.wrap(dob.getData(), 0,
dob.getLength()));
-
- SecretKey clientSecretKey =
- this.rmContext.getClientToAMTokenSecretManager().getMasterKey(
- application.getAppAttemptId());
- String encoded =
- Base64.encodeBase64URLSafeString(clientSecretKey.getEncoded());
- environment.put(
- ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME,
- encoded);
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java Mon Jun 17 23:13:17 2013
@@ -42,7 +42,7 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
@@ -382,9 +382,10 @@ public abstract class RMStateStore {
if(appToken != null){
credentials.addToken(appToken.getService(), appToken);
}
- Token<ClientTokenIdentifier> clientToken = appAttempt.getClientToken();
- if(clientToken != null){
- credentials.addToken(clientToken.getService(), clientToken);
+ Token<ClientToAMTokenIdentifier> clientToAMToken =
+ appAttempt.getClientToAMToken();
+ if(clientToAMToken != null){
+ credentials.addToken(clientToAMToken.getService(), clientToAMToken);
}
return credentials;
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Mon Jun 17 23:13:17 2013
@@ -47,7 +47,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
@@ -430,7 +430,7 @@ public class RMAppImpl implements RMApp,
try {
ApplicationAttemptId currentApplicationAttemptId = null;
- org.apache.hadoop.yarn.api.records.Token clientToken = null;
+ org.apache.hadoop.yarn.api.records.Token clientToAMToken = null;
String trackingUrl = UNAVAILABLE;
String host = UNAVAILABLE;
String origTrackingUrl = UNAVAILABLE;
@@ -445,13 +445,15 @@ public class RMAppImpl implements RMApp,
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
trackingUrl = this.currentAttempt.getTrackingUrl();
origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
- Token<ClientTokenIdentifier> attemptClientToken =
- this.currentAttempt.getClientToken();
- if (attemptClientToken != null) {
- clientToken =
- BuilderUtils.newClientToken(attemptClientToken.getIdentifier(),
- attemptClientToken.getKind().toString(), attemptClientToken
- .getPassword(), attemptClientToken.getService().toString());
+ Token<ClientToAMTokenIdentifier> attemptClientToAMToken =
+ this.currentAttempt.getClientToAMToken();
+ if (attemptClientToAMToken != null) {
+ clientToAMToken =
+ BuilderUtils.newClientToAMToken(
+ attemptClientToAMToken.getIdentifier(),
+ attemptClientToAMToken.getKind().toString(),
+ attemptClientToAMToken.getPassword(),
+ attemptClientToAMToken.getService().toString());
}
host = this.currentAttempt.getHost();
rpcPort = this.currentAttempt.getRpcPort();
@@ -469,7 +471,7 @@ public class RMAppImpl implements RMApp,
return BuilderUtils.newApplicationReport(this.applicationId,
currentApplicationAttemptId, this.user, this.queue,
- this.name, host, rpcPort, clientToken,
+ this.name, host, rpcPort, clientToAMToken,
createApplicationState(this.stateMachine.getCurrentState()), diags,
trackingUrl, this.startTime, this.finishTime, finishState,
appUsageReport, origTrackingUrl, progress, this.applicationType);
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Mon Jun 17 23:13:17 2013
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
/**
@@ -94,7 +94,7 @@ public interface RMAppAttempt extends Ev
* The token required by the clients to talk to the application attempt
* @return the token required by the clients to talk to the application attempt
*/
- Token<ClientTokenIdentifier> getClientToken();
+ Token<ClientToAMTokenIdentifier> getClientToAMToken();
/**
* Diagnostics information for the application attempt.
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Mon Jun 17 23:13:17 2013
@@ -63,8 +63,8 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.AMRMTokenSelector;
-import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientTokenSelector;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
@@ -129,7 +129,7 @@ public class RMAppAttemptImpl implements
private final WriteLock writeLock;
private final ApplicationAttemptId applicationAttemptId;
- private Token<ClientTokenIdentifier> clientToken;
+ private Token<ClientToAMTokenIdentifier> clientToAMToken;
private final ApplicationSubmissionContext submissionContext;
private Token<AMRMTokenIdentifier> amrmToken = null;
@@ -498,8 +498,8 @@ public class RMAppAttemptImpl implements
}
@Override
- public Token<ClientTokenIdentifier> getClientToken() {
- return this.clientToken;
+ public Token<ClientToAMTokenIdentifier> getClientToAMToken() {
+ return this.clientToAMToken;
}
@Override
@@ -673,9 +673,10 @@ public class RMAppAttemptImpl implements
}
if (UserGroupInformation.isSecurityEnabled()) {
- ClientTokenSelector clientTokenSelector = new ClientTokenSelector();
- this.clientToken =
- clientTokenSelector.selectToken(new Text(),
+ ClientToAMTokenSelector clientToAMTokenSelector =
+ new ClientToAMTokenSelector();
+ this.clientToAMToken =
+ clientToAMTokenSelector.selectToken(new Text(),
appAttemptTokens.getAllTokens());
InetSocketAddress serviceAddr = conf.getSocketAddr(
@@ -720,9 +721,9 @@ public class RMAppAttemptImpl implements
appAttempt.rmContext.getClientToAMTokenSecretManager()
.registerApplication(appAttempt.applicationAttemptId);
- // create clientToken
- appAttempt.clientToken =
- new Token<ClientTokenIdentifier>(new ClientTokenIdentifier(
+ // create clientToAMToken
+ appAttempt.clientToAMToken =
+ new Token<ClientToAMTokenIdentifier>(new ClientToAMTokenIdentifier(
appAttempt.applicationAttemptId),
appAttempt.rmContext.getClientToAMTokenSecretManager());
@@ -1050,7 +1051,7 @@ public class RMAppAttemptImpl implements
appAttempt.rmContext.getAMFinishingMonitor().unregister(
appAttempt.getAppAttemptId());
- // Unregister from the ClientTokenSecretManager
+ // Unregister from the ClientToAMTokenSecretManager
if (UserGroupInformation.isSecurityEnabled()) {
appAttempt.rmContext.getClientToAMTokenSecretManager()
.unRegisterApplication(appAttempt.getAppAttemptId());
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java Mon Jun 17 23:13:17 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -49,7 +50,7 @@ public class MockAM {
private final List<ResourceRequest> requests = new ArrayList<ResourceRequest>();
private final List<ContainerId> releases = new ArrayList<ContainerId>();
- MockAM(RMContext context, ApplicationMasterProtocol amRMProtocol,
+ public MockAM(RMContext context, ApplicationMasterProtocol amRMProtocol,
ApplicationAttemptId attemptId) {
this.context = context;
this.amRMProtocol = amRMProtocol;
@@ -77,7 +78,7 @@ public class MockAM {
finalState, attempt.getAppAttemptState());
}
- public void registerAppAttempt() throws Exception {
+ public RegisterApplicationMasterResponse registerAppAttempt() throws Exception {
waitForState(RMAppAttemptState.LAUNCHED);
responseId = 0;
RegisterApplicationMasterRequest req = Records.newRecord(RegisterApplicationMasterRequest.class);
@@ -85,7 +86,7 @@ public class MockAM {
req.setHost("");
req.setRpcPort(1);
req.setTrackingUrl("");
- amRMProtocol.registerApplicationMaster(req);
+ return amRMProtocol.registerApplicationMaster(req);
}
public void addRequests(String[] hosts, int memory, int priority,
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Mon Jun 17 23:13:17 2013
@@ -96,7 +96,7 @@ public class MockRM extends ResourceMana
while (!finalState.equals(app.getState()) && timeoutSecs++ < 40) {
System.out.println("App : " + appId + " State is : " + app.getState()
+ " Waiting for state : " + finalState);
- Thread.sleep(1000);
+ Thread.sleep(2000);
}
System.out.println("App State is : " + app.getState());
Assert.assertEquals("App state is not correct (timedout)", finalState,
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java Mon Jun 17 23:13:17 2013
@@ -338,7 +338,7 @@ public class TestApplicationACLs {
Assert.assertEquals("Enemy should not see app rpc port!",
-1, appReport.getRpcPort());
Assert.assertEquals("Enemy should not see app client token!",
- null, appReport.getClientToken());
+ null, appReport.getClientToAMToken());
Assert.assertEquals("Enemy should not see app diagnostics!",
UNAVAILABLE, appReport.getDiagnostics());
Assert.assertEquals("Enemy should not see app tracking url!",
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Mon Jun 17 23:13:17 2013
@@ -534,10 +534,11 @@ public class TestRMRestart {
Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
attemptState.getMasterContainer().getId());
- // the appToken and clientToken that are generated when RMAppAttempt is created,
+ // the appToken and clientToAMToken that are generated when RMAppAttempt
+ // is created,
HashSet<Token<?>> tokenSet = new HashSet<Token<?>>();
tokenSet.add(attempt1.getAMRMToken());
- tokenSet.add(attempt1.getClientToken());
+ tokenSet.add(attempt1.getClientToAMToken());
// assert application Token is saved
HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
@@ -556,12 +557,13 @@ public class TestRMRestart {
Assert.assertNotNull(loadedAttempt1);
savedTokens.clear();
savedTokens.add(loadedAttempt1.getAMRMToken());
- savedTokens.add(loadedAttempt1.getClientToken());
+ savedTokens.add(loadedAttempt1.getClientToAMToken());
Assert.assertEquals(tokenSet, savedTokens);
- // assert clientToken is recovered back to api-versioned clientToken
- Assert.assertEquals(attempt1.getClientToken(),
- loadedAttempt1.getClientToken());
+ // assert clientToAMToken is recovered back to api-versioned
+ // clientToAMToken
+ Assert.assertEquals(attempt1.getClientToAMToken(),
+ loadedAttempt1.getClientToAMToken());
// Not testing ApplicationTokenSecretManager has the password populated back,
// that is needed in work-preserving restart
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java Mon Jun 17 23:13:17 2013
@@ -18,41 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import junit.framework.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.QueueInfo;
-import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreFactory;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
public class TestSchedulerNegotiator {
// private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java?rev=1493968&r1=1493967&r2=1493968&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java Mon Jun 17 23:13:17 2013
@@ -55,7 +55,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
@@ -206,7 +206,7 @@ public class TestRMStateStore {
ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId,
String containerIdStr, Token<AMRMTokenIdentifier> appToken,
- Token<ClientTokenIdentifier> clientToken, TestDispatcher dispatcher)
+ Token<ClientToAMTokenIdentifier> clientToAMToken, TestDispatcher dispatcher)
throws Exception {
Container container = new ContainerPBImpl();
@@ -215,7 +215,7 @@ public class TestRMStateStore {
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
when(mockAttempt.getMasterContainer()).thenReturn(container);
when(mockAttempt.getAMRMToken()).thenReturn(appToken);
- when(mockAttempt.getClientToken()).thenReturn(clientToken);
+ when(mockAttempt.getClientToAMToken()).thenReturn(clientToAMToken);
dispatcher.attemptId = attemptId;
dispatcher.storedException = null;
store.storeApplicationAttempt(mockAttempt);
@@ -233,7 +233,7 @@ public class TestRMStateStore {
AMRMTokenSecretManager appTokenMgr =
new AMRMTokenSecretManager(conf);
- ClientToAMTokenSecretManagerInRM clientTokenMgr =
+ ClientToAMTokenSecretManagerInRM clientToAMTokenMgr =
new ClientToAMTokenSecretManagerInRM();
ApplicationAttemptId attemptId1 = ConverterUtils
@@ -243,14 +243,14 @@ public class TestRMStateStore {
// create application token1 for attempt1
List<Token<?>> appAttemptToken1 =
- generateTokens(attemptId1, appTokenMgr, clientTokenMgr, conf);
+ generateTokens(attemptId1, appTokenMgr, clientToAMTokenMgr, conf);
HashSet<Token<?>> attemptTokenSet1 = new HashSet<Token<?>>();
attemptTokenSet1.addAll(appAttemptToken1);
ContainerId containerId1 = storeAttempt(store, attemptId1,
"container_1352994193343_0001_01_000001",
(Token<AMRMTokenIdentifier>) (appAttemptToken1.get(0)),
- (Token<ClientTokenIdentifier>)(appAttemptToken1.get(1)),
+ (Token<ClientToAMTokenIdentifier>)(appAttemptToken1.get(1)),
dispatcher);
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
@@ -259,14 +259,14 @@ public class TestRMStateStore {
// create application token2 for attempt2
List<Token<?>> appAttemptToken2 =
- generateTokens(attemptId2, appTokenMgr, clientTokenMgr, conf);
+ generateTokens(attemptId2, appTokenMgr, clientToAMTokenMgr, conf);
HashSet<Token<?>> attemptTokenSet2 = new HashSet<Token<?>>();
attemptTokenSet2.addAll(appAttemptToken2);
ContainerId containerId2 = storeAttempt(store, attemptId2,
"container_1352994193343_0001_02_000001",
(Token<AMRMTokenIdentifier>) (appAttemptToken2.get(0)),
- (Token<ClientTokenIdentifier>)(appAttemptToken2.get(1)),
+ (Token<ClientToAMTokenIdentifier>)(appAttemptToken2.get(1)),
dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils
@@ -373,21 +373,22 @@ public class TestRMStateStore {
private List<Token<?>> generateTokens(ApplicationAttemptId attemptId,
AMRMTokenSecretManager appTokenMgr,
- ClientToAMTokenSecretManagerInRM clientTokenMgr, Configuration conf) {
+ ClientToAMTokenSecretManagerInRM clientToAMTokenMgr, Configuration conf) {
AMRMTokenIdentifier appTokenId =
new AMRMTokenIdentifier(attemptId);
Token<AMRMTokenIdentifier> appToken =
new Token<AMRMTokenIdentifier>(appTokenId, appTokenMgr);
appToken.setService(new Text("appToken service"));
- ClientTokenIdentifier clientTokenId = new ClientTokenIdentifier(attemptId);
- clientTokenMgr.registerApplication(attemptId);
- Token<ClientTokenIdentifier> clientToken =
- new Token<ClientTokenIdentifier>(clientTokenId, clientTokenMgr);
- clientToken.setService(new Text("clientToken service"));
+ ClientToAMTokenIdentifier clientToAMTokenId =
+ new ClientToAMTokenIdentifier(attemptId);
+ clientToAMTokenMgr.registerApplication(attemptId);
+ Token<ClientToAMTokenIdentifier> clientToAMToken =
+ new Token<ClientToAMTokenIdentifier>(clientToAMTokenId, clientToAMTokenMgr);
+ clientToAMToken.setService(new Text("clientToAMToken service"));
List<Token<?>> tokenPair = new ArrayList<Token<?>>();
tokenPair.add(0, appToken);
- tokenPair.add(1, clientToken);
+ tokenPair.add(1, clientToAMToken);
return tokenPair;
}
}
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java?rev=1493968&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java Mon Jun 17 23:13:17 2013
@@ -0,0 +1,323 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.security;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
+
+import javax.security.sasl.SaslException;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
+import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRMWithCustomAMLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.ProtoUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.junit.Test;
+
+public class TestClientToAMTokens {
+
+ private interface CustomProtocol {
+ @SuppressWarnings("unused")
+ public static final long versionID = 1L;
+
+ public void ping() throws YarnException, IOException;
+ }
+
+ private static class CustomSecurityInfo extends SecurityInfo {
+
+ @Override
+ public TokenInfo getTokenInfo(Class<?> protocol, Configuration conf) {
+ return new TokenInfo() {
+
+ @Override
+ public Class<? extends Annotation> annotationType() {
+ return null;
+ }
+
+ @Override
+ public Class<? extends TokenSelector<? extends TokenIdentifier>>
+ value() {
+ return ClientToAMTokenSelector.class;
+ }
+ };
+ }
+
+ @Override
+ public KerberosInfo getKerberosInfo(Class<?> protocol, Configuration conf) {
+ return null;
+ }
+ };
+
+ private static class CustomAM extends AbstractService implements
+ CustomProtocol {
+
+ private final ApplicationAttemptId appAttemptId;
+ private final byte[] secretKey;
+ private InetSocketAddress address;
+ private boolean pinged = false;
+ private ClientToAMTokenSecretManager secretManager;
+
+ public CustomAM(ApplicationAttemptId appId, byte[] secretKey) {
+ super("CustomAM");
+ this.appAttemptId = appId;
+ this.secretKey = secretKey;
+ }
+
+ @Override
+ public void ping() throws YarnException, IOException {
+ this.pinged = true;
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ Configuration conf = getConfig();
+
+ secretManager = new ClientToAMTokenSecretManager(this.appAttemptId, secretKey);
+ Server server;
+ try {
+ server =
+ new RPC.Builder(conf).setProtocol(CustomProtocol.class)
+ .setNumHandlers(1).setSecretManager(secretManager)
+ .setInstance(this).build();
+ } catch (Exception e) {
+ throw new YarnRuntimeException(e);
+ }
+ server.start();
+ this.address = NetUtils.getConnectAddress(server);
+ super.serviceStart();
+ }
+
+ public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
+ return this.secretManager;
+ }
+ }
+
+ @Test
+ public void testClientToAMs() throws Exception {
+
+ final Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+
+ ContainerManagementProtocol containerManager =
+ mock(ContainerManagementProtocol.class);
+ final DrainDispatcher dispatcher = new DrainDispatcher();
+
+ MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
+ protected ClientRMService createClientRMService() {
+ return new ClientRMService(this.rmContext, scheduler,
+ this.rmAppManager, this.applicationACLsManager,
+ this.rmDTSecretManager);
+ };
+
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
+
+ @Override
+ protected void doSecureLogin() throws IOException {
+ }
+ };
+ rm.start();
+
+ // Submit an app
+ RMApp app = rm.submitApp(1024);
+
+ // Set up a node.
+ MockNM nm1 = rm.registerNode("localhost:1234", 3072);
+ nm1.nodeHeartbeat(true);
+ dispatcher.await();
+
+
+ nm1.nodeHeartbeat(true);
+ dispatcher.await();
+
+ // Get the app-report.
+ GetApplicationReportRequest request =
+ Records.newRecord(GetApplicationReportRequest.class);
+ request.setApplicationId(app.getApplicationId());
+ GetApplicationReportResponse reportResponse =
+ rm.getClientRMService().getApplicationReport(request);
+ ApplicationReport appReport = reportResponse.getApplicationReport();
+ org.apache.hadoop.yarn.api.records.Token clientToAMToken =
+ appReport.getClientToAMToken();
+
+ ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
+ final MockAM mockAM =
+ new MockAM(rm.getRMContext(), rm.getApplicationMasterService(),
+ app.getCurrentAppAttempt().getAppAttemptId());
+ UserGroupInformation appUgi =
+ UserGroupInformation.createRemoteUser(appAttempt.toString());
+ RegisterApplicationMasterResponse response =
+ appUgi.doAs(new PrivilegedAction<RegisterApplicationMasterResponse>() {
+
+ @Override
+ public RegisterApplicationMasterResponse run() {
+ RegisterApplicationMasterResponse response = null;
+ try {
+ response = mockAM.registerAppAttempt();
+ } catch (Exception e) {
+ Assert.fail("Exception was not expected");
+ }
+ return response;
+ }
+ });
+
+ // ClientToAMToken master key should have been received on register
+ // application master response.
+ Assert.assertNotNull(response.getClientToAMTokenMasterKey());
+ Assert
+ .assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
+
+ // Start the AM with the correct shared-secret.
+ ApplicationAttemptId appAttemptId =
+ app.getAppAttempts().keySet().iterator().next();
+ Assert.assertNotNull(appAttemptId);
+ final CustomAM am =
+ new CustomAM(appAttemptId, response.getClientToAMTokenMasterKey()
+ .array());
+ am.init(conf);
+ am.start();
+
+ // Now the real test!
+ // Set up clients to be able to pick up correct tokens.
+ SecurityUtil.setSecurityInfoProviders(new CustomSecurityInfo());
+
+ // Verify denial for unauthenticated user
+ try {
+ CustomProtocol client =
+ (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L, am.address,
+ conf);
+ client.ping();
+ fail("Access by unauthenticated user should fail!!");
+ } catch (Exception e) {
+ Assert.assertFalse(am.pinged);
+ }
+
+ // Verify denial for a malicious user
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
+ Token<ClientToAMTokenIdentifier> token =
+ ProtoUtils.convertFromProtoFormat(clientToAMToken, am.address);
+
+ // Malicious user, messes with appId
+ ClientToAMTokenIdentifier maliciousID =
+ new ClientToAMTokenIdentifier(BuilderUtils.newApplicationAttemptId(
+ BuilderUtils.newApplicationId(app.getApplicationId()
+ .getClusterTimestamp(), 42), 43));
+
+ Token<ClientToAMTokenIdentifier> maliciousToken =
+ new Token<ClientToAMTokenIdentifier>(maliciousID.getBytes(),
+ token.getPassword(), token.getKind(),
+ token.getService());
+ ugi.addToken(maliciousToken);
+
+ try {
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ CustomProtocol client =
+ (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L,
+ am.address, conf);
+ client.ping();
+ fail("Connection initiation with illegally modified "
+ + "tokens is expected to fail.");
+ return null;
+ } catch (YarnException ex) {
+ fail("Cannot get a YARN remote exception as "
+ + "it will indicate RPC success");
+ throw ex;
+ }
+ }
+ });
+ } catch (Exception e) {
+ Assert.assertEquals(RemoteException.class.getName(), e.getClass()
+ .getName());
+ e = ((RemoteException)e).unwrapRemoteException();
+ Assert
+ .assertEquals(SaslException.class
+ .getCanonicalName(), e.getClass().getCanonicalName());
+ Assert.assertTrue(e
+ .getMessage()
+ .contains(
+ "DIGEST-MD5: digest response format violation. "
+ + "Mismatched response."));
+ Assert.assertFalse(am.pinged);
+ }
+
+ // Now for an authenticated user
+ ugi = UserGroupInformation.createRemoteUser("me");
+ ugi.addToken(token);
+
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ CustomProtocol client =
+ (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L, am.address,
+ conf);
+ client.ping();
+ Assert.assertTrue(am.pinged);
+ return null;
+ }
+ });
+ }
+
+}