You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/01/14 23:54:22 UTC
git commit: TEZ-727. Fix private resource localization failures on
secure clusters. (sseth)
Updated Branches:
refs/heads/master 462696e0b -> 2a8a92001
TEZ-727. Fix private resource localization failures on secure clusters.
(sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/2a8a9200
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/2a8a9200
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/2a8a9200
Branch: refs/heads/master
Commit: 2a8a92001c63a51b90015c0ae3b104a80fce2d68
Parents: 462696e
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Jan 14 14:53:38 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Jan 14 14:53:38 2014 -0800
----------------------------------------------------------------------
.../app/rm/container/AMContainerHelpers.java | 22 ++++++++++----------
1 file changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2a8a9200/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 4d14c2f..b054992 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -43,7 +42,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.common.security.JobTokenIdentifier;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.records.TezDAGID;
@@ -103,25 +101,27 @@ public class AMContainerHelpers {
// Tokens
// Setup up task credentials buffer
- ByteBuffer taskCredentialsBuffer = ByteBuffer.wrap(new byte[] {});
+ ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(new byte[] {});
try {
- // Setting only the JobToken when launching the container. This is meant to be used for
- // communicating with the AM - and is managed by Tez. The rest of the tokens will be sent
- // along with the task - if required.
Credentials containerCredentials = new Credentials();
- Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(credentials);
- TokenCache.setJobToken(jobToken, containerCredentials);
+
+ // All Credentials need to be set so that YARN can localize the resources
+ // correctly, even though they may not be used by all tasks which will run
+ // on this container.
+ LOG.info("Adding #" + credentials.numberOfTokens() + " tokens and #"
+ + credentials.numberOfSecretKeys() + " secret keys for NM use for launching container");
+ containerCredentials.addAll(credentials);
DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
containerCredentials.writeTokenStorageToStream(containerTokens_dob);
- taskCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0,
+ containerCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0,
containerTokens_dob.getLength());
// Add shuffle token
LOG.info("Putting shuffle token in serviceData");
serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
- ShuffleHandler.serializeServiceData(jobToken));
+ ShuffleHandler.serializeServiceData(TokenCache.getJobToken(containerCredentials)));
} catch (IOException e) {
throw new TezUncheckedException(e);
}
@@ -130,7 +130,7 @@ public class AMContainerHelpers {
// container separately.
ContainerLaunchContext container =
ContainerLaunchContext.newInstance(localResources, environment, null,
- serviceData, taskCredentialsBuffer, applicationACLs);
+ serviceData, containerCredentialsBuffer, applicationACLs);
return container;
}