You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/03/28 20:08:09 UTC
git commit: TEZ-990. Add support for a credentials file read by
TezClient. (sseth)
Repository: incubator-tez
Updated Branches:
refs/heads/master e32252cdc -> 6416102f4
TEZ-990. Add support for a credentials file read by TezClient. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/6416102f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/6416102f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/6416102f
Branch: refs/heads/master
Commit: 6416102f4ffbcde66e3bf4595d268d2539019c2e
Parents: e32252c
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Mar 28 12:07:48 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri Mar 28 12:07:48 2014 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/tez/client/TezClient.java | 1 +
.../main/java/org/apache/tez/client/TezClientUtils.java | 12 ++++++++++++
.../src/main/java/org/apache/tez/client/TezSession.java | 3 +++
.../main/java/org/apache/tez/common/TezJobConfig.java | 5 +++++
.../org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java | 2 ++
.../java/org/apache/tez/mapreduce/hadoop/MRHelpers.java | 4 ++++
6 files changed, 27 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 61de7e0..f4c6011 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -96,6 +96,7 @@ public class TezClient {
if (credentials == null) {
credentials = new Credentials();
}
+ TezClientUtils.processTezLocalCredentialsFile(credentials, conf);
// Add session token for shuffle
TezClientUtils.createSessionToken(appId.toString(),
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index ca37313..8b87fb0 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
+import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezYARNUtils;
import org.apache.tez.common.impl.LogUtils;
import org.apache.tez.common.security.JobTokenIdentifier;
@@ -198,6 +199,16 @@ public class TezClientUtils {
return tezJarResources;
}
+ static void processTezLocalCredentialsFile(Credentials credentials, Configuration conf)
+ throws IOException {
+ String path = conf.get(TezJobConfig.TEZ_CREDENTIALS_PATH);
+ if (path == null) {
+ return;
+ } else {
+ TokenCache.mergeBinaryTokens(credentials, conf, path);
+ }
+ }
+
/**
* Verify or create the Staging area directory on the configured Filesystem
* @param stagingArea Staging area directory path
@@ -316,6 +327,7 @@ public class TezClientUtils {
throws IOException, YarnException{
Preconditions.checkNotNull(sessionCreds);
+
FileSystem fs = TezClientUtils.ensureStagingDirExists(conf,
amConfig.getStagingDir());
Path binaryConfPath = new Path(amConfig.getStagingDir(),
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/client/TezSession.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezSession.java b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
index 055218e..ff7f00a 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezSession.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
@@ -94,6 +94,9 @@ public class TezSession {
yarnClient = YarnClient.createYarnClient();
yarnClient.init(sessionConfig.getYarnConfiguration());
yarnClient.start();
+
+ TezClientUtils.processTezLocalCredentialsFile(sessionCredentials,
+ sessionConfig.getTezConfiguration());
Map<String, LocalResource> tezJarResources =
TezClientUtils.setupTezJarsLocalResources(
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
index a45753b..0af0ffb 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -344,4 +344,9 @@ public class TezJobConfig {
/** Defines the ProcessTree implementation which will be used to collect resource utilization. */
public static final String TEZ_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS = "tez.resource.calculator.process-tree.class";
+
+ /**
+ * Path to a credentials file located on the local file system with serialized credentials
+ */
+ public static final String TEZ_CREDENTIALS_PATH = "tez.credentials.path";
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 621a1b8..f1c5e14 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -191,6 +191,8 @@ public class DeprecatedKeys {
registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS);
registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_RUNTIME_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS);
+
+ registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezJobConfig.TEZ_CREDENTIALS_PATH);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index d92a563..7538ccf 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -1026,6 +1026,10 @@ public class MRHelpers {
* Uses "mapreduce.job.credentials.binary" property to find location of token file.
* @param creds Credentials object to add new tokens to
* @param conf Configuration containing location of token file.
+ *
+ * TezClient reads credentials from the property - TezJobConfig.TEZ_CREDENTIALS_PATH. This method
+ * is not required if that property is set.
+ *
* @throws IOException
*/
public static void mergeMRBinaryTokens(Credentials creds,