You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2014/03/28 18:58:34 UTC
git commit: TEZ-983. Support a helper function to extract required
additional tokens from a file. (hitesh)
Repository: incubator-tez
Updated Branches:
refs/heads/master cde19354d -> e32252cdc
TEZ-983. Support a helper function to extract required additional tokens from a file. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/e32252cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/e32252cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/e32252cd
Branch: refs/heads/master
Commit: e32252cdcf6e52c1f361c780a7373168242e7c6c
Parents: cde1935
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Mar 28 10:58:10 2014 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Mar 28 10:58:10 2014 -0700
----------------------------------------------------------------------
tez-api/pom.xml | 11 ++
.../apache/tez/common/security/TokenCache.java | 23 ++++
.../tez/common/security/TestTokenCache.java | 122 +++++++++++++++++++
.../apache/tez/mapreduce/hadoop/MRHelpers.java | 19 +++
4 files changed, 175 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e32252cd/tez-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index fe3c36b..75f5a62 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -53,6 +53,17 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e32252cd/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
index 6c0b1fc..d270fd4 100644
--- a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
+++ b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
@@ -137,4 +137,27 @@ public class TokenCache {
}
return (Token<JobTokenIdentifier>) token;
}
+
+ /**
+ * Merge tokens from a configured binary file into provided Credentials object
+ * @param creds Credentials object to add new tokens to
+ * @param tokenFilePath Location of tokens' binary file
+ */
+ @InterfaceAudience.Private
+ public static void mergeBinaryTokens(Credentials creds,
+ Configuration conf, String tokenFilePath)
+ throws IOException {
+ if (tokenFilePath == null || tokenFilePath.isEmpty()) {
+ throw new RuntimeException("Invalid file path provided"
+ + ", tokenFilePath=" + tokenFilePath);
+ }
+ LOG.info("Merging additional tokens from binary file"
+ + ", binaryFileName=" + tokenFilePath);
+ Credentials binary = Credentials.readTokenStorageFile(
+ new Path("file:///" + tokenFilePath), conf);
+
+ // supplement existing tokens with the tokens in the binary file
+ creds.mergeAll(binary);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e32252cd/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
new file mode 100644
index 0000000..4706e8a
--- /dev/null
+++ b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.security;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class TestTokenCache {
+
+ private static Configuration conf;
+
+ private static String renewer;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ conf = new Configuration();
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
+ renewer = Master.getMasterPrincipal(conf);
+ }
+
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testBinaryCredentials() throws Exception {
+ String binaryTokenFile = null;
+ try {
+ Path TEST_ROOT_DIR = new Path("target");
+ binaryTokenFile = FileSystem.getLocal(conf).makeQualified(
+ new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath();
+
+ MockFileSystem fs1 = createFileSystemForServiceName("service1");
+ MockFileSystem fs2 = createFileSystemForServiceName("service2");
+ MockFileSystem fs3 = createFileSystemForServiceName("service3");
+
+ // get the tokens for fs1 & fs2 and write out to binary creds file
+ Credentials creds = new Credentials();
+ Token<?> token1 = fs1.getDelegationToken(renewer);
+ Token<?> token2 = fs2.getDelegationToken(renewer);
+ creds.addToken(token1.getService(), token1);
+ creds.addToken(token2.getService(), token2);
+ creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
+
+
+ Credentials newCreds = new Credentials();
+ TokenCache.mergeBinaryTokens(newCreds, conf, binaryTokenFile);
+
+ Assert.assertTrue(newCreds.getAllTokens().size() > 0);
+ checkTokens(creds, newCreds);
+ } finally {
+ if (binaryTokenFile != null) {
+ try {
+ FileSystem.getLocal(conf).delete(new Path(binaryTokenFile));
+ } catch (IOException e) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ private MockFileSystem createFileSystemForServiceName(final String service)
+ throws IOException {
+ MockFileSystem mockFs = new MockFileSystem();
+ when(mockFs.getCanonicalServiceName()).thenReturn(service);
+ when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
+ new Answer<Token<?>>() {
+ int unique = 0;
+ @Override
+ public Token<?> answer(InvocationOnMock invocation) throws Throwable {
+ Token<?> token = new Token<TokenIdentifier>();
+ token.setService(new Text(service));
+ // use unique value so when we restore from token storage, we can
+ // tell if it's really the same token
+ token.setKind(new Text("token" + unique++));
+ return token;
+ }
+ });
+ return mockFs;
+ }
+
+ private void checkTokens(Credentials creds, Credentials newCreds) {
+ Assert.assertEquals(creds.getAllTokens().size(),
+ newCreds.getAllTokens().size());
+ for (Token<?> token : newCreds.getAllTokens()) {
+ Token<?> credsToken = creds.getToken(token.getService());
+ Assert.assertTrue(credsToken != null);
+ Assert.assertEquals(token, credsToken);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e32252cd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 3ed3702..d92a563 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
import org.apache.hadoop.mapreduce.split.JobSplitWriter;
import org.apache.hadoop.mapreduce.v2.proto.MRProtos;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.ContainerLogAppender;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -63,6 +64,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezUtils;
import org.apache.tez.common.TezYARNUtils;
+import org.apache.tez.common.security.TokenCache;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.TezConfiguration;
@@ -1018,4 +1020,21 @@ public class MRHelpers {
return Lists.newArrayList(iterable);
}
+
+ /**
+ * Merge tokens from a configured binary file into provided Credentials object.
+ * Uses "mapreduce.job.credentials.binary" property to find location of token file.
+ * @param creds Credentials object to add new tokens to
+ * @param conf Configuration containing location of token file.
+ * @throws IOException
+ */
+ public static void mergeMRBinaryTokens(Credentials creds,
+ Configuration conf) throws IOException {
+ String tokenFilePath = conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY);
+ if (tokenFilePath == null || tokenFilePath.isEmpty()) {
+ return;
+ }
+ TokenCache.mergeBinaryTokens(creds, conf, tokenFilePath);
+ }
+
}