You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2014/09/19 00:47:04 UTC
git commit: MAPREDUCE-6086. mapreduce.job.credentials.binary should
allow all URIs. (Zhihai Xu via kasha)
Repository: hadoop
Updated Branches:
refs/heads/trunk 1b4b46004 -> 52945a33c
MAPREDUCE-6086. mapreduce.job.credentials.binary should allow all URIs. (Zhihai Xu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52945a33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52945a33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52945a33
Branch: refs/heads/trunk
Commit: 52945a33cc3fcb3b961cce0e7e3ca01291f2a223
Parents: 1b4b460
Author: Karthik Kambatla <ka...@apache.org>
Authored: Thu Sep 18 15:41:55 2014 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Thu Sep 18 15:46:52 2014 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../apache/hadoop/mapreduce/JobSubmitter.java | 4 +-
.../hadoop/mapreduce/security/TokenCache.java | 4 +-
.../mapreduce/security/TestTokenCache.java | 19 ++++-
.../mapreduce/security/TestBinaryTokenFile.java | 86 ++++++++++++++------
5 files changed, 86 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52945a33/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a661266..0c6bbf7 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -353,6 +353,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6090. mapred hsadmin getGroups fails to connect in some cases
(Robert Kanter via jlowe)
+ MAPREDUCE-6086. mapreduce.job.credentials.binary should allow all URIs.
+ (Zhihai Xu via kasha)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52945a33/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index 0734e7f..6cd569a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -578,7 +578,9 @@ class JobSubmitter {
conf.get("mapreduce.job.credentials.binary");
if (binaryTokenFilename != null) {
Credentials binary = Credentials.readTokenStorageFile(
- new Path("file:///" + binaryTokenFilename), conf);
+ FileSystem.getLocal(conf).makeQualified(
+ new Path(binaryTokenFilename)),
+ conf);
credentials.addAll(binary);
}
// add secret keys coming from a json file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52945a33/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
index cadd04b..7b1f657 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
@@ -134,7 +134,9 @@ public class TokenCache {
Credentials binary;
try {
binary = Credentials.readTokenStorageFile(
- new Path("file:///" + binaryTokenFilename), conf);
+ FileSystem.getLocal(conf).makeQualified(
+ new Path(binaryTokenFilename)),
+ conf);
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52945a33/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
index de594d4..127f8ae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
@@ -63,12 +63,25 @@ public class TestTokenCache {
@Test
@SuppressWarnings("deprecation")
- public void testBinaryCredentials() throws Exception {
+ public void testBinaryCredentialsWithoutScheme() throws Exception {
+ testBinaryCredentials(false);
+ }
+
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testBinaryCredentialsWithScheme() throws Exception {
+ testBinaryCredentials(true);
+ }
+
+ private void testBinaryCredentials(boolean hasScheme) throws Exception {
Path TEST_ROOT_DIR =
new Path(System.getProperty("test.build.data","test/build/data"));
// ick, but need fq path minus file:/
- String binaryTokenFile = FileSystem.getLocal(conf).makeQualified(
- new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath();
+ String binaryTokenFile = hasScheme
+ ? FileSystem.getLocal(conf).makeQualified(
+ new Path(TEST_ROOT_DIR, "tokenFile")).toString()
+ : FileSystem.getLocal(conf).makeQualified(
+ new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath();
MockFileSystem fs1 = createFileSystemForServiceName("service1");
MockFileSystem fs2 = createFileSystemForServiceName("service2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52945a33/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
index b92400d..7a2c03b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
@@ -150,30 +150,15 @@ public class TestBinaryTokenFile {
// Credentials in the job will not have delegation tokens
// because security is disabled. Fetch delegation tokens
// and store in binary token file.
- try {
- Credentials cred1 = new Credentials();
- Credentials cred2 = new Credentials();
- TokenCache.obtainTokensForNamenodesInternal(cred1, new Path[] { p1 },
- job.getConfiguration());
- for (Token<? extends TokenIdentifier> t : cred1.getAllTokens()) {
- cred2.addToken(new Text(DELEGATION_TOKEN_KEY), t);
- }
- DataOutputStream os = new DataOutputStream(new FileOutputStream(
- binaryTokenFileName.toString()));
- try {
- cred2.writeTokenStorageToStream(os);
- } finally {
- os.close();
- }
- job.getConfiguration().set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY,
- binaryTokenFileName.toString());
- // NB: the MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY key now gets deleted from config,
- // so it's not accessible in the job's config. So, we use another key to pass the file name into the job configuration:
- job.getConfiguration().set(KEY_SECURITY_TOKEN_FILE_NAME,
- binaryTokenFileName.toString());
- } catch (IOException e) {
- Assert.fail("Exception " + e);
- }
+ createBinaryTokenFile(job.getConfiguration());
+ job.getConfiguration().set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY,
+ binaryTokenFileName.toString());
+ // NB: the MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY
+ // key now gets deleted from config,
+ // so it's not accessible in the job's config. So,
+ // we use another key to pass the file name into the job configuration:
+ job.getConfiguration().set(KEY_SECURITY_TOKEN_FILE_NAME,
+ binaryTokenFileName.toString());
}
}
@@ -225,7 +210,29 @@ public class TestBinaryTokenFile {
dfsCluster = null;
}
}
-
+
+ private static void createBinaryTokenFile(Configuration conf) {
+ // Fetch delegation tokens and store in binary token file.
+ try {
+ Credentials cred1 = new Credentials();
+ Credentials cred2 = new Credentials();
+ TokenCache.obtainTokensForNamenodesInternal(cred1, new Path[] { p1 },
+ conf);
+ for (Token<? extends TokenIdentifier> t : cred1.getAllTokens()) {
+ cred2.addToken(new Text(DELEGATION_TOKEN_KEY), t);
+ }
+ DataOutputStream os = new DataOutputStream(new FileOutputStream(
+ binaryTokenFileName.toString()));
+ try {
+ cred2.writeTokenStorageToStream(os);
+ } finally {
+ os.close();
+ }
+ } catch (IOException e) {
+ Assert.fail("Exception " + e);
+ }
+ }
+
/**
* run a distributed job and verify that TokenCache is available
* @throws IOException
@@ -252,4 +259,33 @@ public class TestBinaryTokenFile {
}
assertEquals("dist job res is not 0:", 0, res);
}
+
+ /**
+ * run a distributed job with -tokenCacheFile option parameter and
+ * verify that no exception happens.
+ * @throws IOException
+ */
+ @Test
+ public void testTokenCacheFile() throws IOException {
+ Configuration conf = mrCluster.getConfig();
+ createBinaryTokenFile(conf);
+ // provide namenodes names for the job to get the delegation tokens for
+ final String nnUri = dfsCluster.getURI(0).toString();
+ conf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
+
+ // using argument to pass the file name
+ final String[] args = {
+ "-tokenCacheFile", binaryTokenFileName.toString(),
+ "-m", "1", "-r", "1", "-mt", "1", "-rt", "1"
+ };
+ int res = -1;
+ try {
+ res = ToolRunner.run(conf, new SleepJob(), args);
+ } catch (Exception e) {
+ System.out.println("Job failed with " + e.getLocalizedMessage());
+ e.printStackTrace(System.out);
+ fail("Job failed");
+ }
+ assertEquals("dist job res is not 0:", 0, res);
+ }
}