You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/04/14 00:10:11 UTC
[43/44] hadoop git commit: MAPREDUCE-7077. Pipe mapreduce job fails
with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda)
MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda)
Change-Id: Ie8f01425d58409fa3661f768205b7616128c8aa4
(cherry picked from commit 035e0f97ea44b0495707949a781d8792dcf6ea6b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/995cba65
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/995cba65
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/995cba65
Branch: refs/heads/HDFS-7240
Commit: 995cba65fe29966583e36f9491d9a27b323918ae
Parents: 9031a76
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Apr 12 14:33:33 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Apr 13 13:52:10 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapred/pipes/Application.java | 5 ++---
.../hadoop/mapred/pipes/TestPipeApplication.java | 19 ++++++++-----------
2 files changed, 10 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
index 5c8aab9..83d2509 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapred.TaskLog;
-import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
@@ -104,8 +103,8 @@ class Application<K1 extends WritableComparable, V1 extends Writable,
// This password is used as shared secret key between this application and
// child pipes process
byte[] password = jobToken.getPassword();
- String localPasswordFile = new File(conf.get(MRConfig.LOCAL_DIR))
- + Path.SEPARATOR + "jobTokenPassword";
+ String localPasswordFile = new File(".") + Path.SEPARATOR
+ + "jobTokenPassword";
writePasswordToLocalFile(localPasswordFile, password, conf);
env.put("hadoop.pipes.shared.secret.location", localPasswordFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
index 13597e0..88d8f95 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.IFile.Writer;
-import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapred.Counters;
@@ -84,10 +83,10 @@ public class TestPipeApplication {
public void testRunner() throws Exception {
// clean old password files
- JobConf conf = new JobConf();
- File[] psw = cleanTokenPasswordFile(conf);
+ File[] psw = cleanTokenPasswordFile();
try {
RecordReader<FloatWritable, NullWritable> rReader = new ReaderPipesMapRunner();
+ JobConf conf = new JobConf();
conf.set(Submitter.IS_JAVA_RR, "true");
// for stdour and stderror
@@ -163,7 +162,7 @@ public class TestPipeApplication {
TestTaskReporter reporter = new TestTaskReporter();
- File[] psw = cleanTokenPasswordFile(conf);
+ File[] psw = cleanTokenPasswordFile();
try {
conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName);
@@ -248,7 +247,7 @@ public class TestPipeApplication {
JobConf conf = new JobConf();
- File[] psw = cleanTokenPasswordFile(conf);
+ File[] psw = cleanTokenPasswordFile();
System.setProperty("test.build.data",
"target/tmp/build/TEST_SUBMITTER_MAPPER/data");
@@ -389,8 +388,8 @@ public class TestPipeApplication {
@Test
public void testPipesReduser() throws Exception {
+ File[] psw = cleanTokenPasswordFile();
JobConf conf = new JobConf();
- File[] psw = cleanTokenPasswordFile(conf);
try {
Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>(
"user".getBytes(), "password".getBytes(), new Text("kind"), new Text(
@@ -507,16 +506,14 @@ public class TestPipeApplication {
}
- private File[] cleanTokenPasswordFile(JobConf conf) throws Exception {
+ private File[] cleanTokenPasswordFile() throws Exception {
File[] result = new File[2];
- result[0] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
- + "jobTokenPassword");
+ result[0] = new File("./jobTokenPassword");
if (result[0].exists()) {
FileUtil.chmod(result[0].getAbsolutePath(), "700");
assertTrue(result[0].delete());
}
- result[1] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
- + ".jobTokenPassword.crc");
+ result[1] = new File("./.jobTokenPassword.crc");
if (result[1].exists()) {
FileUtil.chmod(result[1].getAbsolutePath(), "700");
result[1].delete();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org