You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/08/08 02:02:38 UTC
svn commit: r1511546 - in /hadoop/common/branches/branch-1-win: ./
src/core/org/apache/hadoop/security/ src/core/org/apache/hadoop/util/
src/mapred/org/apache/hadoop/mapred/ src/tools/org/apache/hadoop/tools/
Author: cnauroth
Date: Thu Aug 8 00:02:37 2013
New Revision: 1511546
URL: http://svn.apache.org/r1511546
Log:
HADOOP-9790. Job token path is not unquoted properly. Contributed by Xi Fang.
Modified:
hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Thu Aug 8 00:02:37 2013
@@ -323,6 +323,8 @@ Branch-hadoop-1-win (branched from branc
HADOOP-9609. Remove sh dependency of bin-package target. (Chuan Liu via
cnauroth)
+ HADOOP-9790. Job token path is not unquoted properly. (Xi Fang via cnauroth)
+
Merged from branch-1
HDFS-385. Backport: Add support for an experimental API that allows a
Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java (original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java Thu Aug 8 00:02:37 2013
@@ -543,6 +543,7 @@ public class UserGroupInformation {
if (fileLocation != null && isSecurityEnabled()) {
// load the token storage file and put all of the tokens into the
// user.
+ fileLocation = Shell.preprocessEnvVar(fileLocation);
Credentials cred = Credentials.readTokenStorageFile(
new Path("file:///" + fileLocation), conf);
for (Token<?> token: cred.getAllTokens()) {
Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java (original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java Thu Aug 8 00:02:37 2013
@@ -156,6 +156,14 @@ abstract public class Shell {
return winUtilsPath;
}
+ public static final String preprocessEnvVar(String env) {
+ /* trim leading and trailing quotes of an environment variable if the OS is Windows */
+ if (Shell.WINDOWS && env != null) {
+ env = env.replaceAll("^\"|\"$", "");
+ }
+ return env;
+ }
+
/** a Unix command to get the current user's name */
public final static String USER_NAME_COMMAND = "whoami";
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Child.java Thu Aug 8 00:02:37 2013
@@ -90,12 +90,7 @@ class Child {
// file name is passed thru env
String jobTokenFile =
System.getenv().get(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
- if(Shell.WINDOWS) {
- if(jobTokenFile.charAt(0)=='"')
- jobTokenFile = jobTokenFile.substring(1);
- if(jobTokenFile.charAt(jobTokenFile.length()-1) == '"')
- jobTokenFile = jobTokenFile.substring(0, jobTokenFile.length()-1);
- }
+ jobTokenFile = Shell.preprocessEnvVar(jobTokenFile);
Credentials credentials =
TokenCache.loadTokens(jobTokenFile, defaultConf);
LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() +
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Thu Aug 8 00:02:37 2013
@@ -515,11 +515,7 @@ abstract class TaskRunner extends Thread
// if temp directory path is not absolute, prepend it with workDir.
if (!tmpDir.isAbsolute()) {
- if (Shell.WINDOWS)
- // trim leading and trailing quotes on Windows
- tmpDir = new Path(workDir.toString().replaceAll("^\"|\"$", ""), tmp);
- else
- tmpDir = new Path(workDir.toString(), tmp);
+ tmpDir = new Path(Shell.preprocessEnvVar(workDir.toString()), tmp);
if (createDir) {
FileSystem localFs = FileSystem.getLocal(conf);
if (!localFs.mkdirs(tmpDir) &&
@@ -808,11 +804,7 @@ abstract class TaskRunner extends Thread
private static void symlink(File workDir, String target, String link)
throws IOException {
if (link != null) {
- if (Shell.WINDOWS)
- // trim leading and trailing quotes on Windows
- link = workDir.toString().replaceAll("^\"|\"$", "") + File.separator + link;
- else
- link = workDir.toString() + File.separator + link;
+ link = Shell.preprocessEnvVar(workDir.toString()) + File.separator + link;
File flink = new File(link);
if (!flink.exists()) {
LOG.info(String.format("Creating symlink: %s <- %s", target, link));
Modified: hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java?rev=1511546&r1=1511545&r2=1511546&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/common/branches/branch-1-win/src/tools/org/apache/hadoop/tools/DistCp.java Thu Aug 8 00:02:37 2013
@@ -67,6 +67,7 @@ import org.apache.hadoop.mapred.Sequence
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -982,6 +983,7 @@ public class DistCp implements Tool {
// Propagate delegation related props to DistCp job
String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
if (tokenFile != null) {
+ tokenFile = Shell.preprocessEnvVar(tokenFile);
LOG
.info("Setting env property for mapreduce.job.credentials.binary to: "
+ tokenFile);