You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2014/06/02 20:58:06 UTC
svn commit: r1599308 - in /hive/branches/branch-0.13:
ql/src/java/org/apache/hadoop/hive/ql/exec/
shims/0.20/src/main/java/org/apache/hadoop/hive/shims/
shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/
shims/0.23/src/main/java/org/apache/hadoop/...
Author: khorgath
Date: Mon Jun 2 18:58:05 2014
New Revision: 1599308
URL: http://svn.apache.org/r1599308
Log:
HIVE-7116 : HDFS FileSystem object cache causes permission issues in creating tmp directories (Vikram Dixit K via Thejas Nair)
(from https://issues.apache.org/jira/secure/attachment/12647832/HIVE-7116.branch0.13.patch)
Modified:
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1599308&r1=1599307&r2=1599308&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Mon Jun 2 18:58:05 2014
@@ -3396,9 +3396,24 @@ public final class Utilities {
return createDirsWithPermission(conf, mkdir, fsPermission, recursive);
}
- public static boolean createDirsWithPermission(Configuration conf, Path mkdir,
+ private static void resetConfAndCloseFS (Configuration conf, boolean unsetUmask,
+ String origUmask, FileSystem fs) throws IOException {
+ if (unsetUmask) {
+ if (origUmask != null) {
+ conf.set("fs.permissions.umask-mode", origUmask);
+ } else {
+ conf.unset("fs.permissions.umask-mode");
+ }
+ }
+
+ fs.close();
+ }
+
+ public static boolean createDirsWithPermission(Configuration conf, Path mkdirPath,
FsPermission fsPermission, boolean recursive) throws IOException {
String origUmask = null;
+ LOG.debug("Create dirs " + mkdirPath + " with permission " + fsPermission + " recursive " +
+ recursive);
if (recursive) {
origUmask = conf.get("fs.permissions.umask-mode");
@@ -3406,15 +3421,17 @@ public final class Utilities {
// all parents getting the fsPermission & !(022) permission instead of fsPermission
conf.set("fs.permissions.umask-mode", "000");
}
-
- FileSystem fs = mkdir.getFileSystem(conf);
- boolean retval = fs.mkdirs(mkdir, fsPermission);
-
- if (recursive) {
- if (origUmask != null) {
- conf.set("fs.permissions.umask-mode", origUmask);
- } else {
- conf.unset("fs.permissions.umask-mode");
+ FileSystem fs = ShimLoader.getHadoopShims().getNonCachedFileSystem(mkdirPath.toUri(), conf);
+ boolean retval = false;
+ try {
+ retval = fs.mkdirs(mkdirPath, fsPermission);
+ resetConfAndCloseFS(conf, recursive, origUmask, fs);
+ } catch (IOException ioe) {
+ try {
+ resetConfAndCloseFS(conf, recursive, origUmask, fs);
+ }
+ catch (IOException e) {
+ // do nothing - double failure
}
}
return retval;
Modified: hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1599308&r1=1599307&r2=1599308&view=diff
==============================================================================
--- hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Mon Jun 2 18:58:05 2014
@@ -31,12 +31,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
-import java.util.Set;
-import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import javax.security.auth.Subject;
import javax.security.auth.login.LoginException;
@@ -53,25 +52,17 @@ import org.apache.hadoop.fs.ProxyFileSys
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectDecompressorShim;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.OutputCommitter;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskAttemptContext;
-import org.apache.hadoop.mapred.TaskCompletionEvent;
-import org.apache.hadoop.mapred.TaskID;
import org.apache.hadoop.mapred.TaskLogServlet;
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
import org.apache.hadoop.mapred.lib.CombineFileSplit;
@@ -800,4 +791,15 @@ public class Hadoop20Shims implements Ha
public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext context) {
return context.getConfiguration();
}
-}
+
+ @Override
+ public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException {
+ boolean origDisableHDFSCache =
+ conf.getBoolean("fs." + uri.getScheme() + ".impl.disable.cache", false);
+ // hadoop-20 compatible flag.
+ conf.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true);
+ FileSystem fs = FileSystem.get(uri, conf);
+ conf.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", origDisableHDFSCache);
+ return fs;
+ }
+}
\ No newline at end of file
Modified: hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1599308&r1=1599307&r2=1599308&view=diff
==============================================================================
--- hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Mon Jun 2 18:58:05 2014
@@ -420,4 +420,15 @@ public class Hadoop20SShims extends Hado
public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext context) {
return context.getConfiguration();
}
+
+ @Override
+ public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException {
+ boolean origDisableHDFSCache =
+ conf.getBoolean("fs." + uri.getScheme() + ".impl.disable.cache", false);
+ // hadoop-1 compatible flag.
+ conf.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true);
+ FileSystem fs = FileSystem.get(uri, conf);
+ conf.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", origDisableHDFSCache);
+ return fs;
+ }
}
Modified: hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1599308&r1=1599307&r2=1599308&view=diff
==============================================================================
--- hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Mon Jun 2 18:58:05 2014
@@ -17,20 +17,17 @@
*/
package org.apache.hadoop.hive.shims;
+import java.io.FileNotFoundException;
import java.io.IOException;
-import java.lang.Integer;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URL;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Comparator;
-import java.util.Iterator;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.HashMap;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.io.FileNotFoundException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -46,17 +43,13 @@ import org.apache.hadoop.fs.ProxyFileSys
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hive.shims.HadoopShims.ByteBufferPoolShim;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectCompressionType;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectDecompressorShim;
-import org.apache.hadoop.hive.shims.HadoopShims.ZeroCopyReaderShim;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.WebHCatJTShim23;
+import org.apache.hadoop.mapred.lib.TotalOrderPartitioner;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobID;
@@ -69,9 +62,8 @@ import org.apache.hadoop.mapreduce.task.
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.mapreduce.util.HostUtil;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.mapred.lib.TotalOrderPartitioner;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
import org.apache.tez.test.MiniTezCluster;
/**
@@ -576,4 +568,9 @@ public class Hadoop23Shims extends Hadoo
public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext context) {
return context.getConfiguration();
}
+
+ @Override
+ public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException {
+ return FileSystem.newInstance(uri, conf);
+ }
}
Modified: hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1599308&r1=1599307&r2=1599308&view=diff
==============================================================================
--- hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Mon Jun 2 18:58:05 2014
@@ -34,30 +34,23 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier;
import org.apache.hadoop.hive.thrift.DelegationTokenSelector;
import org.apache.hadoop.http.HtmlQuoting;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.OutputCommitter;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskAttemptContext;
-import org.apache.hadoop.mapred.TaskCompletionEvent;
-import org.apache.hadoop.mapred.TaskID;
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
import org.apache.hadoop.mapred.lib.CombineFileSplit;
import org.apache.hadoop.mapreduce.Job;
@@ -589,7 +582,8 @@ public abstract class HadoopShimsSecure
abstract public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception;
@Override
- abstract public org.apache.hadoop.mapreduce.TaskAttemptContext newTaskAttemptContext(Configuration conf, final Progressable progressable);
+ abstract public org.apache.hadoop.mapreduce.TaskAttemptContext newTaskAttemptContext(
+ Configuration conf, final Progressable progressable);
@Override
abstract public org.apache.hadoop.mapreduce.JobContext newJobContext(Job job);
@@ -614,8 +608,11 @@ public abstract class HadoopShimsSecure
@Override
abstract public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
- throws IOException;
+ throws IOException;
@Override
abstract public FileSystem createProxyFileSystem(FileSystem fs, URI uri);
+
+ @Override
+ abstract public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException;
}
Modified: hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1599308&r1=1599307&r2=1599308&view=diff
==============================================================================
--- hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Mon Jun 2 18:58:05 2014
@@ -27,7 +27,6 @@ import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.Comparator;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -628,4 +627,7 @@ public interface HadoopShims {
* Get configuration from JobContext
*/
public Configuration getConfiguration(JobContext context);
+
+ public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException;
+
}