You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/12/05 02:59:23 UTC
svn commit: r1417267 - in /hive/branches/branch-0.9:
metastore/src/java/org/apache/hadoop/hive/metastore/
shims/src/0.20/java/org/apache/hadoop/hive/shims/
shims/src/0.20S/java/org/apache/hadoop/hive/shims/
shims/src/0.23/java/org/apache/hadoop/hive/sh...
Author: hashutosh
Date: Wed Dec 5 01:59:19 2012
New Revision: 1417267
URL: http://svn.apache.org/viewvc?rev=1417267&view=rev
Log:
HIVE-3648 : HiveMetaStoreFsImpl is not compatible with hadoop viewfs (Arup Malakar via Ashutosh Chauhan)
Modified:
hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java (original)
+++ hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java Wed Dec 5 01:59:19 2012
@@ -25,8 +25,9 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
public class HiveMetaStoreFsImpl implements MetaStoreFS {
@@ -37,16 +38,10 @@ public class HiveMetaStoreFsImpl impleme
public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
Configuration conf) throws MetaException {
LOG.info("deleting " + f);
-
- // older versions of Hadoop don't have a Trash constructor based on the
- // Path or FileSystem. So need to achieve this by creating a dummy conf.
- // this needs to be filtered out based on version
- Configuration dupConf = new Configuration(conf);
- FileSystem.setDefaultUri(dupConf, fs.getUri());
+ HadoopShims hadoopShim = ShimLoader.getHadoopShims();
try {
- Trash trashTmp = new Trash(dupConf);
- if (trashTmp.moveToTrash(f)) {
+ if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) {
LOG.info("Moved to trash: " + f);
return true;
}
Modified: hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Wed Dec 5 01:59:19 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
import org.apache.hadoop.io.Text;
@@ -603,4 +604,17 @@ public class Hadoop20Shims implements Ha
// No such functionality in ancient hadoop
return;
}
+
+ @Override
+ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
+ throws IOException {
+ // older versions of Hadoop don't have a Trash constructor based on the
+ // Path or FileSystem. So need to achieve this by creating a dummy conf.
+ // this needs to be filtered out based on version
+
+ Configuration dupConf = new Configuration(conf);
+ FileSystem.setDefaultUri(dupConf, fs.getUri());
+ Trash trash = new Trash(dupConf);
+ return trash.moveToTrash(path);
+ }
}
Modified: hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Wed Dec 5 01:59:19 2012
@@ -17,12 +17,14 @@
*/
package org.apache.hadoop.hive.shims;
+import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hive.shims.HadoopShimsSecure;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapreduce.Job;
@@ -97,6 +99,19 @@ public class Hadoop20SShims extends Hado
}
@Override
+ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
+ throws IOException {
+ // older versions of Hadoop don't have a Trash constructor based on the
+ // Path or FileSystem. So need to achieve this by creating a dummy conf.
+ // this needs to be filtered out based on version
+
+ Configuration dupConf = new Configuration(conf);
+ FileSystem.setDefaultUri(dupConf, fs.getUri());
+ Trash trash = new Trash(dupConf);
+ return trash.moveToTrash(path);
+ }
+
+ @Override
public long getDefaultBlockSize(FileSystem fs, Path path) {
return fs.getDefaultBlockSize();
}
@@ -104,5 +119,5 @@ public class Hadoop20SShims extends Hado
@Override
public short getDefaultReplication(FileSystem fs, Path path) {
return fs.getDefaultReplication();
- }
+ }
}
Modified: hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Wed Dec 5 01:59:19 2012
@@ -17,12 +17,14 @@
*/
package org.apache.hadoop.hive.shims;
+import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hive.shims.HadoopShims.JobTrackerState;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobConf;
@@ -123,4 +125,9 @@ public class Hadoop23Shims extends Hadoo
return fs.getDefaultReplication(path);
}
+ @Override
+ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
+ throws IOException {
+ return Trash.moveToAppropriateTrash(fs, path, conf);
+ }
}
Modified: hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Wed Dec 5 01:59:19 2012
@@ -555,6 +555,10 @@ public abstract class HadoopShimsSecure
abstract public String getJobLauncherRpcAddress(Configuration conf);
@Override
+ abstract public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
+ throws IOException;
+
+ @Override
abstract public short getDefaultReplication(FileSystem fs, Path path);
@Override
Modified: hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Wed Dec 5 01:59:19 2012
@@ -275,7 +275,7 @@ public interface HadoopShims {
* @return
*/
public String getJobLauncherHttpAddress(Configuration conf);
-
+
/**
* Get the default block size for the path. FileSystem alone is not sufficient to
* determine the same, as in case of CSMT the underlying file system determines that.
@@ -293,11 +293,22 @@ public interface HadoopShims {
* @return
*/
public short getDefaultReplication(FileSystem fs, Path path);
-
/**
- * InputSplitShim.
- *
+ * Move the directory/file to trash. In case of the symlinks or mount points, the file is
+ * moved to the trashbin in the actual volume of the path p being deleted
+ * @param fs
+ * @param path
+ * @param conf
+ * @return false if the item is already in the trash or trash is disabled
+ * @throws IOException
*/
+ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
+ throws IOException;
+
+ /**
+ * InputSplitShim.
+ *
+ */
public interface InputSplitShim extends InputSplit {
JobConf getJob();