You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2013/10/21 17:48:08 UTC
svn commit: r1534240 - in /hive/trunk/shims/src:
0.20/java/org/apache/hadoop/hive/shims/
0.20S/java/org/apache/hadoop/hive/shims/
0.23/java/org/apache/hadoop/hive/shims/
common-secure/java/org/apache/hadoop/hive/shims/
common/java/org/apache/hadoop/fs/...
Author: brock
Date: Mon Oct 21 15:48:08 2013
New Revision: 1534240
URL: http://svn.apache.org/r1534240
Log:
HIVE-5070 - Implement listLocatedStatus() in ProxyFileSystem for 0.23 shim (shanyu zhao via Brock Noland)
Modified:
hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java
hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java
hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Mon Oct 21 15:48:08 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.ProxyFileSystem;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
@@ -783,4 +784,8 @@ public class Hadoop20Shims implements Ha
public WebHCatJTShim getWebHCatShim(Configuration conf, UserGroupInformation ugi) throws IOException {
throw new UnsupportedOperationException("WebHCat does not support Hadoop 0.20.x");
}
+ @Override
+ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
+ return new ProxyFileSystem(fs, uri);
+ }
}
Modified: hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Mon Oct 21 15:48:08 2013
@@ -22,6 +22,7 @@ import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
+import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
@@ -30,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.ProxyFileSystem;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.JobTracker;
@@ -372,4 +374,8 @@ public class Hadoop20SShims extends Hado
return fs.getFileBlockLocations(status, 0, status.getLen());
}
+ @Override
+ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
+ return new ProxyFileSystem(fs, uri);
+ }
}
Modified: hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Mon Oct 21 15:48:08 2013
@@ -24,6 +24,8 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.Map;
+import java.net.URI;
+import java.io.FileNotFoundException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -33,6 +35,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.ProxyFileSystem;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -407,4 +412,40 @@ public class Hadoop23Shims extends Hadoo
}
}
+ class ProxyFileSystem23 extends ProxyFileSystem {
+ public ProxyFileSystem23(FileSystem fs) {
+ super(fs);
+ }
+ public ProxyFileSystem23(FileSystem fs, URI uri) {
+ super(fs, uri);
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
+ throws FileNotFoundException, IOException {
+ return new RemoteIterator<LocatedFileStatus>() {
+ private RemoteIterator<LocatedFileStatus> stats =
+ ProxyFileSystem23.super.listLocatedStatus(
+ ProxyFileSystem23.super.swizzleParamPath(f));
+
+ @Override
+ public boolean hasNext() throws IOException {
+ return stats.hasNext();
+ }
+
+ @Override
+ public LocatedFileStatus next() throws IOException {
+ LocatedFileStatus result = stats.next();
+ return new LocatedFileStatus(
+ ProxyFileSystem23.super.swizzleFileStatus(result, false),
+ result.getBlockLocations());
+ }
+ };
+ }
+ }
+
+ @Override
+ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
+ return new ProxyFileSystem23(fs, uri);
+ }
}
Modified: hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Mon Oct 21 15:48:08 2013
@@ -622,4 +622,7 @@ public abstract class HadoopShimsSecure
@Override
abstract public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
throws IOException;
+
+ @Override
+ abstract public FileSystem createProxyFileSystem(FileSystem fs, URI uri);
}
Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java Mon Oct 21 15:48:08 2013
@@ -45,7 +45,7 @@ public class ProxyFileSystem extends Fil
- private Path swizzleParamPath(Path p) {
+ protected Path swizzleParamPath(Path p) {
String pathUriString = p.toUri().toString();
URI newPathUri = URI.create(pathUriString);
return new Path (realScheme, realAuthority, newPathUri.getPath());
@@ -57,7 +57,7 @@ public class ProxyFileSystem extends Fil
return new Path (myScheme, myAuthority, newPathUri.getPath());
}
- private FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) {
+ protected FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) {
FileStatus ret =
new FileStatus(orig.getLen(), orig.isDir(), orig.getReplication(),
orig.getBlockSize(), orig.getModificationTime(),
Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java Mon Oct 21 15:48:08 2013
@@ -23,6 +23,8 @@ import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.shims.HadoopShims;
/****************************************************************
* A Proxy for LocalFileSystem
@@ -61,7 +63,9 @@ public class ProxyLocalFileSystem extend
String authority = name.getAuthority() != null ? name.getAuthority() : "";
String proxyUriString = nameUriString + "://" + authority + "/";
- fs = new ProxyFileSystem(localFs, URI.create(proxyUriString));
+
+ fs = ShimLoader.getHadoopShims().createProxyFileSystem(
+ localFs, URI.create(proxyUriString));
fs.initialize(name, conf);
}
Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Mon Oct 21 15:48:08 2013
@@ -562,4 +562,10 @@ public interface HadoopShims {
*/
public void close();
}
+
+ /**
+ * Create a proxy file system that can serve a given scheme/authority using some
+ * other file system.
+ */
+ public FileSystem createProxyFileSystem(FileSystem fs, URI uri);
}