You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/12/04 23:12:28 UTC
svn commit: r1417220 - in /hive/branches/branch-0.9:
ql/src/java/org/apache/hadoop/hive/ql/io/
shims/src/0.20/java/org/apache/hadoop/hive/shims/
shims/src/0.20S/java/org/apache/hadoop/hive/shims/
shims/src/0.23/java/org/apache/hadoop/hive/shims/ shims/...
Author: hashutosh
Date: Tue Dec 4 22:12:27 2012
New Revision: 1417220
URL: http://svn.apache.org/viewvc?rev=1417220&view=rev
Log:
HIVE-3645 : RCFileWriter does not implement the right function to support Federation (Arup Malakar via Ashutosh Chauhan)
Modified:
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java Tue Dec 4 22:12:27 2012
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.serde2.Col
import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
import org.apache.hadoop.hive.serde2.columnar.LazyDecompressionCallback;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile.Metadata;
@@ -721,7 +722,8 @@ public class RCFile {
public Writer(FileSystem fs, Configuration conf, Path name,
Progressable progress, Metadata metadata, CompressionCodec codec) throws IOException {
this(fs, conf, name, fs.getConf().getInt("io.file.buffer.size", 4096),
- fs.getDefaultReplication(), fs.getDefaultBlockSize(), progress,
+ ShimLoader.getHadoopShims().getDefaultReplication(fs, name),
+ ShimLoader.getHadoopShims().getDefaultBlockSize(fs, name), progress,
metadata, codec);
}
Modified: hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Tue Dec 4 22:12:27 2012
@@ -589,6 +589,16 @@ public class Hadoop20Shims implements Ha
}
@Override
+ public long getDefaultBlockSize(FileSystem fs, Path path) {
+ return fs.getDefaultBlockSize();
+ }
+
+ @Override
+ public short getDefaultReplication(FileSystem fs, Path path) {
+ return fs.getDefaultReplication();
+ }
+
+ @Override
public void closeAllForUGI(UserGroupInformation ugi) {
// No such functionality in ancient hadoop
return;
Modified: hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Tue Dec 4 22:12:27 2012
@@ -21,6 +21,8 @@ import java.net.MalformedURLException;
import java.net.URL;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.shims.HadoopShimsSecure;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapreduce.Job;
@@ -93,4 +95,14 @@ public class Hadoop20SShims extends Hado
public String getJobLauncherHttpAddress(Configuration conf) {
return conf.get("mapred.job.tracker.http.address");
}
+
+ @Override
+ public long getDefaultBlockSize(FileSystem fs, Path path) {
+ return fs.getDefaultBlockSize();
+ }
+
+ @Override
+ public short getDefaultReplication(FileSystem fs, Path path) {
+ return fs.getDefaultReplication();
+ }
}
Modified: hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Tue Dec 4 22:12:27 2012
@@ -21,6 +21,8 @@ import java.net.MalformedURLException;
import java.net.URL;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.shims.HadoopShims.JobTrackerState;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobConf;
@@ -111,4 +113,14 @@ public class Hadoop23Shims extends Hadoo
return conf.get("yarn.resourcemanager.webapp.address");
}
+ @Override
+ public long getDefaultBlockSize(FileSystem fs, Path path) {
+ return fs.getDefaultBlockSize(path);
+ }
+
+ @Override
+ public short getDefaultReplication(FileSystem fs, Path path) {
+ return fs.getDefaultReplication(path);
+ }
+
}
Modified: hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Tue Dec 4 22:12:27 2012
@@ -553,4 +553,10 @@ public abstract class HadoopShimsSecure
@Override
abstract public String getJobLauncherRpcAddress(Configuration conf);
+
+ @Override
+ abstract public short getDefaultReplication(FileSystem fs, Path path);
+
+ @Override
+ abstract public long getDefaultBlockSize(FileSystem fs, Path path);
}
Modified: hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Tue Dec 4 22:12:27 2012
@@ -277,6 +277,24 @@ public interface HadoopShims {
public String getJobLauncherHttpAddress(Configuration conf);
/**
+ * Get the default block size for the path. FileSystem alone is not sufficient to
+ * determine the same, as in case of CSMT the underlying file system determines that.
+ * @param fs
+ * @param path
+ * @return
+ */
+ public long getDefaultBlockSize(FileSystem fs, Path path);
+
+ /**
+ * Get the default replication for a path. In case of CSMT the given path will be used to
+ * locate the actual filesystem.
+ * @param fs
+ * @param path
+ * @return
+ */
+ public short getDefaultReplication(FileSystem fs, Path path);
+
+ /**
* InputSplitShim.
*
*/