You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/01/07 19:44:09 UTC
svn commit: r1556309 - in /hive/trunk:
ql/src/java/org/apache/hadoop/hive/ql/udf/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
shims/0.20/src/main/java/org/apache/hadoop/hive/shims/
shims/0.20S/src/main/java/org/apache/hadoop...
Author: gunther
Date: Tue Jan 7 18:44:08 2014
New Revision: 1556309
URL: http://svn.apache.org/r1556309
Log:
HIVE-6105: LongWritable.compareTo needs shimming (Navis vis Gunther Hagleitner)
Added:
hive/trunk/ql/src/test/queries/clientpositive/udaf_percentile.q
hive/trunk/ql/src/test/results/clientpositive/udaf_percentile.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java
hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java?rev=1556309&r1=1556308&r2=1556309&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java Tue Jan 7 18:44:08 2014
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.De
import org.apache.hadoop.hive.ql.exec.UDAF;
import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.LongWritable;
/**
@@ -44,6 +45,12 @@ import org.apache.hadoop.io.LongWritable
+ "pc can be a double or double array")
public class UDAFPercentile extends UDAF {
+ private static final Comparator<LongWritable> COMPARATOR;
+
+ static {
+ COMPARATOR = ShimLoader.getHadoopShims().getLongComparator();
+ }
+
/**
* A state class to store intermediate aggregation results.
*/
@@ -59,7 +66,7 @@ public class UDAFPercentile extends UDAF
@Override
public int compare(Map.Entry<LongWritable, LongWritable> o1,
Map.Entry<LongWritable, LongWritable> o2) {
- return o1.getKey().compareTo(o2.getKey());
+ return COMPARATOR.compare(o1.getKey(), o2.getKey());
}
}
Added: hive/trunk/ql/src/test/queries/clientpositive/udaf_percentile.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/udaf_percentile.q?rev=1556309&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/udaf_percentile.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/udaf_percentile.q Tue Jan 7 18:44:08 2014
@@ -0,0 +1 @@
+select percentile(cast(key as bigint), 0.3) from src;
Added: hive/trunk/ql/src/test/results/clientpositive/udaf_percentile.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udaf_percentile.q.out?rev=1556309&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udaf_percentile.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/udaf_percentile.q.out Tue Jan 7 18:44:08 2014
@@ -0,0 +1,9 @@
+PREHOOK: query: select percentile(cast(key as bigint), 0.3) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select percentile(cast(key as bigint), 0.3) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+169.0
Modified: hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1556309&r1=1556308&r2=1556309&view=diff
==============================================================================
--- hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Tue Jan 7 18:44:08 2014
@@ -29,6 +29,7 @@ import java.security.PrivilegedActionExc
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -47,6 +48,7 @@ import org.apache.hadoop.fs.ProxyFileSys
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.FileInputFormat;
@@ -160,6 +162,16 @@ public class Hadoop20Shims implements Ha
TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile);
}
+ @Override
+ public Comparator<LongWritable> getLongComparator() {
+ return new Comparator<LongWritable>() {
+ @Override
+ public int compare(LongWritable o1, LongWritable o2) {
+ return o1.compareTo(o2);
+ }
+ };
+ }
+
public static class InputSplitShim extends CombineFileSplit implements HadoopShims.InputSplitShim {
long shrinkedLength;
boolean _isShrinked;
Modified: hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1556309&r1=1556308&r2=1556309&view=diff
==============================================================================
--- hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Tue Jan 7 18:44:08 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.Comparator;
import java.util.Iterator;
import java.net.URI;
@@ -34,6 +35,7 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.ProxyFileSystem;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapred.JobTracker;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.ClusterStatus;
@@ -150,6 +152,16 @@ public class Hadoop20SShims extends Hado
TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile);
}
+ @Override
+ public Comparator<LongWritable> getLongComparator() {
+ return new Comparator<LongWritable>() {
+ @Override
+ public int compare(LongWritable o1, LongWritable o2) {
+ return o1.compareTo(o2);
+ }
+ };
+ }
+
/**
* Returns a shim to wrap MiniMrCluster
*/
Modified: hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1556309&r1=1556308&r2=1556309&view=diff
==============================================================================
--- hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Tue Jan 7 18:44:08 2014
@@ -22,6 +22,7 @@ import java.lang.Integer;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
import java.net.URI;
@@ -39,6 +40,7 @@ import org.apache.hadoop.fs.ProxyFileSys
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
@@ -166,6 +168,16 @@ public class Hadoop23Shims extends Hadoo
TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile);
}
+ @Override
+ public Comparator<LongWritable> getLongComparator() {
+ return new Comparator<LongWritable>() {
+ @Override
+ public int compare(LongWritable o1, LongWritable o2) {
+ return o1.compareTo(o2);
+ }
+ };
+ }
+
/**
* Returns a shim to wrap MiniMrCluster
*/
Modified: hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1556309&r1=1556308&r2=1556309&view=diff
==============================================================================
--- hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Tue Jan 7 18:44:08 2014
@@ -25,6 +25,7 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
@@ -38,17 +39,14 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapred.ClusterStatus;
-import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobProfile;
import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobID;
@@ -354,6 +352,9 @@ public interface HadoopShims {
* @param partition
*/
void setTotalOrderPartitionFile(JobConf jobConf, Path partition);
+
+ Comparator<LongWritable> getLongComparator();
+
/**
* InputSplitShim.
*