You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/03/24 02:00:53 UTC
svn commit: r926889 - in /hadoop/hive/trunk: CHANGES.txt
ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java
ql/src/test/queries/clientpositive/udf_percentile.q
ql/src/test/results/clientpositive/udf_percentile.q.out
Author: namit
Date: Wed Mar 24 01:00:53 2010
New Revision: 926889
URL: http://svn.apache.org/viewvc?rev=926889&view=rev
Log:
HIVE-1273 UDF_Percentile NullPointerException
(Zheng Shao via namit)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java
hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_percentile.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_percentile.q.out
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=926889&r1=926888&r2=926889&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Mar 24 01:00:53 2010
@@ -297,6 +297,9 @@ Trunk - Unreleased
HIVE-1257. join between hbase and other tables does not work
(John Sichi via namit)
+ HIVE-1273 UDF_Percentile NullPointerException
+ (Zheng Shao via namit)
+
Release 0.5.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java?rev=926889&r1=926888&r2=926889&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java Wed Mar 24 01:00:53 2010
@@ -124,7 +124,7 @@ public class UDAFPercentile extends UDAF
*/
public static class PercentileLongEvaluator implements UDAFEvaluator {
- private State state;
+ private final State state;
public PercentileLongEvaluator() {
state = new State();
@@ -155,8 +155,10 @@ public class UDAFPercentile extends UDAF
if (state.percentiles == null) {
state.percentiles = new ArrayList<DoubleWritable>(other.percentiles);
}
- for (Map.Entry<LongWritable, LongWritable> e: other.counts.entrySet()) {
- increment(state, e.getKey(), e.getValue().get());
+ if (other.counts != null) {
+ for (Map.Entry<LongWritable, LongWritable> e: other.counts.entrySet()) {
+ increment(state, e.getKey(), e.getValue().get());
+ }
}
return true;
}
@@ -201,7 +203,7 @@ public class UDAFPercentile extends UDAF
*/
public static class PercentileLongArrayEvaluator implements UDAFEvaluator {
- private State state;
+ private final State state;
public PercentileLongArrayEvaluator() {
state = new State();
@@ -231,8 +233,10 @@ public class UDAFPercentile extends UDAF
if (state.percentiles == null) {
state.percentiles = new ArrayList<DoubleWritable>(other.percentiles);
}
- for (Map.Entry<LongWritable, LongWritable> e: other.counts.entrySet()) {
- increment(state, e.getKey(), e.getValue().get());
+ if (other.counts != null) {
+ for (Map.Entry<LongWritable, LongWritable> e: other.counts.entrySet()) {
+ increment(state, e.getKey(), e.getValue().get());
+ }
}
return true;
}
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_percentile.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_percentile.q?rev=926889&r1=926888&r2=926889&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_percentile.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_percentile.q Wed Mar 24 01:00:53 2010
@@ -49,3 +49,14 @@ SELECT CAST(key AS INT) DIV 10,
percentile(CAST(substr(value, 5) AS INT), array(0.0, 0.5, 0.99, 1.0))
FROM src
GROUP BY CAST(key AS INT) DIV 10;
+
+
+set hive.map.aggr = true;
+set hive.groupby.skewindata = false;
+
+-- test null handling
+SELECT CAST(key AS INT) DIV 10,
+ percentile(NULL, 0.0),
+ percentile(NULL, array(0.0, 0.5, 0.99, 1.0))
+FROM src
+GROUP BY CAST(key AS INT) DIV 10;
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_percentile.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_percentile.q.out?rev=926889&r1=926888&r2=926889&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_percentile.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_percentile.q.out Wed Mar 24 01:00:53 2010
@@ -17,7 +17,7 @@ FROM src
GROUP BY CAST(key AS INT) DIV 10
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-28_01-30-57_926_4752964414489468009/10000
+PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-29_731_4236783179618139554/10000
POSTHOOK: query: SELECT CAST(key AS INT) DIV 10,
percentile(CAST(substr(value, 5) AS INT), 0.0),
percentile(CAST(substr(value, 5) AS INT), 0.5),
@@ -27,7 +27,7 @@ FROM src
GROUP BY CAST(key AS INT) DIV 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-28_01-30-57_926_4752964414489468009/10000
+POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-29_731_4236783179618139554/10000
0 0.0 4.5 9.0 [0.0,4.5,8.91,9.0]
1 10.0 15.0 19.0 [10.0,15.0,18.91,19.0]
2 20.0 26.0 28.0 [20.0,26.0,27.939999999999998,28.0]
@@ -87,7 +87,7 @@ FROM src
GROUP BY CAST(key AS INT) DIV 10
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-28_01-31-02_222_1225909126411093990/10000
+PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-34_361_7854657244563137261/10000
POSTHOOK: query: SELECT CAST(key AS INT) DIV 10,
percentile(CAST(substr(value, 5) AS INT), 0.0),
percentile(CAST(substr(value, 5) AS INT), 0.5),
@@ -97,7 +97,7 @@ FROM src
GROUP BY CAST(key AS INT) DIV 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-28_01-31-02_222_1225909126411093990/10000
+POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-34_361_7854657244563137261/10000
0 0.0 4.5 9.0 [0.0,4.5,8.91,9.0]
1 10.0 15.0 19.0 [10.0,15.0,18.91,19.0]
2 20.0 26.0 28.0 [20.0,26.0,27.939999999999998,28.0]
@@ -157,7 +157,7 @@ FROM src
GROUP BY CAST(key AS INT) DIV 10
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-28_01-31-06_237_1106980062235474069/10000
+PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-38_451_6456445188085972700/10000
POSTHOOK: query: SELECT CAST(key AS INT) DIV 10,
percentile(CAST(substr(value, 5) AS INT), 0.0),
percentile(CAST(substr(value, 5) AS INT), 0.5),
@@ -167,7 +167,7 @@ FROM src
GROUP BY CAST(key AS INT) DIV 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-28_01-31-06_237_1106980062235474069/10000
+POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-38_451_6456445188085972700/10000
0 0.0 4.5 9.0 [0.0,4.5,8.91,9.0]
1 10.0 15.0 19.0 [10.0,15.0,18.91,19.0]
2 20.0 26.0 28.0 [20.0,26.0,27.939999999999998,28.0]
@@ -227,7 +227,7 @@ FROM src
GROUP BY CAST(key AS INT) DIV 10
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-28_01-31-13_357_1069357782802139112/10000
+PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-45_662_669810203047990628/10000
POSTHOOK: query: SELECT CAST(key AS INT) DIV 10,
percentile(CAST(substr(value, 5) AS INT), 0.0),
percentile(CAST(substr(value, 5) AS INT), 0.5),
@@ -237,7 +237,7 @@ FROM src
GROUP BY CAST(key AS INT) DIV 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-28_01-31-13_357_1069357782802139112/10000
+POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-45_662_669810203047990628/10000
0 0.0 4.5 9.0 [0.0,4.5,8.91,9.0]
1 10.0 15.0 19.0 [10.0,15.0,18.91,19.0]
2 20.0 26.0 28.0 [20.0,26.0,27.939999999999998,28.0]
@@ -288,3 +288,71 @@ POSTHOOK: Output: file:/data/users/zshao
47 470.0 477.0 479.0 [470.0,477.0,478.94,479.0]
48 480.0 484.0 489.0 [480.0,484.0,489.0,489.0]
49 490.0 494.5 498.0 [490.0,494.5,498.0,498.0]
+PREHOOK: query: -- test null handling
+SELECT CAST(key AS INT) DIV 10,
+ percentile(NULL, 0.0),
+ percentile(NULL, array(0.0, 0.5, 0.99, 1.0))
+FROM src
+GROUP BY CAST(key AS INT) DIV 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-53_228_1913274328173734747/10000
+POSTHOOK: query: -- test null handling
+SELECT CAST(key AS INT) DIV 10,
+ percentile(NULL, 0.0),
+ percentile(NULL, array(0.0, 0.5, 0.99, 1.0))
+FROM src
+GROUP BY CAST(key AS INT) DIV 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-23_16-16-53_228_1913274328173734747/10000
+0 NULL null
+1 NULL null
+2 NULL null
+3 NULL null
+4 NULL null
+5 NULL null
+6 NULL null
+7 NULL null
+8 NULL null
+9 NULL null
+10 NULL null
+11 NULL null
+12 NULL null
+13 NULL null
+14 NULL null
+15 NULL null
+16 NULL null
+17 NULL null
+18 NULL null
+19 NULL null
+20 NULL null
+21 NULL null
+22 NULL null
+23 NULL null
+24 NULL null
+25 NULL null
+26 NULL null
+27 NULL null
+28 NULL null
+29 NULL null
+30 NULL null
+31 NULL null
+32 NULL null
+33 NULL null
+34 NULL null
+35 NULL null
+36 NULL null
+37 NULL null
+38 NULL null
+39 NULL null
+40 NULL null
+41 NULL null
+42 NULL null
+43 NULL null
+44 NULL null
+45 NULL null
+46 NULL null
+47 NULL null
+48 NULL null
+49 NULL null