You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/05/23 01:03:52 UTC
svn commit: r1596981 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
test/queries/clientpositive/windowing_decimal.q
test/results/clientpositive/windowing_decimal.q.out
Author: hashutosh
Date: Thu May 22 23:03:51 2014
New Revision: 1596981
URL: http://svn.apache.org/r1596981
Log:
HIVE-7099 : Add Decimal datatype support for Windowing (Harish Butani via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/queries/clientpositive/windowing_decimal.q
hive/trunk/ql/src/test/results/clientpositive/windowing_decimal.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java?rev=1596981&r1=1596980&r2=1596981&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java Thu May 22 23:03:51 2014
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.PTFOperator;
import org.apache.hadoop.hive.ql.exec.PTFPartition;
import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator;
@@ -730,6 +731,8 @@ public class WindowingTableFunction exte
case DOUBLE:
case FLOAT:
return new DoubleValueBoundaryScanner(vbDef, order, vbDef.getExpressionDef());
+ case DECIMAL:
+ return new HiveDecimalValueBoundaryScanner(vbDef, order, vbDef.getExpressionDef());
case STRING:
return new StringValueBoundaryScanner(vbDef, order, vbDef.getExpressionDef());
}
@@ -789,6 +792,37 @@ public class WindowingTableFunction exte
}
}
+ public static class HiveDecimalValueBoundaryScanner extends ValueBoundaryScanner {
+ public HiveDecimalValueBoundaryScanner(BoundaryDef bndDef, Order order,
+ PTFExpressionDef expressionDef) {
+ super(bndDef,order,expressionDef);
+ }
+
+ @Override
+ public boolean isGreater(Object v1, Object v2, int amt) {
+ HiveDecimal d1 = PrimitiveObjectInspectorUtils.getHiveDecimal(v1,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ HiveDecimal d2 = PrimitiveObjectInspectorUtils.getHiveDecimal(v2,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ if ( d1 == null || d2 == null ) {
+ return false;
+ }
+ return d1.subtract(d2).intValue() > amt;
+ }
+
+ @Override
+ public boolean isEqual(Object v1, Object v2) {
+ HiveDecimal d1 = PrimitiveObjectInspectorUtils.getHiveDecimal(v1,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ HiveDecimal d2 = PrimitiveObjectInspectorUtils.getHiveDecimal(v2,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ if ( d1 == null || d2 == null ) {
+ return false;
+ }
+ return d1.equals(d2);
+ }
+ }
+
public static class StringValueBoundaryScanner extends ValueBoundaryScanner {
public StringValueBoundaryScanner(BoundaryDef bndDef, Order order,
PTFExpressionDef expressionDef) {
Added: hive/trunk/ql/src/test/queries/clientpositive/windowing_decimal.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/windowing_decimal.q?rev=1596981&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/windowing_decimal.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/windowing_decimal.q Thu May 22 23:03:51 2014
@@ -0,0 +1,41 @@
+DROP TABLE part;
+
+-- data setup
+CREATE TABLE part(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice DOUBLE,
+ p_comment STRING
+);
+
+LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part;
+
+
+create table part_dec(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice decimal(10,3),
+ p_comment STRING
+);
+
+insert overwrite table part_dec select * from part;
+
+select p_mfgr, p_retailprice,
+first_value(p_retailprice) over(partition by p_mfgr order by p_retailprice) ,
+sum(p_retailprice) over(partition by p_mfgr order by p_retailprice)
+from part_dec;
+
+select p_mfgr, p_retailprice,
+first_value(p_retailprice) over(partition by p_mfgr order by p_retailprice range between 5 preceding and current row) ,
+sum(p_retailprice) over(partition by p_mfgr order by p_retailprice range between 5 preceding and current row)
+from part_dec;
\ No newline at end of file
Added: hive/trunk/ql/src/test/results/clientpositive/windowing_decimal.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/windowing_decimal.q.out?rev=1596981&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/windowing_decimal.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/windowing_decimal.q.out Thu May 22 23:03:51 2014
@@ -0,0 +1,183 @@
+PREHOOK: query: DROP TABLE part
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE part
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: -- data setup
+CREATE TABLE part(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice DOUBLE,
+ p_comment STRING
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: -- data setup
+CREATE TABLE part(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice DOUBLE,
+ p_comment STRING
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@part
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@part
+PREHOOK: query: create table part_dec(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice decimal(10,3),
+ p_comment STRING
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table part_dec(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice decimal(10,3),
+ p_comment STRING
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_dec
+PREHOOK: query: insert overwrite table part_dec select * from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+PREHOOK: Output: default@part_dec
+POSTHOOK: query: insert overwrite table part_dec select * from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+POSTHOOK: Output: default@part_dec
+POSTHOOK: Lineage: part_dec.p_brand SIMPLE [(part)part.FieldSchema(name:p_brand, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_comment SIMPLE [(part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_container SIMPLE [(part)part.FieldSchema(name:p_container, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_name SIMPLE [(part)part.FieldSchema(name:p_name, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_partkey SIMPLE [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_retailprice EXPRESSION [(part)part.FieldSchema(name:p_retailprice, type:double, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_size SIMPLE [(part)part.FieldSchema(name:p_size, type:int, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_type SIMPLE [(part)part.FieldSchema(name:p_type, type:string, comment:null), ]
+PREHOOK: query: select p_mfgr, p_retailprice,
+first_value(p_retailprice) over(partition by p_mfgr order by p_retailprice) ,
+sum(p_retailprice) over(partition by p_mfgr order by p_retailprice)
+from part_dec
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_dec
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_retailprice,
+first_value(p_retailprice) over(partition by p_mfgr order by p_retailprice) ,
+sum(p_retailprice) over(partition by p_mfgr order by p_retailprice)
+from part_dec
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_dec
+#### A masked pattern was here ####
+POSTHOOK: Lineage: part_dec.p_brand SIMPLE [(part)part.FieldSchema(name:p_brand, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_comment SIMPLE [(part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_container SIMPLE [(part)part.FieldSchema(name:p_container, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_name SIMPLE [(part)part.FieldSchema(name:p_name, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_partkey SIMPLE [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_retailprice EXPRESSION [(part)part.FieldSchema(name:p_retailprice, type:double, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_size SIMPLE [(part)part.FieldSchema(name:p_size, type:int, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_type SIMPLE [(part)part.FieldSchema(name:p_type, type:string, comment:null), ]
+Manufacturer#1 1173.15 1173.15 2346.3
+Manufacturer#1 1173.15 1173.15 2346.3
+Manufacturer#1 1414.42 1173.15 3760.72
+Manufacturer#1 1602.59 1173.15 5363.31
+Manufacturer#1 1632.66 1173.15 6995.97
+Manufacturer#1 1753.76 1173.15 8749.73
+Manufacturer#2 1690.68 1690.68 1690.68
+Manufacturer#2 1698.66 1690.68 3389.34
+Manufacturer#2 1701.6 1690.68 5090.94
+Manufacturer#2 1800.7 1690.68 6891.64
+Manufacturer#2 2031.98 1690.68 8923.62
+Manufacturer#3 1190.27 1190.27 1190.27
+Manufacturer#3 1337.29 1190.27 2527.56
+Manufacturer#3 1410.39 1190.27 3937.95
+Manufacturer#3 1671.68 1190.27 5609.63
+Manufacturer#3 1922.98 1190.27 7532.61
+Manufacturer#4 1206.26 1206.26 1206.26
+Manufacturer#4 1290.35 1206.26 2496.61
+Manufacturer#4 1375.42 1206.26 3872.03
+Manufacturer#4 1620.67 1206.26 5492.7
+Manufacturer#4 1844.92 1206.26 7337.62
+Manufacturer#5 1018.1 1018.1 1018.1
+Manufacturer#5 1464.48 1018.1 2482.58
+Manufacturer#5 1611.66 1018.1 4094.24
+Manufacturer#5 1788.73 1018.1 5882.97
+Manufacturer#5 1789.69 1018.1 7672.66
+PREHOOK: query: select p_mfgr, p_retailprice,
+first_value(p_retailprice) over(partition by p_mfgr order by p_retailprice range between 5 preceding and current row) ,
+sum(p_retailprice) over(partition by p_mfgr order by p_retailprice range between 5 preceding and current row)
+from part_dec
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_dec
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_retailprice,
+first_value(p_retailprice) over(partition by p_mfgr order by p_retailprice range between 5 preceding and current row) ,
+sum(p_retailprice) over(partition by p_mfgr order by p_retailprice range between 5 preceding and current row)
+from part_dec
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_dec
+#### A masked pattern was here ####
+POSTHOOK: Lineage: part_dec.p_brand SIMPLE [(part)part.FieldSchema(name:p_brand, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_comment SIMPLE [(part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_container SIMPLE [(part)part.FieldSchema(name:p_container, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_name SIMPLE [(part)part.FieldSchema(name:p_name, type:string, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_partkey SIMPLE [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_retailprice EXPRESSION [(part)part.FieldSchema(name:p_retailprice, type:double, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_size SIMPLE [(part)part.FieldSchema(name:p_size, type:int, comment:null), ]
+POSTHOOK: Lineage: part_dec.p_type SIMPLE [(part)part.FieldSchema(name:p_type, type:string, comment:null), ]
+Manufacturer#1 1173.15 1173.15 2346.3
+Manufacturer#1 1173.15 1173.15 2346.3
+Manufacturer#1 1414.42 1414.42 1414.42
+Manufacturer#1 1602.59 1602.59 1602.59
+Manufacturer#1 1632.66 1632.66 1632.66
+Manufacturer#1 1753.76 1753.76 1753.76
+Manufacturer#2 1690.68 1690.68 1690.68
+Manufacturer#2 1698.66 1698.66 1698.66
+Manufacturer#2 1701.6 1698.66 3400.26
+Manufacturer#2 1800.7 1800.7 1800.7
+Manufacturer#2 2031.98 2031.98 2031.98
+Manufacturer#3 1190.27 1190.27 1190.27
+Manufacturer#3 1337.29 1337.29 1337.29
+Manufacturer#3 1410.39 1410.39 1410.39
+Manufacturer#3 1671.68 1671.68 1671.68
+Manufacturer#3 1922.98 1922.98 1922.98
+Manufacturer#4 1206.26 1206.26 1206.26
+Manufacturer#4 1290.35 1290.35 1290.35
+Manufacturer#4 1375.42 1375.42 1375.42
+Manufacturer#4 1620.67 1620.67 1620.67
+Manufacturer#4 1844.92 1844.92 1844.92
+Manufacturer#5 1018.1 1018.1 1018.1
+Manufacturer#5 1464.48 1464.48 1464.48
+Manufacturer#5 1611.66 1611.66 1611.66
+Manufacturer#5 1788.73 1788.73 1788.73
+Manufacturer#5 1789.69 1788.73 3578.42