You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/11/20 20:58:49 UTC
svn commit: r1640798 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/
java/org/apache/hadoop/hive/ql/optimizer/
java/org/apache/hadoop/hive/ql/parse/
java/org/apache/hadoop/hive/ql/udf/generic/ test/queries/clientpositive/
test/results/cli...
Author: hashutosh
Date: Thu Nov 20 19:58:48 2014
New Revision: 1640798
URL: http://svn.apache.org/r1640798
Log:
HIVE-8642 : Hive stack() UDTF Doesn't Support NULL Insert Values (Navis via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
hive/trunk/ql/src/test/queries/clientpositive/udtf_stack.q
hive/trunk/ql/src/test/results/clientpositive/annotate_stats_select.q.out
hive/trunk/ql/src/test/results/clientpositive/input9.q.out
hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out
hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java Thu Nov 20 19:58:48 2014
@@ -92,8 +92,8 @@ public class UnionOperator extends Opera
columns);
for (int c = 0; c < columns; c++) {
// can be null for void type
- ObjectInspector oi = columnTypeResolvers[c].get();
- outputFieldOIs.add(oi == null ? parentFields[0].get(c).getFieldObjectInspector() : oi);
+ ObjectInspector fieldOI = parentFields[0].get(c).getFieldObjectInspector();
+ outputFieldOIs.add(columnTypeResolvers[c].get(fieldOI));
}
// create output row ObjectInspector
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java Thu Nov 20 19:58:48 2014
@@ -59,6 +59,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.ListSinkDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -244,7 +245,9 @@ public class SimpleFetchOptimizer implem
}
private boolean checkExpression(ExprNodeDesc expr) {
- if (expr instanceof ExprNodeConstantDesc || expr instanceof ExprNodeColumnDesc) {
+ if (expr instanceof ExprNodeConstantDesc ||
+ expr instanceof ExprNodeNullDesc||
+ expr instanceof ExprNodeColumnDesc) {
return true;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Nov 20 19:58:48 2014
@@ -158,7 +158,6 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.ExtractDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
@@ -3702,11 +3701,6 @@ public class SemanticAnalyzer extends Ba
ArrayList<String> columnNames = new ArrayList<String>();
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
for (int i = 0; i < col_list.size(); i++) {
- // Replace NULL with CAST(NULL AS STRING)
- if (col_list.get(i) instanceof ExprNodeNullDesc) {
- col_list.set(i, new ExprNodeConstantDesc(
- TypeInfoFactory.stringTypeInfo, null));
- }
String outputCol = getColumnInternalName(i);
colExprMap.put(outputCol, col_list.get(i));
columnNames.add(outputCol);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java Thu Nov 20 19:58:48 2014
@@ -28,7 +28,6 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
/**
@@ -59,11 +58,8 @@ public class GenericUDFArray extends Gen
converters = new Converter[arguments.length];
- ObjectInspector returnOI = returnOIResolver.get();
- if (returnOI == null) {
- returnOI = PrimitiveObjectInspectorFactory
- .getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
- }
+ ObjectInspector returnOI =
+ returnOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
for (int i = 0; i < arguments.length; i++) {
converters[i] = ObjectInspectorConverters.getConverter(arguments[i],
returnOI);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java Thu Nov 20 19:58:48 2014
@@ -82,17 +82,10 @@ public class GenericUDFMap extends Gener
}
}
- ObjectInspector keyOI = keyOIResolver.get();
- ObjectInspector valueOI = valueOIResolver.get();
-
- if (keyOI == null) {
- keyOI = PrimitiveObjectInspectorFactory
- .getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
- }
- if (valueOI == null) {
- valueOI = PrimitiveObjectInspectorFactory
- .getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
- }
+ ObjectInspector keyOI =
+ keyOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ ObjectInspector valueOI =
+ valueOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
converters = new Converter[arguments.length];
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java Thu Nov 20 19:58:48 2014
@@ -82,10 +82,7 @@ public class GenericUDFSortArray extends
((ListObjectInspector)(arguments[0])).getListElementObjectInspector();
argumentOIs = arguments;
converters = new Converter[arguments.length];
- ObjectInspector returnOI = returnOIResolver.get();
- if (returnOI == null) {
- returnOI = elementObjectInspector;
- }
+ ObjectInspector returnOI = returnOIResolver.get(elementObjectInspector);
converters[0] = ObjectInspectorConverters.getConverter(elementObjectInspector, returnOI);
return ObjectInspectorFactory.getStandardListObjectInspector(returnOI);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java Thu Nov 20 19:58:48 2014
@@ -22,7 +22,6 @@ import java.lang.reflect.Array;
import java.lang.reflect.Method;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
-import java.nio.ByteBuffer;
import java.util.HashMap;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -41,6 +40,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
@@ -191,7 +191,11 @@ public final class GenericUDFUtils {
* Returns the ObjectInspector of the return value.
*/
public ObjectInspector get() {
- return returnObjectInspector;
+ return get(PrimitiveObjectInspectorFactory.javaVoidObjectInspector);
+ }
+
+ public ObjectInspector get(ObjectInspector defaultOI) {
+ return returnObjectInspector != null ? returnObjectInspector : defaultOI;
}
/**
Modified: hive/trunk/ql/src/test/queries/clientpositive/udtf_stack.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/udtf_stack.q?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/udtf_stack.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/udtf_stack.q Thu Nov 20 19:58:48 2014
@@ -5,3 +5,7 @@ EXPLAIN SELECT x, y FROM src LATERAL VIE
SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2;
SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2;
+
+EXPLAIN
+SELECT stack(1, "en", "dbpedia", NULL );
+SELECT stack(1, "en", "dbpedia", NULL );
\ No newline at end of file
Modified: hive/trunk/ql/src/test/results/clientpositive/annotate_stats_select.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/annotate_stats_select.q.out?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/annotate_stats_select.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/annotate_stats_select.q.out Thu Nov 20 19:58:48 2014
@@ -282,7 +282,7 @@ STAGE PLANS:
alias: alltypes_orc
Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: null (type: string)
+ expressions: null (type: void)
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
ListSink
@@ -497,33 +497,21 @@ POSTHOOK: query: -- numRows: 2 rawDataSi
explain select cast("1970-12-31 15:59:58.174" as DATE) from alltypes_orc
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: alltypes_orc
- Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: null (type: void)
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
- ListSink
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: null (type: void)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ ListSink
PREHOOK: query: -- numRows: 2 rawDataSize: 224
explain select cast("58.174" as DECIMAL) from alltypes_orc
Modified: hive/trunk/ql/src/test/results/clientpositive/input9.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/input9.q.out?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/input9.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/input9.q.out Thu Nov 20 19:58:48 2014
@@ -35,7 +35,7 @@ STAGE PLANS:
predicate: (null = null) (type: boolean)
Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: null (type: string), UDFToInteger(key) (type: int)
+ expressions: null (type: void), UDFToInteger(key) (type: int)
outputColumnNames: _col0, _col1
Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE
File Output Operator
Modified: hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out Thu Nov 20 19:58:48 2014
@@ -157,3 +157,51 @@ POSTHOOK: Input: default@src
#### A masked pattern was here ####
x [1]
z [4]
+PREHOOK: query: EXPLAIN
+SELECT stack(1, "en", "dbpedia", NULL )
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT stack(1, "en", "dbpedia", NULL )
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator
+ expressions: 1 (type: int), 'en' (type: string), 'dbpedia' (type: string), null (type: void)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ UDTF Operator
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ function name: stack
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT stack(1, "en", "dbpedia", NULL )
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT stack(1, "en", "dbpedia", NULL )
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+en dbpedia NULL
Modified: hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out?rev=1640798&r1=1640797&r2=1640798&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out Thu Nov 20 19:58:48 2014
@@ -91,37 +91,24 @@ SELECT elt(2, 'abc', 'defg'),
FROM alltypesorc LIMIT 1
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: void), null (type: void)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
- Statistics: Num rows: 12288 Data size: 8687616 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 707 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 707 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized
-
Stage: Stage-0
Fetch Operator
limit: 1
Processor Tree:
- ListSink
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: void), null (type: void)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
+ Statistics: Num rows: 12288 Data size: 8687616 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 707 Basic stats: COMPLETE Column stats: COMPLETE
+ ListSink
PREHOOK: query: SELECT elt(2, 'abc', 'defg'),
elt(3, 'aa', 'bb', 'cc', 'dd', 'ee', 'ff', 'gg'),