You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2019/01/21 13:54:57 UTC
[1/2] hive git commit: HIVE-20733: GenericUDFOPEqualNS may not use =
in plan descriptions (David Lavati via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master ccba23f1a -> 34c8ca432
HIVE-20733: GenericUDFOPEqualNS may not use = in plan descriptions (David Lavati via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/750b7c1f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/750b7c1f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/750b7c1f
Branch: refs/heads/master
Commit: 750b7c1f40201cc0463b95941b822e64c8aaed44
Parents: ccba23f
Author: David Lavati <da...@gmail.com>
Authored: Mon Jan 21 14:45:08 2019 +0100
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Mon Jan 21 14:45:08 2019 +0100
----------------------------------------------------------------------
.../hive/ql/udf/generic/GenericUDFOPEqualNS.java | 14 +++++++++++---
.../hive/ql/udf/generic/GenericUDFOPNotEqualNS.java | 10 +++++++++-
.../filter_cond_pushdown_HIVE_15647.q.out | 8 ++++----
.../clientpositive/llap/is_distinct_from.q.out | 16 ++++++++--------
.../llap/orc_predicate_pushdown.q.out | 6 +++---
.../llap/parquet_predicate_pushdown.q.out | 6 +++---
ql/src/test/results/clientpositive/udf_equal.q.out | 4 ++--
7 files changed, 40 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/750b7c1f/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualNS.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualNS.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualNS.java
index cd339c0..8321cd3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualNS.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualNS.java
@@ -21,11 +21,19 @@ package org.apache.hadoop.hive.ql.udf.generic;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-@Description(name = "<=>", value = "a _FUNC_ b - Returns same result with EQUAL(=) operator " +
- "for non-null operands, but returns TRUE if both are NULL, FALSE if one of the them is NULL")
+/**
+ * GenericUDF Class for operation EQUALNS.
+ */
+@Description(name = "IS NOT DISTINCT FROM", value = "a _FUNC_ b - Returns same result with EQUALNS (IS NOT DISTINCT " +
+ "FROM) operator for non-null operands, but returns TRUE if both are NULL, FALSE if one of the them is NULL")
@NDV(maxNdv = 2)
public class GenericUDFOPEqualNS extends GenericUDFOPEqual {
+ public GenericUDFOPEqualNS(){
+ this.opName = "EQUALNS";
+ this.opDisplayName = "IS NOT DISTINCT FROM";
+ }
+
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
Object o0 = arguments[0].get();
@@ -43,6 +51,6 @@ public class GenericUDFOPEqualNS extends GenericUDFOPEqual {
@Override
public GenericUDF negative() {
- return new GenericUDFOPNotEqualNS();
+ return new GenericUDFOPNotEqualNS();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/750b7c1f/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqualNS.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqualNS.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqualNS.java
index d48bbbb..f4b5e8f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqualNS.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqualNS.java
@@ -18,11 +18,19 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.metadata.HiveException;
// this function is for internal use only
public class GenericUDFOPNotEqualNS extends GenericUDFOPNotEqual {
+ @Description(name = "IS DISTINCT FROM", value = "a _FUNC_ b - Returns same result with NOTEQUALNS (IS DISTINCT " +
+ "FROM) operator for non-null operands, but returns FALSE if both are NULL, TRUE if one of the them is NULL")
+ public GenericUDFOPNotEqualNS(){
+ this.opName = "NOTEQUALNS";
+ this.opDisplayName = "IS DISTINCT FROM";
+ }
+
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
Object o0 = arguments[0].get();
@@ -40,6 +48,6 @@ public class GenericUDFOPNotEqualNS extends GenericUDFOPNotEqual {
@Override
public GenericUDF negative() {
- return new GenericUDFOPEqualNS();
+ return new GenericUDFOPEqualNS();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/750b7c1f/ql/src/test/results/clientpositive/filter_cond_pushdown_HIVE_15647.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/filter_cond_pushdown_HIVE_15647.q.out b/ql/src/test/results/clientpositive/filter_cond_pushdown_HIVE_15647.q.out
index bc023a6..424fc71 100644
--- a/ql/src/test/results/clientpositive/filter_cond_pushdown_HIVE_15647.q.out
+++ b/ql/src/test/results/clientpositive/filter_cond_pushdown_HIVE_15647.q.out
@@ -41,10 +41,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: sales
- filterExpr: ((store_number = customer_id) and store_id is not null) (type: boolean)
+ filterExpr: ((store_number IS NOT DISTINCT FROM customer_id) and store_id is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((store_number = customer_id) and store_id is not null) (type: boolean)
+ predicate: ((store_number IS NOT DISTINCT FROM customer_id) and store_id is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
key expressions: store_id (type: int)
@@ -237,10 +237,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: sales
- filterExpr: ((store_number = customer_id) and store_id is not null) (type: boolean)
+ filterExpr: ((store_number IS NOT DISTINCT FROM customer_id) and store_id is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((store_number = customer_id) and store_id is not null) (type: boolean)
+ predicate: ((store_number IS NOT DISTINCT FROM customer_id) and store_id is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
key expressions: store_id (type: int)
http://git-wip-us.apache.org/repos/asf/hive/blob/750b7c1f/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out b/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out
index 010c4a1..fa581f7 100644
--- a/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out
+++ b/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out
@@ -206,9 +206,9 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: test_n5
- filterExpr: (y <> null) (type: boolean)
+ filterExpr: (y IS DISTINCT FROM null) (type: boolean)
Filter Operator
- predicate: (y <> null) (type: boolean)
+ predicate: (y IS DISTINCT FROM null) (type: boolean)
Select Operator
expressions: x (type: string), y (type: string)
outputColumnNames: _col0, _col1
@@ -243,9 +243,9 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: test_n5
- filterExpr: (y = null) (type: boolean)
+ filterExpr: (y IS NOT DISTINCT FROM null) (type: boolean)
Filter Operator
- predicate: (y = null) (type: boolean)
+ predicate: (y IS NOT DISTINCT FROM null) (type: boolean)
Select Operator
expressions: x (type: string), null (type: string)
outputColumnNames: _col0, _col1
@@ -287,9 +287,9 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: part
- filterExpr: (p_size <> 2) (type: boolean)
+ filterExpr: (p_size IS DISTINCT FROM 2) (type: boolean)
Filter Operator
- predicate: (p_size <> 2) (type: boolean)
+ predicate: (p_size IS DISTINCT FROM 2) (type: boolean)
Select Operator
expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
@@ -343,9 +343,9 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: part
- filterExpr: (p_size = 2) (type: boolean)
+ filterExpr: (p_size IS NOT DISTINCT FROM 2) (type: boolean)
Filter Operator
- predicate: (p_size = 2) (type: boolean)
+ predicate: (p_size IS NOT DISTINCT FROM 2) (type: boolean)
Select Operator
expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), 2 (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
http://git-wip-us.apache.org/repos/asf/hive/blob/750b7c1f/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
index 797eaa8..06b7703 100644
--- a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
@@ -503,7 +503,7 @@ STAGE PLANS:
TableScan
alias: orc_pred
Filter Operator
- predicate: ((s like 'bob%') and (t = -1) and s is not null) (type: boolean)
+ predicate: ((s like 'bob%') and (t IS NOT DISTINCT FROM -1) and s is not null) (type: boolean)
Select Operator
expressions: -1Y (type: tinyint), s (type: string)
outputColumnNames: _col0, _col1
@@ -533,9 +533,9 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: orc_pred
- filterExpr: ((t = -1) and s is not null and (s like 'bob%')) (type: boolean)
+ filterExpr: ((t IS NOT DISTINCT FROM -1) and s is not null and (s like 'bob%')) (type: boolean)
Filter Operator
- predicate: ((s like 'bob%') and (t = -1) and s is not null) (type: boolean)
+ predicate: ((s like 'bob%') and (t IS NOT DISTINCT FROM -1) and s is not null) (type: boolean)
Select Operator
expressions: -1Y (type: tinyint), s (type: string)
outputColumnNames: _col0, _col1
http://git-wip-us.apache.org/repos/asf/hive/blob/750b7c1f/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
index 81af3d4..2cb69dc 100644
--- a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
@@ -441,7 +441,7 @@ STAGE PLANS:
TableScan
alias: tbl_pred
Filter Operator
- predicate: ((s like 'bob%') and (t = -1) and s is not null) (type: boolean)
+ predicate: ((s like 'bob%') and (t IS NOT DISTINCT FROM -1) and s is not null) (type: boolean)
Select Operator
expressions: -1Y (type: tinyint), s (type: string)
outputColumnNames: _col0, _col1
@@ -471,9 +471,9 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: tbl_pred
- filterExpr: ((t = -1) and s is not null and (s like 'bob%')) (type: boolean)
+ filterExpr: ((t IS NOT DISTINCT FROM -1) and s is not null and (s like 'bob%')) (type: boolean)
Filter Operator
- predicate: ((s like 'bob%') and (t = -1) and s is not null) (type: boolean)
+ predicate: ((s like 'bob%') and (t IS NOT DISTINCT FROM -1) and s is not null) (type: boolean)
Select Operator
expressions: -1Y (type: tinyint), s (type: string)
outputColumnNames: _col0, _col1
http://git-wip-us.apache.org/repos/asf/hive/blob/750b7c1f/ql/src/test/results/clientpositive/udf_equal.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_equal.q.out b/ql/src/test/results/clientpositive/udf_equal.q.out
index 52bd843..5ccfac0 100644
--- a/ql/src/test/results/clientpositive/udf_equal.q.out
+++ b/ql/src/test/results/clientpositive/udf_equal.q.out
@@ -37,12 +37,12 @@ PREHOOK: query: DESCRIBE FUNCTION <=>
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION <=>
POSTHOOK: type: DESCFUNCTION
-a <=> b - Returns same result with EQUAL(=) operator for non-null operands, but returns TRUE if both are NULL, FALSE if one of the them is NULL
+a <=> b - Returns same result with EQUALNS (IS NOT DISTINCT FROM) operator for non-null operands, but returns TRUE if both are NULL, FALSE if one of the them is NULL
PREHOOK: query: DESCRIBE FUNCTION EXTENDED <=>
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION EXTENDED <=>
POSTHOOK: type: DESCFUNCTION
-a <=> b - Returns same result with EQUAL(=) operator for non-null operands, but returns TRUE if both are NULL, FALSE if one of the them is NULL
+a <=> b - Returns same result with EQUALNS (IS NOT DISTINCT FROM) operator for non-null operands, but returns TRUE if both are NULL, FALSE if one of the them is NULL
Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS
Function type:BUILTIN
PREHOOK: query: SELECT true<=>false, false<=>true, false<=>false, true<=>true, NULL<=>NULL, true<=>NULL, NULL<=>true, false<=>NULL, NULL<=>false FROM src tablesample (1 rows)
[2/2] hive git commit: HIVE-20879: Using null in a projection
expression leads to CastException (David Lavati via Zoltan Haindrich)
Posted by kg...@apache.org.
HIVE-20879: Using null in a projection expression leads to CastException (David Lavati via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/34c8ca43
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/34c8ca43
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/34c8ca43
Branch: refs/heads/master
Commit: 34c8ca432c5802192ef07d92ca81e5eea22d6c6b
Parents: 750b7c1
Author: David Lavati <da...@gmail.com>
Authored: Mon Jan 21 14:48:10 2019 +0100
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Mon Jan 21 14:48:10 2019 +0100
----------------------------------------------------------------------
.../hive/ql/parse/TypeCheckProcFactory.java | 6 +++
ql/src/test/queries/clientpositive/cast3.q | 9 +++++
ql/src/test/results/clientpositive/cast3.q.out | 40 ++++++++++++++++++++
3 files changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/34c8ca43/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
index fa4abf9..b49bb36 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
@@ -1229,6 +1229,9 @@ public class TypeCheckProcFactory {
List<ExprNodeDesc> childrenList = new ArrayList<ExprNodeDesc>(
children.size());
for (ExprNodeDesc child : children) {
+ if (TypeInfoFactory.getPrimitiveTypeInfo("void").equals(child.getTypeInfo())) {
+ child.setTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("boolean"));
+ }
if (FunctionRegistry.isOpOr(child)) {
childrenList.addAll(child.getChildren());
} else {
@@ -1242,6 +1245,9 @@ public class TypeCheckProcFactory {
List<ExprNodeDesc> childrenList = new ArrayList<ExprNodeDesc>(
children.size());
for (ExprNodeDesc child : children) {
+ if (TypeInfoFactory.getPrimitiveTypeInfo("void").equals(child.getTypeInfo())) {
+ child.setTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("boolean"));
+ }
if (FunctionRegistry.isOpAnd(child)) {
childrenList.addAll(child.getChildren());
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/34c8ca43/ql/src/test/queries/clientpositive/cast3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/cast3.q b/ql/src/test/queries/clientpositive/cast3.q
new file mode 100644
index 0000000..a01eb3a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cast3.q
@@ -0,0 +1,9 @@
+create table cx1(bool0 boolean);
+
+select cast(NULL as boolean) or bool0 from cx1;
+
+select NULL or bool0 from cx1;
+
+select cast(NULL as boolean) and bool0 from cx1;
+
+select NULL and bool0 from cx1;
http://git-wip-us.apache.org/repos/asf/hive/blob/34c8ca43/ql/src/test/results/clientpositive/cast3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cast3.q.out b/ql/src/test/results/clientpositive/cast3.q.out
new file mode 100644
index 0000000..d3234e9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/cast3.q.out
@@ -0,0 +1,40 @@
+PREHOOK: query: create table cx1(bool0 boolean)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cx1
+POSTHOOK: query: create table cx1(bool0 boolean)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cx1
+PREHOOK: query: select cast(NULL as boolean) or bool0 from cx1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cx1
+#### A masked pattern was here ####
+POSTHOOK: query: select cast(NULL as boolean) or bool0 from cx1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cx1
+#### A masked pattern was here ####
+PREHOOK: query: select NULL or bool0 from cx1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cx1
+#### A masked pattern was here ####
+POSTHOOK: query: select NULL or bool0 from cx1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cx1
+#### A masked pattern was here ####
+PREHOOK: query: select cast(NULL as boolean) and bool0 from cx1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cx1
+#### A masked pattern was here ####
+POSTHOOK: query: select cast(NULL as boolean) and bool0 from cx1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cx1
+#### A masked pattern was here ####
+PREHOOK: query: select NULL and bool0 from cx1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cx1
+#### A masked pattern was here ####
+POSTHOOK: query: select NULL and bool0 from cx1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cx1
+#### A masked pattern was here ####