You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2019/03/30 01:26:05 UTC
[hive] branch master updated: HIVE-21544: Constant propagation
corrupts coalesce/case/when expressions during folding (Zoltan Haindrich,
reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 146611b HIVE-21544: Constant propagation corrupts coalesce/case/when expressions during folding (Zoltan Haindrich, reviewed by Jesus Camacho Rodriguez)
146611b is described below
commit 146611b9c120e5a392c951df41bb74dd8ed11ec9
Author: Zoltan Haindrich <ki...@rxd.hu>
AuthorDate: Fri Mar 29 18:25:01 2019 -0700
HIVE-21544: Constant propagation corrupts coalesce/case/when expressions during folding (Zoltan Haindrich, reviewed by Jesus Camacho Rodriguez)
---
.../test/resources/testconfiguration.properties | 1 +
.../ql/optimizer/ConstantPropagateProcFactory.java | 2 +-
.../queries/clientpositive/constant_prop_when.q | 20 +++++
.../clientpositive/llap/constant_prop_when.q.out | 87 ++++++++++++++++++++++
4 files changed, 109 insertions(+), 1 deletion(-)
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 3a2807f..7f79995 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -494,6 +494,7 @@ minillaplocal.query.files=\
colstats_date_min_max.q,\
compare_double_bigint_2.q,\
constprog_dpp.q,\
+ constant_prop_when.q,\
constraints_optimization.q,\
current_date_timestamp.q,\
correlationoptimizer1.q,\
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
index 736185f..c965742 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
@@ -588,7 +588,7 @@ public final class ConstantPropagateProcFactory {
return null;
}
GenericUDF childUDF = caseOrWhenexpr.getGenericUDF();
- List<ExprNodeDesc> children = caseOrWhenexpr.getChildren();
+ List<ExprNodeDesc> children = new ArrayList(caseOrWhenexpr.getChildren());
int i;
if (childUDF instanceof GenericUDFWhen) {
for (i = 1; i < children.size(); i+=2) {
diff --git a/ql/src/test/queries/clientpositive/constant_prop_when.q b/ql/src/test/queries/clientpositive/constant_prop_when.q
new file mode 100644
index 0000000..fc1ab2b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/constant_prop_when.q
@@ -0,0 +1,20 @@
+set hive.fetch.task.conversion=none;
+set hive.optimize.ppd=false;
+
+create table t (s1 string,s2 string);
+
+insert into t values (null,null);
+
+explain
+select
+ coalesce(s1, 'null_value' ), coalesce(s2, 'null_value' ),
+ coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ),
+ case when coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ) then 'eq' else 'noteq' end
+from t;
+
+select
+ coalesce(s1, 'null_value' ), coalesce(s2, 'null_value' ),
+ coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ),
+ case when coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ) then 'eq' else 'noteq' end
+from t;
+
diff --git a/ql/src/test/results/clientpositive/llap/constant_prop_when.q.out b/ql/src/test/results/clientpositive/llap/constant_prop_when.q.out
new file mode 100644
index 0000000..c1fe2f6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/constant_prop_when.q.out
@@ -0,0 +1,87 @@
+PREHOOK: query: create table t (s1 string,s2 string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t (s1 string,s2 string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: insert into t values (null,null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t
+POSTHOOK: query: insert into t values (null,null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.s1 EXPRESSION []
+POSTHOOK: Lineage: t.s2 EXPRESSION []
+PREHOOK: query: explain
+select
+ coalesce(s1, 'null_value' ), coalesce(s2, 'null_value' ),
+ coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ),
+ case when coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ) then 'eq' else 'noteq' end
+from t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select
+ coalesce(s1, 'null_value' ), coalesce(s2, 'null_value' ),
+ coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ),
+ case when coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ) then 'eq' else 'noteq' end
+from t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: CASE WHEN (s1 is not null) THEN (s1) ELSE ('null_value') END (type: string), CASE WHEN (s2 is not null) THEN (s2) ELSE ('null_value') END (type: string), CASE WHEN (s1 is not null) THEN (CASE WHEN (s2 is not null) THEN ((s2 = s1)) ELSE (('null_value' = s1)) END) ELSE (CASE WHEN (s2 is not null) THEN ((s2 = 'null_value')) ELSE (('null_value' = 'null_value')) END) END (type: boolean), CASE WHEN (CASE WHEN (s1 is not null) THEN (CASE WHEN (s2 is not null) TH [...]
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 556 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 556 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ coalesce(s1, 'null_value' ), coalesce(s2, 'null_value' ),
+ coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ),
+ case when coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ) then 'eq' else 'noteq' end
+from t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ coalesce(s1, 'null_value' ), coalesce(s2, 'null_value' ),
+ coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ),
+ case when coalesce(s1, 'null_value' )=coalesce(s2, 'null_value' ) then 'eq' else 'noteq' end
+from t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+null_value null_value true eq