You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by px...@apache.org on 2015/10/29 18:48:25 UTC
hive git commit: CBO: Calcite Operator To Hive Operator (Calcite
Return Path) : columnPruner prunes everything when union is the last operator
before FS (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 63dc1fa61 -> 034280ce0
CBO: Calcite Operator To Hive Operator (Calcite Return Path) : columnPruner prunes everything when union is the last operator before FS (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/034280ce
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/034280ce
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/034280ce
Branch: refs/heads/master
Commit: 034280ce070d812f1eb312567a974a8720943647
Parents: 63dc1fa
Author: pengchengxiong <px...@hortonworks.com>
Authored: Thu Oct 29 10:27:28 2015 -0700
Committer: pengchengxiong <px...@hortonworks.com>
Committed: Thu Oct 29 10:27:28 2015 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/ColumnPrunerProcCtx.java | 7 +
.../clientpositive/cbo_rp_unionDistinct_2.q | 128 +++++
.../clientpositive/cbo_rp_unionDistinct_2.q.out | 545 +++++++++++++++++++
3 files changed, 680 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/034280ce/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
index 2207cfb..b18a034 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
@@ -115,6 +116,12 @@ public class ColumnPrunerProcCtx implements NodeProcessorCtx {
prunList.add(colInfo.getInternalName());
}
}
+ } else if (child instanceof FileSinkOperator) {
+ prunList = new ArrayList<>();
+ RowSchema oldRS = curOp.getSchema();
+ for (ColumnInfo colInfo : oldRS.getSignature()) {
+ prunList.add(colInfo.getInternalName());
+ }
} else {
prunList = prunedColLists.get(child);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/034280ce/ql/src/test/queries/clientpositive/cbo_rp_unionDistinct_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/cbo_rp_unionDistinct_2.q b/ql/src/test/queries/clientpositive/cbo_rp_unionDistinct_2.q
new file mode 100644
index 0000000..ea98bd2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cbo_rp_unionDistinct_2.q
@@ -0,0 +1,128 @@
+set hive.cbo.returnpath.hiveop=true;
+-- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5;
+
+CREATE TABLE u2 as select key, value from src order by key limit 3;
+
+CREATE TABLE u3 as select key, value from src order by key desc limit 5;
+
+select * from u1;
+
+select * from u2;
+
+select * from u3;
+
+select key, value from
+(
+select key, value from u1
+union all
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+select key, value from
+(
+select key, value from u1
+union
+select key, value from u2
+union all
+select key, value from u3
+) tab;
+
+select key, value from
+(
+select key, value from u1
+union distinct
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+select key, value from
+(
+select key, value from u1
+union all
+select key, value from u2
+union
+select key, value from u3
+) tab;
+
+select key, value from
+(
+select key, value from u1
+union
+select key, value from u2
+union
+select key as key, value from u3
+) tab;
+
+select distinct * from
+(
+select key, value from u1
+union all
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+select distinct * from
+(
+select distinct * from u1
+union
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+drop view if exists v;
+
+create view v as select distinct * from
+(
+select distinct * from u1
+union
+select key, value from u2
+union all
+select key as key, value from u3
+) tab;
+
+describe extended v;
+
+select * from v;
+
+drop view if exists v;
+
+create view v as select tab.* from
+(
+select distinct * from u1
+union
+select distinct * from u2
+) tab;
+
+describe extended v;
+
+select * from v;
+
+drop view if exists v;
+
+create view v as select * from
+(
+select distinct u1.* from u1
+union all
+select distinct * from u2
+) tab;
+
+describe extended v;
+
+select * from v;
+
+select distinct * from
+(
+select key, value from u1
+union all
+select key, value from u2
+union
+select key as key, value from u3
+) tab;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/034280ce/ql/src/test/results/clientpositive/cbo_rp_unionDistinct_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_unionDistinct_2.q.out b/ql/src/test/results/clientpositive/cbo_rp_unionDistinct_2.q.out
new file mode 100644
index 0000000..6d59369
--- /dev/null
+++ b/ql/src/test/results/clientpositive/cbo_rp_unionDistinct_2.q.out
@@ -0,0 +1,545 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@u1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@u1
+PREHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@u2
+POSTHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@u2
+PREHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@u3
+POSTHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@u3
+PREHOOK: query: select * from u1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from u1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
+PREHOOK: query: select * from u2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from u2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u2
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+PREHOOK: query: select * from u3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from u3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+96 val_96
+97 val_97
+97 val_97
+98 val_98
+98 val_98
+PREHOOK: query: select key, value from
+(
+select key, value from u1
+union all
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from
+(
+select key, value from u1
+union all
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+97 val_97
+98 val_98
+98 val_98
+PREHOOK: query: select key, value from
+(
+select key, value from u1
+union
+select key, value from u2
+union all
+select key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from
+(
+select key, value from u1
+union
+select key, value from u2
+union all
+select key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+97 val_97
+98 val_98
+98 val_98
+PREHOOK: query: select key, value from
+(
+select key, value from u1
+union distinct
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from
+(
+select key, value from u1
+union distinct
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+97 val_97
+98 val_98
+98 val_98
+PREHOOK: query: select key, value from
+(
+select key, value from u1
+union all
+select key, value from u2
+union
+select key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from
+(
+select key, value from u1
+union all
+select key, value from u2
+union
+select key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+98 val_98
+PREHOOK: query: select key, value from
+(
+select key, value from u1
+union
+select key, value from u2
+union
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from
+(
+select key, value from u1
+union
+select key, value from u2
+union
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+98 val_98
+PREHOOK: query: select distinct * from
+(
+select key, value from u1
+union all
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct * from
+(
+select key, value from u1
+union all
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+98 val_98
+PREHOOK: query: select distinct * from
+(
+select distinct * from u1
+union
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct * from
+(
+select distinct * from u1
+union
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+98 val_98
+PREHOOK: query: drop view if exists v
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: drop view if exists v
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: create view v as select distinct * from
+(
+select distinct * from u1
+union
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as select distinct * from
+(
+select distinct * from u1
+union
+select key, value from u2
+union all
+select key as key, value from u3
+) tab
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: describe extended v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: describe extended v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+key string
+value string
+
+#### A masked pattern was here ####
+(
+select distinct * from u1
+union
+select key, value from u2
+union all
+select key as key, value from u3
+) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from
+(
+select distinct `u1`.`key`, `u1`.`value` from `default`.`u1`
+union
+select `u2`.`key`, `u2`.`value` from `default`.`u2`
+union all
+select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3`
+) `tab`, tableType:VIRTUAL_VIEW)
+PREHOOK: query: select * from v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+98 val_98
+PREHOOK: query: drop view if exists v
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@v
+PREHOOK: Output: default@v
+POSTHOOK: query: drop view if exists v
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@v
+POSTHOOK: Output: default@v
+PREHOOK: query: create view v as select tab.* from
+(
+select distinct * from u1
+union
+select distinct * from u2
+) tab
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as select tab.* from
+(
+select distinct * from u1
+union
+select distinct * from u2
+) tab
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: describe extended v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: describe extended v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+key string
+value string
+
+#### A masked pattern was here ####
+(
+select distinct * from u1
+union
+select distinct * from u2
+) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from
+(
+select distinct `u1`.`key`, `u1`.`value` from `default`.`u1`
+union
+select distinct `u2`.`key`, `u2`.`value` from `default`.`u2`
+) `tab`, tableType:VIRTUAL_VIEW)
+PREHOOK: query: select * from v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+0 val_0
+10 val_10
+100 val_100
+PREHOOK: query: drop view if exists v
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@v
+PREHOOK: Output: default@v
+POSTHOOK: query: drop view if exists v
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@v
+POSTHOOK: Output: default@v
+PREHOOK: query: create view v as select * from
+(
+select distinct u1.* from u1
+union all
+select distinct * from u2
+) tab
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as select * from
+(
+select distinct u1.* from u1
+union all
+select distinct * from u2
+) tab
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: describe extended v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: describe extended v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+key string
+value string
+
+#### A masked pattern was here ####
+(
+select distinct u1.* from u1
+union all
+select distinct * from u2
+) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from
+(
+select distinct `u1`.`key`, `u1`.`value` from `default`.`u1`
+union all
+select distinct `u2`.`key`, `u2`.`value` from `default`.`u2`
+) `tab`, tableType:VIRTUAL_VIEW)
+PREHOOK: query: select * from v
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@v
+#### A masked pattern was here ####
+POSTHOOK: query: select * from v
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@v
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+10 val_10
+100 val_100
+PREHOOK: query: select distinct * from
+(
+select key, value from u1
+union all
+select key, value from u2
+union
+select key as key, value from u3
+) tab
+PREHOOK: type: QUERY
+PREHOOK: Input: default@u1
+PREHOOK: Input: default@u2
+PREHOOK: Input: default@u3
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct * from
+(
+select key, value from u1
+union all
+select key, value from u2
+union
+select key as key, value from u3
+) tab
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@u1
+POSTHOOK: Input: default@u2
+POSTHOOK: Input: default@u3
+#### A masked pattern was here ####
+0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+98 val_98