You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/11/16 03:01:54 UTC
hive git commit: HIVE-20918: Flag to enable/disable pushdown of
computation from Calcite into JDBC connection (Jesus Camacho Rodriguez,
reviewed by Thejas M Nair)
Repository: hive
Updated Branches:
refs/heads/master 1225ca9cc -> 156ed46a3
HIVE-20918: Flag to enable/disable pushdown of computation from Calcite into JDBC connection (Jesus Camacho Rodriguez, reviewed by Thejas M Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/156ed46a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/156ed46a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/156ed46a
Branch: refs/heads/master
Commit: 156ed46a35a0aa4b2e4f8a05049d966d7894d617
Parents: 1225ca9
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed Nov 14 14:34:23 2018 -0800
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Thu Nov 15 19:01:48 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 4 +
.../hadoop/hive/ql/parse/CalcitePlanner.java | 24 +++---
.../clientpositive/external_jdbc_table2.q | 7 ++
.../llap/external_jdbc_table2.q.out | 88 ++++++++++++++++++++
4 files changed, 113 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/156ed46a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 6ca1315..00dcf50 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1682,6 +1682,10 @@ public class HiveConf extends Configuration {
HIVE_MATERIALIZED_VIEW_SERDE("hive.materializedview.serde",
"org.apache.hadoop.hive.ql.io.orc.OrcSerde", "Default SerDe used for materialized views"),
+ HIVE_ENABLE_JDBC_PUSHDOWN("hive.jdbc.pushdown.enable", true,
+ "Flag to control enabling pushdown of operators into JDBC connection and subsequent SQL generation\n" +
+ "using Calcite."),
+
// hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row,
// need to remove by hive .13. Also, do not change default (see SMB operator)
HIVEMAPJOINBUCKETCACHESIZE("hive.mapjoin.bucket.cache.size", 100, ""),
http://git-wip-us.apache.org/repos/asf/hive/blob/156ed46a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 6e9c68b..b3c6806 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -1924,16 +1924,20 @@ public class CalcitePlanner extends SemanticAnalyzer {
);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Druid transformation rules");
- calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null,
- HepMatchOrder.TOP_DOWN,
- JDBCExtractJoinFilterRule.INSTANCE,
- JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_JOIN,
- JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_CONVERTER,
- JDBCFilterJoinRule.INSTANCE,
- JDBCJoinPushDownRule.INSTANCE, JDBCUnionPushDownRule.INSTANCE,
- JDBCFilterPushDownRule.INSTANCE, JDBCProjectPushDownRule.INSTANCE,
- JDBCAggregationPushDownRule.INSTANCE, JDBCSortPushDownRule.INSTANCE
- );
+ if (conf.getBoolVar(ConfVars.HIVE_ENABLE_JDBC_PUSHDOWN)) {
+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null,
+ HepMatchOrder.TOP_DOWN,
+ JDBCExtractJoinFilterRule.INSTANCE,
+ JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_JOIN,
+ JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_CONVERTER,
+ JDBCFilterJoinRule.INSTANCE,
+ JDBCJoinPushDownRule.INSTANCE, JDBCUnionPushDownRule.INSTANCE,
+ JDBCFilterPushDownRule.INSTANCE, JDBCProjectPushDownRule.INSTANCE,
+ JDBCAggregationPushDownRule.INSTANCE, JDBCSortPushDownRule.INSTANCE
+ );
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: JDBC transformation rules");
+ }
// 11. Run rules to aid in translation from Calcite tree to Hive tree
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/156ed46a/ql/src/test/queries/clientpositive/external_jdbc_table2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/external_jdbc_table2.q b/ql/src/test/queries/clientpositive/external_jdbc_table2.q
index 558adf0..5059b44 100644
--- a/ql/src/test/queries/clientpositive/external_jdbc_table2.q
+++ b/ql/src/test/queries/clientpositive/external_jdbc_table2.q
@@ -127,6 +127,13 @@ SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2;
SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2;
+set hive.jdbc.pushdown.enable=false;
+
+EXPLAIN
+SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2;
+
+SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2;
+
DROP TABLE db1_ext_auth1;
DROP TABLE db2_ext_auth2;
DROP TABLE db1_ext_auth2;
http://git-wip-us.apache.org/repos/asf/hive/blob/156ed46a/ql/src/test/results/clientpositive/llap/external_jdbc_table2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table2.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table2.q.out
index 9bfee47..13c049f 100644
--- a/ql/src/test/results/clientpositive/llap/external_jdbc_table2.q.out
+++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table2.q.out
@@ -553,6 +553,94 @@ POSTHOOK: Input: default@db1_ext_auth2
-20 8 9.0 11.0
101 -16 66.0 -75.0
40 50 -455.4543 330.767
+PREHOOK: query: EXPLAIN
+SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@db1_ext_auth1
+PREHOOK: Input: default@db1_ext_auth2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@db1_ext_auth1
+POSTHOOK: Input: default@db1_ext_auth2
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Union 2 (CONTAINS)
+ Map 3 <- Union 2 (CONTAINS)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: db1_ext_auth1
+ Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ikey (type: int), bkey (type: bigint), fkey (type: float), dkey (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: db1_ext_auth2
+ Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ikey (type: int), bkey (type: bigint), fkey (type: float), dkey (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Union 2
+ Vertex: Union 2
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@db1_ext_auth1
+PREHOOK: Input: default@db1_ext_auth2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@db1_ext_auth1
+POSTHOOK: Input: default@db1_ext_auth2
+#### A masked pattern was here ####
+20 20 20.0 20.0
+-20 -20 -20.0 -20.0
+100 -15 65.0 -74.0
+44 53 -455.454 330.76
+20 20 20.0 20.0
+-20 8 9.0 11.0
+101 -16 66.0 -75.0
+40 50 -455.4543 330.767
PREHOOK: query: DROP TABLE db1_ext_auth1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@db1_ext_auth1