You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2019/03/15 23:38:38 UTC
[hive] 02/02: HIVE-21385: Allow disabling pushdown of
non-splittable computation to JDBC sources (Jesus Camacho Rodriguez,
reviewed by Daniel Dai, Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 8c44806c4f2316b973ae48e9292250afc40c44e4
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Mon Mar 4 16:46:37 2019 -0800
HIVE-21385: Allow disabling pushdown of non-splittable computation to JDBC sources (Jesus Camacho Rodriguez, reviewed by Daniel Dai, Ashutosh Chauhan)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 5 +-
.../test/resources/testconfiguration.properties | 1 +
.../hadoop/hive/ql/parse/CalcitePlanner.java | 32 +-
.../queries/clientpositive/external_jdbc_table3.q | 100 +++++
.../clientpositive/llap/external_jdbc_table3.q.out | 407 +++++++++++++++++++++
5 files changed, 530 insertions(+), 15 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 024f4d3..27041a7 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1698,7 +1698,10 @@ public class HiveConf extends Configuration {
HIVE_ENABLE_JDBC_PUSHDOWN("hive.jdbc.pushdown.enable", true,
"Flag to control enabling pushdown of operators into JDBC connection and subsequent SQL generation\n" +
- "using Calcite."),
+ "using Calcite"),
+ HIVE_ENABLE_JDBC_SAFE_PUSHDOWN("hive.jdbc.pushdown.safe.enable", false,
+ "Flag to control enabling pushdown of operators using Calcite that prevent splitting results\n" +
+ "retrieval in the JDBC storage handler"),
// hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row,
// need to remove by hive .13. Also, do not change default (see SMB operator)
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 20cda6a..fe3630b 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -526,6 +526,7 @@ minillaplocal.query.files=\
external_jdbc_auth.q,\
external_jdbc_table.q,\
external_jdbc_table2.q,\
+ external_jdbc_table3.q,\
external_jdbc_table_partition.q,\
external_jdbc_table_typeconversion.q,\
fullouter_mapjoin_1_optimized.q,\
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index b87c507..f0b6f8f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -1911,11 +1911,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
}
// 8. convert SemiJoin + GBy to SemiJoin
- perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
- HiveRemoveGBYSemiJoinRule.INSTANCE);
- perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Removal of gby from semijoin");
-
+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
+ HiveRemoveGBYSemiJoinRule.INSTANCE);
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Removal of gby from semijoin");
// 9. Run rule to fix windowing issue when it is done over
// aggregation columns (HIVE-10627)
@@ -1948,17 +1947,22 @@ public class CalcitePlanner extends SemanticAnalyzer {
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Druid transformation rules");
if (conf.getBoolVar(ConfVars.HIVE_ENABLE_JDBC_PUSHDOWN)) {
+ List<RelOptRule> rules = Lists.newArrayList();
+ rules.add(JDBCExtractJoinFilterRule.INSTANCE);
+ rules.add(JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_JOIN);
+ rules.add(JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_CONVERTER);
+ rules.add(JDBCFilterJoinRule.INSTANCE);
+ rules.add(JDBCFilterPushDownRule.INSTANCE);
+ rules.add(JDBCProjectPushDownRule.INSTANCE);
+ if (!conf.getBoolVar(ConfVars.HIVE_ENABLE_JDBC_SAFE_PUSHDOWN)) {
+ rules.add(JDBCJoinPushDownRule.INSTANCE);
+ rules.add(JDBCUnionPushDownRule.INSTANCE);
+ rules.add(JDBCAggregationPushDownRule.INSTANCE);
+ rules.add(JDBCSortPushDownRule.INSTANCE);
+ }
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null,
- HepMatchOrder.TOP_DOWN,
- JDBCExtractJoinFilterRule.INSTANCE,
- JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_JOIN,
- JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_CONVERTER,
- JDBCFilterJoinRule.INSTANCE,
- JDBCJoinPushDownRule.INSTANCE, JDBCUnionPushDownRule.INSTANCE,
- JDBCFilterPushDownRule.INSTANCE, JDBCProjectPushDownRule.INSTANCE,
- JDBCAggregationPushDownRule.INSTANCE, JDBCSortPushDownRule.INSTANCE
- );
+ HepMatchOrder.TOP_DOWN, rules.toArray(new RelOptRule[rules.size()]));
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: JDBC transformation rules");
}
diff --git a/ql/src/test/queries/clientpositive/external_jdbc_table3.q b/ql/src/test/queries/clientpositive/external_jdbc_table3.q
new file mode 100644
index 0000000..3b6092a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/external_jdbc_table3.q
@@ -0,0 +1,100 @@
+--! qt:dataset:src
+
+-- SORT_QUERY_RESULTS
+
+set hive.jdbc.pushdown.safe.enable=true;
+
+
+CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput';
+
+FROM src
+SELECT
+dboutput ('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1;create=true','user1','passwd1',
+'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ),
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'),
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'),
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76')
+limit 1;
+
+FROM src
+SELECT
+dboutput ('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ),
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'),
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'),
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767')
+limit 1;
+
+
+
+CREATE EXTERNAL TABLE db1_ext_auth3_1
+(
+ ikey int,
+ bkey bigint,
+ fkey float,
+ dkey double
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+ "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1;collation=TERRITORY_BASED:PRIMARY",
+ "hive.sql.dbcp.username" = "user1",
+ "hive.sql.dbcp.password" = "passwd1",
+ "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1",
+ "hive.sql.dbcp.maxActive" = "1"
+);
+
+CREATE EXTERNAL TABLE db1_ext_auth3_2
+(
+ ikey int,
+ bkey bigint,
+ fkey float,
+ dkey double
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+ "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1;collation=TERRITORY_BASED:PRIMARY",
+ "hive.sql.dbcp.username" = "user1",
+ "hive.sql.dbcp.password" = "passwd1",
+ "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2",
+ "hive.sql.dbcp.maxActive" = "1"
+);
+
+
+SELECT * FROM db1_ext_auth3_1;
+
+SELECT * FROM db1_ext_auth3_2;
+
+EXPLAIN
+SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20;
+
+SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20;
+
+EXPLAIN
+SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100;
+
+SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100;
+
+DROP TABLE db1_ext_auth3_1;
+DROP TABLE db1_ext_auth3_2;
+
+FROM src
+SELECT
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1' ),
+dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1',
+'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' )
+limit 1;
diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table3.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table3.q.out
new file mode 100644
index 0000000..dd1cda6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table3.q.out
@@ -0,0 +1,407 @@
+PREHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: dboutput
+POSTHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: dboutput
+PREHOOK: query: FROM src
+SELECT
+#### A masked pattern was here ####
+'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76')
+limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: FROM src
+SELECT
+#### A masked pattern was here ####
+'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76')
+limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 0 0 0 0
+PREHOOK: query: FROM src
+SELECT
+#### A masked pattern was here ####
+'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767')
+limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: FROM src
+SELECT
+#### A masked pattern was here ####
+'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'),
+#### A masked pattern was here ####
+'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767')
+limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 0 0 0 0
+PREHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth3_1
+(
+ ikey int,
+ bkey bigint,
+ fkey float,
+ dkey double
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "user1",
+ "hive.sql.dbcp.password" = "passwd1",
+ "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1",
+ "hive.sql.dbcp.maxActive" = "1"
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@db1_ext_auth3_1
+POSTHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth3_1
+(
+ ikey int,
+ bkey bigint,
+ fkey float,
+ dkey double
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "user1",
+ "hive.sql.dbcp.password" = "passwd1",
+ "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1",
+ "hive.sql.dbcp.maxActive" = "1"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@db1_ext_auth3_1
+PREHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth3_2
+(
+ ikey int,
+ bkey bigint,
+ fkey float,
+ dkey double
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "user1",
+ "hive.sql.dbcp.password" = "passwd1",
+ "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2",
+ "hive.sql.dbcp.maxActive" = "1"
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@db1_ext_auth3_2
+POSTHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth3_2
+(
+ ikey int,
+ bkey bigint,
+ fkey float,
+ dkey double
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+#### A masked pattern was here ####
+ "hive.sql.dbcp.username" = "user1",
+ "hive.sql.dbcp.password" = "passwd1",
+ "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2",
+ "hive.sql.dbcp.maxActive" = "1"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@db1_ext_auth3_2
+PREHOOK: query: SELECT * FROM db1_ext_auth3_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@db1_ext_auth3_1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM db1_ext_auth3_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@db1_ext_auth3_1
+#### A masked pattern was here ####
+-20 -20 -20.0 -20.0
+100 -15 65.0 -74.0
+20 20 20.0 20.0
+44 53 -455.454 330.76
+PREHOOK: query: SELECT * FROM db1_ext_auth3_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM db1_ext_auth3_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+-20 8 9.0 11.0
+101 -16 66.0 -75.0
+20 20 20.0 20.0
+40 50 -455.4543 330.767
+PREHOOK: query: EXPLAIN
+SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@db1_ext_auth3_1
+PREHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@db1_ext_auth3_1
+POSTHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: db1_ext_auth3_1
+ properties:
+ hive.sql.query SELECT "ikey", "fkey"
+FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1"
+WHERE "ikey" > 20
+ hive.sql.query.fieldNames ikey,fkey
+ hive.sql.query.fieldTypes int,float
+ hive.sql.query.split true
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ikey (type: int), fkey (type: float)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: float)
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: db1_ext_auth3_2
+ properties:
+ hive.sql.query SELECT "ikey", "bkey"
+FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2"
+WHERE "ikey" > 20
+ hive.sql.query.fieldNames ikey,bkey
+ hive.sql.query.fieldTypes int,bigint
+ hive.sql.query.split true
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ikey (type: int), bkey (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col3
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: float), _col3 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@db1_ext_auth3_1
+PREHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@db1_ext_auth3_1
+POSTHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+PREHOOK: query: EXPLAIN
+SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@db1_ext_auth3_1
+PREHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@db1_ext_auth3_1
+POSTHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Union 2 (CONTAINS)
+ Map 3 <- Union 2 (CONTAINS)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: db1_ext_auth3_1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: bkey (type: bigint), fkey (type: float)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: db1_ext_auth3_2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: bkey (type: bigint), fkey (type: float)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Union 2
+ Vertex: Union 2
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@db1_ext_auth3_1
+PREHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@db1_ext_auth3_1
+POSTHOOK: Input: default@db1_ext_auth3_2
+#### A masked pattern was here ####
+-15 65.0
+-16 66.0
+53 -455.454
+PREHOOK: query: DROP TABLE db1_ext_auth3_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@db1_ext_auth3_1
+PREHOOK: Output: default@db1_ext_auth3_1
+POSTHOOK: query: DROP TABLE db1_ext_auth3_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@db1_ext_auth3_1
+POSTHOOK: Output: default@db1_ext_auth3_1
+PREHOOK: query: DROP TABLE db1_ext_auth3_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@db1_ext_auth3_2
+PREHOOK: Output: default@db1_ext_auth3_2
+POSTHOOK: query: DROP TABLE db1_ext_auth3_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@db1_ext_auth3_2
+POSTHOOK: Output: default@db1_ext_auth3_2
+PREHOOK: query: FROM src
+SELECT
+#### A masked pattern was here ####
+'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1' ),
+#### A masked pattern was here ####
+'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' )
+limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: FROM src
+SELECT
+#### A masked pattern was here ####
+'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1' ),
+#### A masked pattern was here ####
+'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' )
+limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 0