You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2017/09/01 18:11:29 UTC
hive git commit: HIVE-17399: Semijoin: Do not remove semijoin branch
if it feeds to TS->DPP_EVENT (Deepak Jaiswal, reviewed by Gopal V)
Repository: hive
Updated Branches:
refs/heads/master b82d38aa3 -> 2e226d22f
HIVE-17399: Semijoin: Do not remove semijoin branch if it feeds to TS->DPP_EVENT (Deepak Jaiswal, reviewed by Gopal V)
Signed-off-by: Gopal V <go...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2e226d22
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2e226d22
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2e226d22
Branch: refs/heads/master
Commit: 2e226d22facea6ea362dc90ce37ae59aeb1666a0
Parents: b82d38a
Author: Deepak Jaiswal <dj...@hortonworks.com>
Authored: Fri Sep 1 10:51:52 2017 -0700
Committer: Gopal V <go...@apache.org>
Committed: Fri Sep 1 10:52:44 2017 -0700
----------------------------------------------------------------------
.../hive/ql/parse/SemiJoinBranchInfo.java | 18 +
.../hadoop/hive/ql/parse/TezCompiler.java | 105 +++++-
.../clientpositive/dynamic_semijoin_reduction.q | 10 +
.../llap/dynamic_semijoin_reduction.q.out | 365 +++++++++++++++++++
4 files changed, 490 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2e226d22/ql/src/java/org/apache/hadoop/hive/ql/parse/SemiJoinBranchInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemiJoinBranchInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemiJoinBranchInfo.java
index 5d7b9e5..c960b05 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemiJoinBranchInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemiJoinBranchInfo.java
@@ -24,15 +24,21 @@ import org.apache.hadoop.hive.ql.exec.TableScanOperator;
public class SemiJoinBranchInfo {
private TableScanOperator ts;
private boolean isHint;
+ // Default value is true, however, if an optimization deems this edge
+ // important, it should set this to false. This does not guarantee that
+ // the edge will stay, however, it increases the chances.
+ private boolean shouldRemove;
public SemiJoinBranchInfo(TableScanOperator ts) {
this.ts = ts;
isHint = false;
+ shouldRemove = true;
}
public SemiJoinBranchInfo(TableScanOperator ts, boolean isHint) {
this.ts = ts;
this.isHint = isHint;
+ shouldRemove = !isHint; // If hint is true, shouldRemove is redundant anyway
}
public TableScanOperator getTsOp() {
@@ -42,4 +48,16 @@ public class SemiJoinBranchInfo {
public boolean getIsHint() {
return isHint;
}
+
+ public boolean getShouldRemove() {
+ return shouldRemove;
+ }
+
+ public void setShouldRemove(boolean shouldRemove) {
+ // The state only changes from true->false
+ // Once set to false, it may not change back to true
+ if (this.shouldRemove) {
+ this.shouldRemove = shouldRemove;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2e226d22/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index 5921594..15836ec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -165,8 +165,14 @@ public class TezCompiler extends TaskCompiler {
runRemoveDynamicPruningOptimization(procCtx, inputs, outputs);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run remove dynamic pruning by size");
+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
+ markSemiJoinForDPP(procCtx);
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Mark certain semijoin edges important based ");
+
// Removing semijoin optimization when it may not be beneficial
+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
removeSemijoinOptimizationByBenefit(procCtx);
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Remove Semijoins based on cost benefits");
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
// Remove any parallel edge between semijoin and mapjoin.
@@ -955,12 +961,14 @@ public class TezCompiler extends TaskCompiler {
if (ts.getStatistics() != null) {
long numRows = ts.getStatistics().getNumRows();
if (numRows < pCtx.getConf().getLongVar(ConfVars.TEZ_BIGTABLE_MIN_SIZE_SEMIJOIN_REDUCTION)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Insufficient rows (" + numRows + ") to justify semijoin optimization. Removing semijoin "
- + OperatorUtils.getOpNamePretty(rs) + " - " + OperatorUtils.getOpNamePretty(ts));
+ if (sjInfo.getShouldRemove()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Insufficient rows (" + numRows + ") to justify semijoin optimization. Removing semijoin "
+ + OperatorUtils.getOpNamePretty(rs) + " - " + OperatorUtils.getOpNamePretty(ts));
+ }
+ GenTezUtils.removeBranch(rs);
+ GenTezUtils.removeSemiJoinOperator(pCtx, rs, ts);
}
- GenTezUtils.removeBranch(rs);
- GenTezUtils.removeSemiJoinOperator(pCtx, rs, ts);
}
}
return null;
@@ -1055,7 +1063,7 @@ public class TezCompiler extends TaskCompiler {
parallelEdges = true;
- if (sjInfo.getIsHint()) {
+ if (sjInfo.getIsHint() || !sjInfo.getShouldRemove()) {
// Created by hint, skip it
continue;
}
@@ -1295,8 +1303,8 @@ public class TezCompiler extends TaskCompiler {
HiveConf.ConfVars.TEZ_DYNAMIC_SEMIJOIN_REDUCTION_THRESHOLD);
for (ReduceSinkOperator rs : map.keySet()) {
SemiJoinBranchInfo sjInfo = map.get(rs);
- if (sjInfo.getIsHint()) {
- // Semijoin created using hint, skip it
+ if (sjInfo.getIsHint() || !sjInfo.getShouldRemove()) {
+ // Semijoin created using hint or marked useful, skip it
continue;
}
// rs is semijoin optimization branch, which should look like <Parent>-SEL-GB1-RS1-GB2-RS2
@@ -1342,4 +1350,85 @@ public class TezCompiler extends TaskCompiler {
GenTezUtils.removeSemiJoinOperator(procCtx.parseContext, rs, ts);
}
}
+
+ private void markSemiJoinForDPP(OptimizeTezProcContext procCtx)
+ throws SemanticException {
+ if(!procCtx.conf.getBoolVar(ConfVars.TEZ_DYNAMIC_SEMIJOIN_REDUCTION)) {
+ // Not needed without semi-join reduction
+ return;
+ }
+
+ // Stores the Tablescan operators processed to avoid redoing them.
+ Map<TableScanOperator, TableScanOperator> tsOps = new HashMap<>();
+ Map<ReduceSinkOperator, SemiJoinBranchInfo> map = procCtx.parseContext.getRsToSemiJoinBranchInfo();
+
+ for (ReduceSinkOperator rs : map.keySet()) {
+ SemiJoinBranchInfo sjInfo = map.get(rs);
+ TableScanOperator ts = sjInfo.getTsOp();
+ TableScanOperator tsInMap = tsOps.putIfAbsent(ts, ts);
+ if (tsInMap != null) {
+ // Already processed, skip
+ continue;
+ }
+
+ if (sjInfo.getIsHint() || !sjInfo.getShouldRemove()) {
+ continue;
+ }
+
+ // A TS can have multiple branches due to DPP Or Semijoin Opt.
+ // Use DFS to traverse all the branches until RS or DPP is hit.
+ Deque<Operator<?>> deque = new LinkedList<>();
+ deque.add(ts);
+ while (!deque.isEmpty()) {
+ Operator<?> op = deque.pollLast();
+ if (op instanceof AppMasterEventOperator &&
+ ((AppMasterEventOperator) op).getConf() instanceof DynamicPruningEventDesc) {
+ // DPP. Now look up nDVs on both sides to see the selectivity.
+ // <Parent Ops>-SEL-GB1-RS1-GB2-RS2
+ SelectOperator selOp = null;
+ try {
+ selOp = (SelectOperator)
+ (rs.getParentOperators().get(0)
+ .getParentOperators().get(0)
+ .getParentOperators().get(0)
+ .getParentOperators().get(0));
+ } catch (NullPointerException e) {
+ LOG.warn("markSemiJoinForDPP : Null pointer exception caught while accessing semijoin operators");
+ assert false;
+ return;
+ }
+ try {
+ // If stats are not available, just assume its a useful edge
+ Statistics stats = selOp.getStatistics();
+ ExprNodeColumnDesc colExpr = ExprNodeDescUtils.getColumnExpr(
+ selOp.getConf().getColList().get(0));
+ long nDVs = stats.getColumnStatisticsFromColName(
+ colExpr.getColumn()).getCountDistint();
+ if (nDVs > 0) {
+ // Lookup nDVs on TS side.
+ RuntimeValuesInfo rti = procCtx.parseContext
+ .getRsToRuntimeValuesInfoMap().get(rs);
+ ExprNodeDesc tsExpr = rti.getTsColExpr();
+ FilterOperator fil = (FilterOperator) (ts.getChildOperators().get(0));
+ Statistics filStats = fil.getStatistics();
+ ExprNodeColumnDesc tsColExpr = ExprNodeDescUtils.getColumnExpr(tsExpr);
+ long nDVsOfTS = filStats.getColumnStatisticsFromColName(
+ tsColExpr.getColumn()).getCountDistint();
+ if (nDVsOfTS >= nDVs) {
+ sjInfo.setShouldRemove(false);
+ }
+ }
+ } catch (NullPointerException e) {
+ sjInfo.setShouldRemove(false);
+ }
+ break;
+ }
+ if (op instanceof ReduceSinkOperator) {
+ // Done with this branch
+ continue;
+ }
+ deque.addAll(op.getChildOperators());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2e226d22/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
index a36e981..24933db 100644
--- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
+++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
@@ -113,6 +113,16 @@ set hive.tez.dynamic.semijoin.reduction=true;
EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring);
select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring);
+-- HIVE-17399
+create table srcpart_small10 as select * from srcpart_small limit 10;
+analyze table srcpart_small10 compute statistics for columns;
+set hive.tez.dynamic.semijoin.reduction=false;
+EXPLAIN select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds;
+select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds;
+set hive.tez.dynamic.semijoin.reduction=true;
+EXPLAIN select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds;
+select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds;
+
-- With unions
explain select * from alltypesorc_int join
(select srcpart_date.key as key from srcpart_date
http://git-wip-us.apache.org/repos/asf/hive/blob/2e226d22/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
index 61dcf3b..24fbc61 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
@@ -3046,6 +3046,371 @@ POSTHOOK: Input: default@srcpart_small@ds=2008-04-08
POSTHOOK: Input: default@srcpart_small@ds=2008-04-09
#### A masked pattern was here ####
0
+PREHOOK: query: create table srcpart_small10 as select * from srcpart_small limit 10
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@srcpart_small
+PREHOOK: Input: default@srcpart_small@ds=2008-04-08
+PREHOOK: Input: default@srcpart_small@ds=2008-04-09
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcpart_small10
+POSTHOOK: query: create table srcpart_small10 as select * from srcpart_small limit 10
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@srcpart_small
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-09
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcpart_small10
+POSTHOOK: Lineage: srcpart_small10.ds SIMPLE [(srcpart_small)srcpart_small.FieldSchema(name:ds, type:string, comment:null), ]
+POSTHOOK: Lineage: srcpart_small10.key1 SIMPLE [(srcpart_small)srcpart_small.FieldSchema(name:key1, type:string, comment:null), ]
+POSTHOOK: Lineage: srcpart_small10.value1 SIMPLE [(srcpart_small)srcpart_small.FieldSchema(name:value1, type:string, comment:null), ]
+PREHOOK: query: analyze table srcpart_small10 compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_small10
+PREHOOK: Output: default@srcpart_small10
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_small10 compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_small10
+POSTHOOK: Output: default@srcpart_small10
+#### A masked pattern was here ####
+PREHOOK: query: EXPLAIN select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+ Map 3 <- Map 1 (BROADCAST_EDGE)
+ Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_small
+ filterExpr: key1 is not null (type: boolean)
+ Statistics: Num rows: 20 Data size: 5420 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key1 is not null (type: boolean)
+ Statistics: Num rows: 20 Data size: 5420 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key1 (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 20 Data size: 5420 Basic stats: COMPLETE Column stats: PARTIAL
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: _col1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: PARTIAL
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL
+ Dynamic Partitioning Event Operator
+ Target column: ds (string)
+ Target Input: srcpart_date
+ Partition key expr: ds
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL
+ Target Vertex: Map 3
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_small10
+ filterExpr: key1 is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key1 is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_date
+ filterExpr: ds is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 720000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: ds (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 10000 Data size: 80000 Basic stats: COMPLETE Column stats: PARTIAL
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_date
+PREHOOK: Input: default@srcpart_date@ds=2008-04-08
+PREHOOK: Input: default@srcpart_date@ds=2008-04-09
+PREHOOK: Input: default@srcpart_small
+PREHOOK: Input: default@srcpart_small10
+PREHOOK: Input: default@srcpart_small@ds=2008-04-08
+PREHOOK: Input: default@srcpart_small@ds=2008-04-09
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_date
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-09
+POSTHOOK: Input: default@srcpart_small
+POSTHOOK: Input: default@srcpart_small10
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-09
+#### A masked pattern was here ####
+10000
+PREHOOK: query: EXPLAIN select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE), Reducer 3 (BROADCAST_EDGE)
+ Map 4 <- Map 1 (BROADCAST_EDGE)
+ Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE)
+ Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_small
+ filterExpr: (key1 is not null and (key1 BETWEEN DynamicValue(RS_10_srcpart_small10_key1_min) AND DynamicValue(RS_10_srcpart_small10_key1_max) and in_bloom_filter(key1, DynamicValue(RS_10_srcpart_small10_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 20 Data size: 5420 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: (key1 is not null and (key1 BETWEEN DynamicValue(RS_10_srcpart_small10_key1_min) AND DynamicValue(RS_10_srcpart_small10_key1_max) and in_bloom_filter(key1, DynamicValue(RS_10_srcpart_small10_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 20 Data size: 5420 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key1 (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 20 Data size: 5420 Basic stats: COMPLETE Column stats: PARTIAL
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: _col1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: PARTIAL
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL
+ Dynamic Partitioning Event Operator
+ Target column: ds (string)
+ Target Input: srcpart_date
+ Partition key expr: ds
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL
+ Target Vertex: Map 4
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_small10
+ filterExpr: key1 is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key1 is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=20)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_date
+ filterExpr: ds is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 720000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: ds (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 10000 Data size: 80000 Basic stats: COMPLETE Column stats: PARTIAL
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=20)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_date
+PREHOOK: Input: default@srcpart_date@ds=2008-04-08
+PREHOOK: Input: default@srcpart_date@ds=2008-04-09
+PREHOOK: Input: default@srcpart_small
+PREHOOK: Input: default@srcpart_small10
+PREHOOK: Input: default@srcpart_small@ds=2008-04-08
+PREHOOK: Input: default@srcpart_small@ds=2008-04-09
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_small10, srcpart_small, srcpart_date where srcpart_small.key1 = srcpart_small10.key1 and srcpart_date.ds = srcpart_small.ds
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_date
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-09
+POSTHOOK: Input: default@srcpart_small
+POSTHOOK: Input: default@srcpart_small10
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-09
+#### A masked pattern was here ####
+10000
PREHOOK: query: explain select * from alltypesorc_int join
(select srcpart_date.key as key from srcpart_date
union all