You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2017/03/25 20:49:01 UTC
hive git commit: HIVE-16282: Semijoin: Disable slow-start for the
bloom filter aggregate task (Deepak Jaiswal, reviewed by Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master cd4a3d15c -> de0dfa000
HIVE-16282: Semijoin: Disable slow-start for the bloom filter aggregate task (Deepak Jaiswal, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/de0dfa00
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/de0dfa00
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/de0dfa00
Branch: refs/heads/master
Commit: de0dfa000ebcb13a871df2762b6b79e203bb9ca4
Parents: cd4a3d1
Author: Jason Dere <jd...@hortonworks.com>
Authored: Sat Mar 25 13:48:23 2017 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Sat Mar 25 13:48:23 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/tez/DagUtils.java | 19 +
.../DynamicPartitionPruningOptimization.java | 3 +
.../hadoop/hive/ql/parse/GenTezUtils.java | 4 +-
.../apache/hadoop/hive/ql/parse/GenTezWork.java | 3 +-
.../hadoop/hive/ql/plan/ReduceSinkDesc.java | 12 +-
.../apache/hadoop/hive/ql/plan/ReduceWork.java | 11 +
.../hadoop/hive/ql/plan/TezEdgeProperty.java | 12 +-
.../clientpositive/dynamic_semijoin_reduction.q | 3 +
.../llap/dynamic_semijoin_reduction.q.out | 348 +++++++++++++++++++
9 files changed, 411 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/de0dfa00/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index aa2dfc7..9589d36 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -336,6 +336,11 @@ public class DagUtils {
setupAutoReducerParallelism(edgeProp, w);
break;
}
+ case CUSTOM_SIMPLE_EDGE: {
+ setupQuickStart(edgeProp, w);
+ break;
+ }
+
default:
// nothing
}
@@ -1265,6 +1270,20 @@ public class DagUtils {
}
}
+ private void setupQuickStart(TezEdgeProperty edgeProp, Vertex v)
+ throws IOException {
+ if (!edgeProp.isSlowStart()) {
+ Configuration pluginConf = new Configuration(false);
+ VertexManagerPluginDescriptor desc =
+ VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName());
+ pluginConf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0);
+ pluginConf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, 0);
+ UserPayload payload = TezUtils.createUserPayloadFromConf(pluginConf);
+ desc.setUserPayload(payload);
+ v.setVertexManagerPlugin(desc);
+ }
+ }
+
public String createDagName(Configuration conf, QueryPlan plan) {
String name = getUserSpecifiedDagName(conf);
if (name == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/de0dfa00/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
index b9f5912..838cc69 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.optimizer;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -560,6 +561,8 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>();
rsOp.setColumnExprMap(columnExprMap);
+ rsOp.getConf().setReducerTraits(EnumSet.of(ReduceSinkDesc.ReducerTraits.QUICKSTART));
+
// Create the final Group By Operator
ArrayList<AggregationDesc> aggsFinal = new ArrayList<AggregationDesc>();
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/de0dfa00/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index 905431f..d58f447 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -95,6 +95,7 @@ public class GenTezUtils {
ReduceSinkOperator reduceSink = (ReduceSinkOperator) context.parentOfRoot;
reduceWork.setNumReduceTasks(reduceSink.getConf().getNumReducers());
+ reduceWork.setSlowStart(reduceSink.getConf().isSlowStart());
if (isAutoReduceParallelism && reduceSink.getConf().getReducerTraits().contains(AUTOPARALLEL)) {
@@ -132,10 +133,11 @@ public class GenTezUtils {
EdgeType edgeType = determineEdgeType(context.preceedingWork, reduceWork, reduceSink);
if (reduceWork.isAutoReduceParallelism()) {
edgeProp =
- new TezEdgeProperty(context.conf, edgeType, true,
+ new TezEdgeProperty(context.conf, edgeType, true, reduceWork.isSlowStart(),
reduceWork.getMinReduceTasks(), reduceWork.getMaxReduceTasks(), bytesPerReducer);
} else {
edgeProp = new TezEdgeProperty(edgeType);
+ edgeProp.setSlowStart(reduceWork.isSlowStart());
}
tezWork.connect(
http://git-wip-us.apache.org/repos/asf/hive/blob/de0dfa00/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
index 97f3300..c87de16 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
@@ -455,10 +455,11 @@ public class GenTezWork implements NodeProcessor {
EdgeType edgeType = GenTezUtils.determineEdgeType(work, followingWork, rs);
if (rWork.isAutoReduceParallelism()) {
edgeProp =
- new TezEdgeProperty(context.conf, edgeType, true,
+ new TezEdgeProperty(context.conf, edgeType, true, rWork.isSlowStart(),
rWork.getMinReduceTasks(), rWork.getMaxReduceTasks(), bytesPerReducer);
} else {
edgeProp = new TezEdgeProperty(edgeType);
+ edgeProp.setSlowStart(rWork.isSlowStart());
}
tezWork.connect(work, followingWork, edgeProp);
context.connectedReduceSinks.add(rs);
http://git-wip-us.apache.org/repos/asf/hive/blob/de0dfa00/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
index 38461d5..79d19b5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
@@ -109,7 +109,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
UNSET(0), // unset
FIXED(1), // distribution of keys is fixed
AUTOPARALLEL(2), // can change reducer count (ORDER BY can concat adjacent buckets)
- UNIFORM(3); // can redistribute into buckets uniformly (GROUP BY can)
+ UNIFORM(3), // can redistribute into buckets uniformly (GROUP BY can)
+ QUICKSTART(4); // do not wait for downstream tasks
private final int trait;
@@ -441,6 +442,15 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
return (this.reduceTraits.contains(ReducerTraits.AUTOPARALLEL));
}
+ public final boolean isSlowStart() {
+ return !(this.reduceTraits.contains(ReducerTraits.QUICKSTART));
+ }
+
+ @Explain(displayName = "quick start", displayOnlyOnTrue = true, explainLevels = {Explain.Level.EXTENDED })
+ public final boolean isQuickStart() {
+ return !isSlowStart();
+ }
+
public final EnumSet<ReducerTraits> getReducerTraits() {
return this.reduceTraits;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/de0dfa00/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
index ee784dc..dfed017 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
@@ -86,6 +86,9 @@ public class ReduceWork extends BaseWork {
// boolean that says whether tez auto reduce parallelism should be used
private boolean isAutoReduceParallelism;
+ // boolean that says whether to slow start or not
+ private boolean isSlowStart = true;
+
// for auto reduce parallelism - minimum reducers requested
private int minReduceTasks;
@@ -217,6 +220,14 @@ public class ReduceWork extends BaseWork {
return isAutoReduceParallelism;
}
+ public boolean isSlowStart() {
+ return isSlowStart;
+ }
+
+ public void setSlowStart(boolean isSlowStart) {
+ this.isSlowStart = isSlowStart;
+ }
+
public void setMinReduceTasks(int minReduceTasks) {
this.minReduceTasks = minReduceTasks;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/de0dfa00/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
index a3aa12f..d87bee3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
@@ -35,6 +35,7 @@ public class TezEdgeProperty {
private int numBuckets;
private boolean isAutoReduce;
+ private boolean isSlowStart = true;
private int minReducer;
private int maxReducer;
private long inputSizePerReducer;
@@ -47,12 +48,13 @@ public class TezEdgeProperty {
}
public TezEdgeProperty(HiveConf hiveConf, EdgeType edgeType, boolean isAutoReduce,
- int minReducer, int maxReducer, long bytesPerReducer) {
+ boolean isSlowStart, int minReducer, int maxReducer, long bytesPerReducer) {
this(hiveConf, edgeType, -1);
this.minReducer = minReducer;
this.maxReducer = maxReducer;
this.isAutoReduce = isAutoReduce;
this.inputSizePerReducer = bytesPerReducer;
+ this.isSlowStart = isSlowStart;
}
public TezEdgeProperty(EdgeType edgeType) {
@@ -86,4 +88,12 @@ public class TezEdgeProperty {
public long getInputSizePerReducer() {
return inputSizePerReducer;
}
+
+ public boolean isSlowStart() {
+ return isSlowStart;
+ }
+
+ public void setSlowStart(boolean slowStart) {
+ this.isSlowStart = slowStart;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/de0dfa00/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
index 6338ac3..d631401 100644
--- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
+++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
@@ -71,6 +71,9 @@ select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcp
set hive.tez.dynamic.semijoin.reduction=true;
EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) join alltypesorc_int on (srcpart_date.value = alltypesorc_int.cstring);
select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) join alltypesorc_int on (srcpart_date.value = alltypesorc_int.cstring);
+
+-- Explain extended to verify fast start for Reducer in semijoin branch
+EXPLAIN extended select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1);
set hive.tez.dynamic.semijoin.reduction=false;
-- With Mapjoins.
http://git-wip-us.apache.org/repos/asf/hive/blob/de0dfa00/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
index c1dd8c0..a47ce6e 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
@@ -1555,6 +1555,354 @@ POSTHOOK: Input: default@srcpart_small@ds=2008-04-08
POSTHOOK: Input: default@srcpart_small@ds=2008-04-09
#### A masked pattern was here ####
0
+PREHOOK: query: EXPLAIN extended select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN extended select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Reducer 5 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+ Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_date
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: 0
+ auto parallelism: true
+ Execution mode: llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-08
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ partition values:
+ ds 2008-04-08
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+#### A masked pattern was here ####
+ name default.srcpart_date
+ numFiles 1
+ numRows 1000
+ partition_columns ds
+ partition_columns.types string
+ rawDataSize 176000
+ serialization.ddl struct srcpart_date { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ totalSize 3038
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ properties:
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart_date
+ partition_columns ds
+ partition_columns.types string
+ serialization.ddl struct srcpart_date { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_date
+ name: default.srcpart_date
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-09
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ partition values:
+ ds 2008-04-09
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+#### A masked pattern was here ####
+ name default.srcpart_date
+ numFiles 1
+ numRows 1000
+ partition_columns ds
+ partition_columns.types string
+ rawDataSize 176000
+ serialization.ddl struct srcpart_date { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ totalSize 3038
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ properties:
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart_date
+ partition_columns ds
+ partition_columns.types string
+ serialization.ddl struct srcpart_date { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_date
+ name: default.srcpart_date
+ Truncated Path -> Alias:
+ /srcpart_date/ds=2008-04-08 [srcpart_date]
+ /srcpart_date/ds=2008-04-09 [srcpart_date]
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_small
+ filterExpr: key1 is not null (type: boolean)
+ Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: key1 is not null (type: boolean)
+ Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
+ tag: 1
+ auto parallelism: true
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=32)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ tag: -1
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ auto parallelism: false
+ quick start: true
+ Execution mode: llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-08
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ partition values:
+ ds 2008-04-08
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ bucket_count -1
+#### A masked pattern was here ####
+ name default.srcpart_small
+ numFiles 0
+ numRows 0
+ partition_columns ds
+ partition_columns.types string
+ rawDataSize 0
+ serialization.ddl struct srcpart_small { string key1, string value1}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ totalSize 0
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ properties:
+ bucket_count -1
+ column.name.delimiter ,
+ columns key1,value1
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart_small
+ partition_columns ds
+ partition_columns.types string
+ serialization.ddl struct srcpart_small { string key1, string value1}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_small
+ name: default.srcpart_small
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-09
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ partition values:
+ ds 2008-04-09
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key1":"true","value1":"true"}}
+ bucket_count -1
+#### A masked pattern was here ####
+ name default.srcpart_small
+ numFiles 1
+ numRows 20
+ partition_columns ds
+ partition_columns.types string
+ rawDataSize 3520
+ serialization.ddl struct srcpart_small { string key1, string value1}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ totalSize 459
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ properties:
+ bucket_count -1
+ column.name.delimiter ,
+ columns key1,value1
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart_small
+ partition_columns ds
+ partition_columns.types string
+ serialization.ddl struct srcpart_small { string key1, string value1}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_small
+ name: default.srcpart_small
+ Truncated Path -> Alias:
+ /srcpart_small/ds=2008-04-08 [srcpart_small]
+ /srcpart_small/ds=2008-04-09 [srcpart_small]
+ Reducer 2
+ Execution mode: llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ Position of Big Table: 0
+ Statistics: Num rows: 195 Data size: 1560 Basic stats: COMPLETE Column stats: PARTIAL
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Reducer 3
+ Execution mode: llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Reducer 5
+ Execution mode: llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=32)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ tag: -1
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ auto parallelism: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1)
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1)