You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2016/05/30 22:26:35 UTC
hive git commit: HIVE-13807: Extend metadata provider to pull up
predicates through Union (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master d17d07ef5 -> 20c3bd9fb
HIVE-13807: Extend metadata provider to pull up predicates through Union (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/20c3bd9f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/20c3bd9f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/20c3bd9f
Branch: refs/heads/master
Commit: 20c3bd9fb25b38cb37639b9625db6b37eeac881a
Parents: d17d07e
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon May 30 15:48:31 2016 +0100
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon May 30 23:24:23 2016 +0100
----------------------------------------------------------------------
.../rules/HiveUnionPullUpConstantsRule.java | 33 +++--
.../calcite/stats/HiveRelMdPredicates.java | 50 ++++++++
.../results/clientpositive/cbo_input26.q.out | 120 +++++++++++--------
.../clientpositive/correlationoptimizer8.q.out | 6 +-
.../test/results/clientpositive/input26.q.out | 40 ++++---
ql/src/test/results/clientpositive/join34.q.out | 2 +-
ql/src/test/results/clientpositive/join35.q.out | 2 +-
.../llap/tez_union_dynamic_partition.q.out | 48 ++++----
.../results/clientpositive/spark/join34.q.out | 2 +-
.../results/clientpositive/spark/join35.q.out | 2 +-
.../tez/tez_union_dynamic_partition.q.out | 48 ++++----
.../clientpositive/unionall_unbalancedppd.q.out | 36 +++---
12 files changed, 252 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java
index 8333982..4f6c610 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java
@@ -26,6 +26,7 @@ import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Union;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataTypeField;
@@ -34,6 +35,7 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.mapping.Mappings;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
@@ -96,10 +98,11 @@ public class HiveUnionPullUpConstantsRule extends RelOptRule {
}
// Create expressions for Project operators before and after the Union
- List<Pair<RexNode, String>> newChildExprs = new ArrayList<>();
List<RelDataTypeField> fields = union.getRowType().getFieldList();
List<RexNode> topChildExprs = new ArrayList<>();
List<String> topChildExprsFields = new ArrayList<>();
+ List<RexNode> refs = new ArrayList<>();
+ ImmutableBitSet.Builder refsIndexBuilder = ImmutableBitSet.builder();
for (int i = 0; i < count ; i++) {
RexNode expr = rexBuilder.makeInputRef(union, i);
RelDataTypeField field = fields.get(i);
@@ -107,27 +110,37 @@ public class HiveUnionPullUpConstantsRule extends RelOptRule {
topChildExprs.add(constants.get(expr));
topChildExprsFields.add(field.getName());
} else {
- newChildExprs.add(Pair.<RexNode,String>of(expr, field.getName()));
topChildExprs.add(expr);
topChildExprsFields.add(field.getName());
+ refs.add(expr);
+ refsIndexBuilder.set(i);
}
}
-
- if (newChildExprs.isEmpty()) {
- // At least a single item in project is required.
- newChildExprs.add(Pair.<RexNode,String>of(
- topChildExprs.get(0), topChildExprsFields.get(0)));
- }
+ ImmutableBitSet refsIndex = refsIndexBuilder.build();
// Update top Project positions
final Mappings.TargetMapping mapping =
- RelOptUtil.permutation(Pair.left(newChildExprs), union.getInput(0).getRowType()).inverse();
+ RelOptUtil.permutation(refs, union.getInput(0).getRowType()).inverse();
topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs));
// Create new Project-Union-Project sequences
final RelBuilder relBuilder = call.builder();
for (int i = 0; i < union.getInputs().size() ; i++) {
- relBuilder.push(union.getInput(i));
+ RelNode input = union.getInput(i);
+ List<Pair<RexNode, String>> newChildExprs = new ArrayList<>();
+ for (int j = 0; j < refsIndex.cardinality(); j++ ) {
+ int pos = refsIndex.nth(j);
+ newChildExprs.add(
+ Pair.<RexNode, String>of(rexBuilder.makeInputRef(input, pos),
+ input.getRowType().getFieldList().get(pos).getName()));
+ }
+ if (newChildExprs.isEmpty()) {
+ // At least a single item in project is required.
+ newChildExprs.add(Pair.<RexNode,String>of(
+ topChildExprs.get(0), topChildExprsFields.get(0)));
+ }
+ // Add the input with project on top
+ relBuilder.push(input);
relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs));
}
relBuilder.union(union.all, union.getInputs().size());
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
index 85e66d5..9cec6ca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
@@ -18,13 +18,17 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.stats;
import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.Union;
import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMdPredicates;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -35,6 +39,7 @@ import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexPermuteInputsShuttle;
+import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.util.BuiltInMethod;
import org.apache.calcite.util.ImmutableBitSet;
@@ -126,4 +131,49 @@ public class HiveRelMdPredicates extends RelMdPredicates {
return RelOptPredicateList.of(projectPullUpPredicates);
}
+ /**
+ * Infers predicates for a Union.
+ */
+ @Override
+ public RelOptPredicateList getPredicates(Union union, RelMetadataQuery mq) {
+ RexBuilder rB = union.getCluster().getRexBuilder();
+
+ Map<String, RexNode> finalPreds = new LinkedHashMap<>();
+ Map<String, RexNode> finalResidualPreds = new LinkedHashMap<>();
+ for (int i = 0; i < union.getInputs().size(); i++) {
+ RelNode input = union.getInputs().get(i);
+ RelOptPredicateList info = mq.getPulledUpPredicates(input);
+ if (info.pulledUpPredicates.isEmpty()) {
+ return RelOptPredicateList.EMPTY;
+ }
+ Map<String, RexNode> preds = new LinkedHashMap<>();
+ for (RexNode pred : info.pulledUpPredicates) {
+ final String predString = pred.toString();
+ if (i == 0) {
+ preds.put(predString, pred);
+ continue;
+ }
+ if (finalPreds.containsKey(predString)) {
+ preds.put(predString, pred);
+ } else {
+ finalResidualPreds.put(predString, pred);
+ }
+ }
+ // Add those that are not part of the final set to residual
+ for (Entry<String, RexNode> e : finalPreds.entrySet()) {
+ if (!preds.containsKey(e.getKey())) {
+ finalResidualPreds.put(e.getKey(), e.getValue());
+ }
+ }
+ finalPreds = preds;
+ }
+
+ List<RexNode> preds = new ArrayList<>(finalPreds.values());
+ RexNode disjPred = RexUtil.composeDisjunction(rB, finalResidualPreds.values(), true);
+ if (disjPred != null) {
+ preds.add(disjPred);
+ }
+ return RelOptPredicateList.of(preds);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/cbo_input26.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_input26.q.out b/ql/src/test/results/clientpositive/cbo_input26.q.out
index c9ed67a..ecfcb3f 100644
--- a/ql/src/test/results/clientpositive/cbo_input26.q.out
+++ b/ql/src/test/results/clientpositive/cbo_input26.q.out
@@ -44,8 +44,8 @@ STAGE PLANS:
Number of rows: 5
Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '11' (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: _col0 (type: string), _col1 (type: string), '11' (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -60,23 +60,31 @@ STAGE PLANS:
TableScan
Union
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TableScan
Union
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-3
Map Reduce
@@ -108,8 +116,8 @@ STAGE PLANS:
Number of rows: 5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '14' (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: _col0 (type: string), _col1 (type: string), '14' (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
@@ -192,8 +200,8 @@ STAGE PLANS:
Number of rows: 5
Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: '2008-04-08' (type: string), _col0 (type: string), '11' (type: string)
- outputColumnNames: _col0, _col1, _col2
+ expressions: _col0 (type: string), '11' (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -208,23 +216,31 @@ STAGE PLANS:
TableScan
Union
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: '2008-04-08' (type: string), _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TableScan
Union
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: '2008-04-08' (type: string), _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-3
Map Reduce
@@ -256,8 +272,8 @@ STAGE PLANS:
Number of rows: 5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: '2008-04-08' (type: string), _col0 (type: string), '14' (type: string)
- outputColumnNames: _col0, _col1, _col2
+ expressions: _col0 (type: string), '14' (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
@@ -340,8 +356,8 @@ STAGE PLANS:
Number of rows: 5
Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: '2008-04-08' (type: string), _col0 (type: string), '11' (type: string)
- outputColumnNames: _col0, _col1, _col2
+ expressions: _col0 (type: string), '11' (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -356,23 +372,31 @@ STAGE PLANS:
TableScan
Union
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: '2008-04-08' (type: string), _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TableScan
Union
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: '2008-04-08' (type: string), _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-3
Map Reduce
@@ -404,8 +428,8 @@ STAGE PLANS:
Number of rows: 5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: '2008-04-08' (type: string), _col0 (type: string), '14' (type: string)
- outputColumnNames: _col0, _col1, _col2
+ expressions: _col0 (type: string), '14' (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/correlationoptimizer8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/correlationoptimizer8.q.out b/ql/src/test/results/clientpositive/correlationoptimizer8.q.out
index 1a51fc5..af45e39 100644
--- a/ql/src/test/results/clientpositive/correlationoptimizer8.q.out
+++ b/ql/src/test/results/clientpositive/correlationoptimizer8.q.out
@@ -103,7 +103,7 @@ STAGE PLANS:
alias: x
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean)
+ predicate: (((UDFToDouble(key) > 100.0) or (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
Statistics: Num rows: 16 Data size: 122 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -290,7 +290,7 @@ STAGE PLANS:
alias: x
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean)
+ predicate: (((UDFToDouble(key) > 100.0) or (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
Statistics: Num rows: 16 Data size: 122 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -963,7 +963,7 @@ STAGE PLANS:
alias: x
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean)
+ predicate: (((UDFToDouble(key) > 100.0) or (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
Statistics: Num rows: 16 Data size: 122 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/input26.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input26.q.out b/ql/src/test/results/clientpositive/input26.q.out
index 74035c5..70f7f65 100644
--- a/ql/src/test/results/clientpositive/input26.q.out
+++ b/ql/src/test/results/clientpositive/input26.q.out
@@ -44,8 +44,8 @@ STAGE PLANS:
Number of rows: 5
Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '11' (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: _col0 (type: string), _col1 (type: string), '11' (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -60,23 +60,31 @@ STAGE PLANS:
TableScan
Union
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TableScan
Union
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-3
Map Reduce
@@ -108,8 +116,8 @@ STAGE PLANS:
Number of rows: 5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '14' (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: _col0 (type: string), _col1 (type: string), '14' (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/join34.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join34.q.out b/ql/src/test/results/clientpositive/join34.q.out
index 365992b..ca9168e 100644
--- a/ql/src/test/results/clientpositive/join34.q.out
+++ b/ql/src/test/results/clientpositive/join34.q.out
@@ -55,7 +55,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean)
+ predicate: (((UDFToDouble(key) > 100.0) or (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
Statistics: Num rows: 16 Data size: 122 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/join35.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join35.q.out b/ql/src/test/results/clientpositive/join35.q.out
index 2c2681f..f53ae07 100644
--- a/ql/src/test/results/clientpositive/join35.q.out
+++ b/ql/src/test/results/clientpositive/join35.q.out
@@ -162,7 +162,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean)
+ predicate: (((UDFToDouble(key) > 100.0) or (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
Statistics: Num rows: 16 Data size: 122 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out b/ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out
index ce86640..faa3adb 100644
--- a/ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out
+++ b/ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out
@@ -66,17 +66,21 @@ STAGE PLANS:
alias: dummy
Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: 1 (type: int), '2014' (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
+ expressions: 1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), '2014' (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.partunion1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.partunion1
Execution mode: llap
LLAP IO: no inputs
Map 3
@@ -85,17 +89,21 @@ STAGE PLANS:
alias: dummy
Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: 2 (type: int), '2014' (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
+ expressions: 2 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), '2014' (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.partunion1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.partunion1
Execution mode: llap
LLAP IO: no inputs
Union 2
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/spark/join34.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join34.q.out b/ql/src/test/results/clientpositive/spark/join34.q.out
index ebd9c89..f243e11 100644
--- a/ql/src/test/results/clientpositive/spark/join34.q.out
+++ b/ql/src/test/results/clientpositive/spark/join34.q.out
@@ -198,7 +198,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean)
+ predicate: (((UDFToDouble(key) > 100.0) or (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
Statistics: Num rows: 16 Data size: 122 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/spark/join35.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join35.q.out b/ql/src/test/results/clientpositive/spark/join35.q.out
index d14dadf..418161c 100644
--- a/ql/src/test/results/clientpositive/spark/join35.q.out
+++ b/ql/src/test/results/clientpositive/spark/join35.q.out
@@ -212,7 +212,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean)
+ predicate: (((UDFToDouble(key) > 100.0) or (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
Statistics: Num rows: 16 Data size: 122 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out b/ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out
index a1221a4..83c6c82 100644
--- a/ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out
+++ b/ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out
@@ -66,34 +66,42 @@ STAGE PLANS:
alias: dummy
Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: 1 (type: int), '2014' (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
+ expressions: 1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), '2014' (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.partunion1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.partunion1
Map 3
Map Operator Tree:
TableScan
alias: dummy
Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: 2 (type: int), '2014' (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
+ expressions: 2 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), '2014' (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.partunion1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.partunion1
Union 2
Vertex: Union 2
http://git-wip-us.apache.org/repos/asf/hive/blob/20c3bd9f/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out b/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out
index 5e166e6..ba3a0b8 100644
--- a/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out
+++ b/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out
@@ -386,18 +386,20 @@ STAGE PLANS:
predicate: (f1 = 1) (type: boolean)
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: 1 (type: int)
- outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Union
Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: 1 (type: int)
+ outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TableScan
alias: union_all_bug_test_2
Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE
@@ -405,18 +407,20 @@ STAGE PLANS:
predicate: (f1 = 1) (type: boolean)
Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: 1 (type: int)
- outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
Union
Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: 1 (type: int)
+ outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator