You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2015/05/10 19:38:40 UTC
hive git commit: HIVE-10568 : Select count(distinct()) can have more
optimal execution plan (Ashutosh Chauhan via John Pullokkaran)
Repository: hive
Updated Branches:
refs/heads/master 08b0708b4 -> 1538a71ef
HIVE-10568 : Select count(distinct()) can have more optimal execution plan (Ashutosh Chauhan via John Pullokkaran)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1538a71e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1538a71e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1538a71e
Branch: refs/heads/master
Commit: 1538a71ef55c315b9d5e620f63be5895ac2dbb76
Parents: 08b0708
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Thu Apr 30 18:33:09 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sun May 10 10:37:02 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 5 +-
.../rules/HiveExpandDistinctAggregatesRule.java | 278 +++++++++++++++++++
.../hadoop/hive/ql/parse/CalcitePlanner.java | 13 +-
.../clientpositive/tez/limit_pushdown.q.out | 31 ++-
.../test/results/clientpositive/tez/mrr.q.out | 52 ++--
.../tez/vector_count_distinct.q.out | 28 +-
.../tez/vectorization_limit.q.out | 31 ++-
.../tez/vectorized_distinct_gby.q.out | 51 +++-
8 files changed, 417 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1538a71e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5d5a928..115a5d0 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1214,7 +1214,10 @@ public class HiveConf extends Configuration {
HIVESAMPLINGNUMBERFORORDERBY("hive.optimize.sampling.orderby.number", 1000, "Total number of samples to be obtained."),
HIVESAMPLINGPERCENTFORORDERBY("hive.optimize.sampling.orderby.percent", 0.1f, new RatioValidator(),
"Probability with which a row will be chosen."),
-
+ HIVEOPTIMIZEDISTINCTREWRITE("hive.optimize.distinct.rewrite", true, "When applicable this "
+ + "optimization rewrites distinct aggregates from a single stage to multi-stage "
+ + "aggregation. This may not be optimal in all cases. Ideally, whether to trigger it or "
+ + "not should be cost based decision. Until Hive formalizes cost model for this, this is config driven."),
// whether to optimize union followed by select followed by filesink
// It creates sub-directories in the final output, so should not be turned on in systems
// where MAPREDUCE-1501 is not present
http://git-wip-us.apache.org/repos/asf/hive/blob/1538a71e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
new file mode 100644
index 0000000..73c7cac
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
@@ -0,0 +1,278 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.metadata.RelColumnOrigin;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Planner rule that expands distinct aggregates
+ * (such as {@code COUNT(DISTINCT x)}) from a
+ * {@link org.apache.calcite.rel.core.Aggregate}.
+ *
+ * <p>How this is done depends upon the arguments to the function. If all
+ * functions have the same argument
+ * (e.g. {@code COUNT(DISTINCT x), SUM(DISTINCT x)} both have the argument
+ * {@code x}) then one extra {@link org.apache.calcite.rel.core.Aggregate} is
+ * sufficient.
+ *
+ * <p>If there are multiple arguments
+ * (e.g. {@code COUNT(DISTINCT x), COUNT(DISTINCT y)})
+ * the rule creates separate {@code Aggregate}s and combines using a
+ * {@link org.apache.calcite.rel.core.Join}.
+ */
+
+// Stripped down version of org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule
+// This is adapted for Hive, but should eventually be deleted from Hive and make use of above.
+
+public final class HiveExpandDistinctAggregatesRule extends RelOptRule {
+ //~ Static fields/initializers ---------------------------------------------
+
+ /** The default instance of the rule; operates only on logical expressions. */
+ public static final HiveExpandDistinctAggregatesRule INSTANCE =
+ new HiveExpandDistinctAggregatesRule(HiveAggregate.class,
+ HiveProject.DEFAULT_PROJECT_FACTORY);
+
+ private static RelFactories.ProjectFactory projFactory;
+
+ //~ Constructors -----------------------------------------------------------
+
+ public HiveExpandDistinctAggregatesRule(
+ Class<? extends Aggregate> clazz,RelFactories.ProjectFactory projectFactory) {
+ super(operand(clazz, any()));
+ projFactory = projectFactory;
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final Aggregate aggregate = call.rel(0);
+ if (!aggregate.containsDistinctCall()) {
+ return;
+ }
+
+ // Find all of the agg expressions. We use a LinkedHashSet to ensure
+ // determinism.
+ int nonDistinctCount = 0;
+ Set<List<Integer>> argListSets = new LinkedHashSet<List<Integer>>();
+ for (AggregateCall aggCall : aggregate.getAggCallList()) {
+ if (!aggCall.isDistinct()) {
+ ++nonDistinctCount;
+ continue;
+ }
+ ArrayList<Integer> argList = new ArrayList<Integer>();
+ for (Integer arg : aggCall.getArgList()) {
+ argList.add(arg);
+ }
+ argListSets.add(argList);
+ }
+ Util.permAssert(argListSets.size() > 0, "containsDistinctCall lied");
+
+ // If all of the agg expressions are distinct and have the same
+ // arguments then we can use a more efficient form.
+ if ((nonDistinctCount == 0) && (argListSets.size() == 1)) {
+ for (Integer arg : argListSets.iterator().next()) {
+ Set<RelColumnOrigin> colOrigs = RelMetadataQuery.getColumnOrigins(aggregate, arg);
+ if (null != colOrigs) {
+ for (RelColumnOrigin colOrig : colOrigs) {
+ RelOptHiveTable hiveTbl = (RelOptHiveTable)colOrig.getOriginTable();
+ if(hiveTbl.getPartColInfoMap().containsKey(colOrig.getOriginColumnOrdinal())) {
+ // Encountered partitioning column, this will be better handled by MetadataOnly optimizer.
+ return;
+ }
+ }
+ }
+ }
+ RelNode converted =
+ convertMonopole(
+ aggregate,
+ argListSets.iterator().next());
+ call.transformTo(converted);
+ return;
+ }
+ }
+
+ /**
+ * Converts an aggregate relational expression that contains just one
+ * distinct aggregate function (or perhaps several over the same arguments)
+ * and no non-distinct aggregate functions.
+ */
+ private RelNode convertMonopole(
+ Aggregate aggregate,
+ List<Integer> argList) {
+ // For example,
+ // SELECT deptno, COUNT(DISTINCT sal), SUM(DISTINCT sal)
+ // FROM emp
+ // GROUP BY deptno
+ //
+ // becomes
+ //
+ // SELECT deptno, COUNT(distinct_sal), SUM(distinct_sal)
+ // FROM (
+ // SELECT DISTINCT deptno, sal AS distinct_sal
+ // FROM EMP GROUP BY deptno)
+ // GROUP BY deptno
+
+ // Project the columns of the GROUP BY plus the arguments
+ // to the agg function.
+ Map<Integer, Integer> sourceOf = new HashMap<Integer, Integer>();
+ final Aggregate distinct =
+ createSelectDistinct(aggregate, argList, sourceOf);
+
+ // Create an aggregate on top, with the new aggregate list.
+ final List<AggregateCall> newAggCalls =
+ Lists.newArrayList(aggregate.getAggCallList());
+ rewriteAggCalls(newAggCalls, argList, sourceOf);
+ final int cardinality = aggregate.getGroupSet().cardinality();
+ return aggregate.copy(aggregate.getTraitSet(), distinct,
+ aggregate.indicator, ImmutableBitSet.range(cardinality), null,
+ newAggCalls);
+ }
+
+ private static void rewriteAggCalls(
+ List<AggregateCall> newAggCalls,
+ List<Integer> argList,
+ Map<Integer, Integer> sourceOf) {
+ // Rewrite the agg calls. Each distinct agg becomes a non-distinct call
+ // to the corresponding field from the right; for example,
+ // "COUNT(DISTINCT e.sal)" becomes "COUNT(distinct_e.sal)".
+ for (int i = 0; i < newAggCalls.size(); i++) {
+ final AggregateCall aggCall = newAggCalls.get(i);
+
+ // Ignore agg calls which are not distinct or have the wrong set
+ // arguments. If we're rewriting aggs whose args are {sal}, we will
+ // rewrite COUNT(DISTINCT sal) and SUM(DISTINCT sal) but ignore
+ // COUNT(DISTINCT gender) or SUM(sal).
+ if (!aggCall.isDistinct()) {
+ continue;
+ }
+ if (!aggCall.getArgList().equals(argList)) {
+ continue;
+ }
+
+ // Re-map arguments.
+ final int argCount = aggCall.getArgList().size();
+ final List<Integer> newArgs = new ArrayList<Integer>(argCount);
+ for (int j = 0; j < argCount; j++) {
+ final Integer arg = aggCall.getArgList().get(j);
+ newArgs.add(sourceOf.get(arg));
+ }
+ final AggregateCall newAggCall =
+ new AggregateCall(
+ aggCall.getAggregation(),
+ false,
+ newArgs,
+ aggCall.getType(),
+ aggCall.getName());
+ newAggCalls.set(i, newAggCall);
+ }
+ }
+
+ /**
+ * Given an {@link org.apache.calcite.rel.logical.LogicalAggregate}
+ * and the ordinals of the arguments to a
+ * particular call to an aggregate function, creates a 'select distinct'
+ * relational expression which projects the group columns and those
+ * arguments but nothing else.
+ *
+ * <p>For example, given
+ *
+ * <blockquote>
+ * <pre>select f0, count(distinct f1), count(distinct f2)
+ * from t group by f0</pre>
+ * </blockquote>
+ *
+ * and the arglist
+ *
+ * <blockquote>{2}</blockquote>
+ *
+ * returns
+ *
+ * <blockquote>
+ * <pre>select distinct f0, f2 from t</pre>
+ * </blockquote>
+ *
+ * '
+ *
+ * <p>The <code>sourceOf</code> map is populated with the source of each
+ * column; in this case sourceOf.get(0) = 0, and sourceOf.get(1) = 2.</p>
+ *
+ * @param aggregate Aggregate relational expression
+ * @param argList Ordinals of columns to make distinct
+ * @param sourceOf Out parameter, is populated with a map of where each
+ * output field came from
+ * @return Aggregate relational expression which projects the required
+ * columns
+ */
+ private static Aggregate createSelectDistinct(
+ Aggregate aggregate,
+ List<Integer> argList,
+ Map<Integer, Integer> sourceOf) {
+ final List<Pair<RexNode, String>> projects =
+ new ArrayList<Pair<RexNode, String>>();
+ final RelNode child = aggregate.getInput();
+ final List<RelDataTypeField> childFields =
+ child.getRowType().getFieldList();
+ for (int i : aggregate.getGroupSet()) {
+ sourceOf.put(i, projects.size());
+ projects.add(RexInputRef.of2(i, childFields));
+ }
+ for (Integer arg : argList) {
+ if (sourceOf.get(arg) != null) {
+ continue;
+ }
+ sourceOf.put(arg, projects.size());
+ projects.add(RexInputRef.of2(arg, childFields));
+ }
+ final RelNode project =
+ projFactory.createProject(child, Pair.left(projects), Pair.right(projects));
+
+ // Get the distinct values of the GROUP BY fields and the arguments
+ // to the agg functions.
+ return aggregate.copy(aggregate.getTraitSet(), project, false,
+ ImmutableBitSet.range(projects.size()),
+ null, ImmutableList.<AggregateCall>of());
+ }
+}
+
+// End AggregateExpandDistinctAggregatesRule.java
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/1538a71e/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 48f488f..6e6923c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -137,6 +137,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveExpandDistinctAggregatesRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterJoinRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTransposeRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterSetOpTransposeRule;
@@ -267,7 +268,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
this.ctx.setCboInfo("Plan optimized by CBO.");
this.ctx.setCboSucceeded(true);
LOG.debug(newAST.dump());
- }
+ }
} catch (Exception e) {
boolean isMissingStats = noColsMissingStats.get() > 0;
if (isMissingStats) {
@@ -901,6 +902,16 @@ public class CalcitePlanner extends SemanticAnalyzer {
// Partition Pruning; otherwise Expression evaluation may try to execute
// corelated sub query.
+ //0. Distinct aggregate rewrite
+ // Run this optimization early, since it is expanding the operator pipeline.
+ if (conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") &&
+ conf.getBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEDISTINCTREWRITE)) {
+ // Its not clear, if this rewrite is always performant on MR, since extra map phase
+ // introduced for 2nd MR job may offset gains of this multi-stage aggregation.
+ // We need a cost model for MR to enable this on MR.
+ basePlan = hepPlan(basePlan, true, mdProvider, HiveExpandDistinctAggregatesRule.INSTANCE);
+ }
+
// 1. Push Down Semi Joins
basePlan = hepPlan(basePlan, true, mdProvider, SemiJoinJoinTransposeRule.INSTANCE,
SemiJoinFilterTransposeRule.INSTANCE, SemiJoinProjectTransposeRule.INSTANCE);
http://git-wip-us.apache.org/repos/asf/hive/blob/1538a71e/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out b/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
index 7038b4d..2a41aae 100644
--- a/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
@@ -476,35 +476,38 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT _col1)
keys: _col0 (type: tinyint), _col1 (type: double)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint), _col1 (type: double)
sort order: ++
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.3
Reducer 2
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col1:0._col0)
- keys: KEY._col0 (type: tinyint)
+ keys: KEY._col0 (type: tinyint), KEY._col1 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 20
- Statistics: Num rows: 20 Data size: 4300 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Group By Operator
+ aggregations: count(_col1)
+ keys: _col0 (type: tinyint)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
Statistics: Num rows: 20 Data size: 4300 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 20 Data size: 4300 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/1538a71e/ql/src/test/results/clientpositive/tez/mrr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/mrr.q.out b/ql/src/test/results/clientpositive/tez/mrr.q.out
index ac5ac5c..d90b27f 100644
--- a/ql/src/test/results/clientpositive/tez/mrr.q.out
+++ b/ql/src/test/results/clientpositive/tez/mrr.q.out
@@ -452,10 +452,9 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT _col1)
keys: _col0 (type: string), _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
@@ -465,25 +464,30 @@ STAGE PLANS:
Reducer 3
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col1:0._col0)
- keys: KEY._col0 (type: string)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: bigint)
- sort order: +
- Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string)
+ Group By Operator
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: bigint)
+ sort order: +
+ Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Reducer 4
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -855,10 +859,9 @@ STAGE PLANS:
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
- aggregations: count(DISTINCT _col1)
keys: _col0 (type: string), _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
@@ -885,25 +888,30 @@ STAGE PLANS:
Reducer 2
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col1:0._col0)
- keys: KEY._col0 (type: string)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: bigint)
- sort order: +
- Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string)
+ Group By Operator
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: bigint)
+ sort order: +
+ Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/1538a71e/ql/src/test/results/clientpositive/tez/vector_count_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_count_distinct.q.out b/ql/src/test/results/clientpositive/tez/vector_count_distinct.q.out
index f1da471..e6d34ff 100644
--- a/ql/src/test/results/clientpositive/tez/vector_count_distinct.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_count_distinct.q.out
@@ -1248,6 +1248,7 @@ STAGE PLANS:
Tez
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1260,30 +1261,47 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 2000 Data size: 3504000 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT _col0)
keys: _col0 (type: int)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 2000 Data size: 3504000 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 2000 Data size: 3504000 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col0:0._col0)
+ keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1000 Data size: 1752000 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/1538a71e/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out b/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out
index d815938..1c5b51f 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out
@@ -345,36 +345,39 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT _col1)
keys: _col0 (type: tinyint), _col1 (type: double)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint), _col1 (type: double)
sort order: ++
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.3
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col1:0._col0)
- keys: KEY._col0 (type: tinyint)
+ keys: KEY._col0 (type: tinyint), KEY._col1 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 20
- Statistics: Num rows: 20 Data size: 4300 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Group By Operator
+ aggregations: count(_col1)
+ keys: _col0 (type: tinyint)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
Statistics: Num rows: 20 Data size: 4300 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 20 Data size: 4300 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/1538a71e/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out b/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out
index 90c9934..cf95c8a 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out
@@ -41,31 +41,35 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(DISTINCT _col0), count(DISTINCT _col0)
- bucketGroup: true
keys: _col0 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ mode: final
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), count(_col0)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
Group By Operator
- aggregations: sum(DISTINCT KEY._col0:0._col0), count(DISTINCT KEY._col0:1._col0)
+ aggregations: sum(VALUE._col0), count(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -95,6 +99,7 @@ STAGE PLANS:
Tez
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -107,20 +112,36 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(DISTINCT _col0), count(DISTINCT _col0), avg(DISTINCT _col0), std(DISTINCT _col0)
keys: _col0 (type: int)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
Group By Operator
- aggregations: sum(DISTINCT KEY._col0:0._col0), count(DISTINCT KEY._col0:1._col0), avg(DISTINCT KEY._col0:2._col0), std(DISTINCT KEY._col0:3._col0)
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), count(_col0), avg(_col0), std(_col0)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: struct<count:bigint,sum:double,input:int>), _col3 (type: struct<count:bigint,sum:double,variance:double>)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), count(VALUE._col1), avg(VALUE._col2), std(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE