You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/04/20 19:34:41 UTC
[6/6] hive git commit: HIVE-16296 : use LLAP executor count to
configure reducer auto-parallelism (Sergey Shelukhin, Gopal Vijayaraghavan,
reviewed by Gopal Vijayaraghavan, Sergey Shelukhin, Gunther Hagleitner)
HIVE-16296 : use LLAP executor count to configure reducer auto-parallelism (Sergey Shelukhin, Gopal Vijayaraghavan, reviewed by Gopal Vijayaraghavan, Sergey Shelukhin, Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/10bd23eb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/10bd23eb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/10bd23eb
Branch: refs/heads/master
Commit: 10bd23eb5c85948d7e246047818a628b9ab0a680
Parents: 656a99e
Author: sergey <se...@apache.org>
Authored: Thu Apr 20 12:11:22 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Apr 20 12:31:01 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../llap/registry/impl/LlapRegistryService.java | 5 +-
.../hadoop/hive/ql/exec/FetchOperator.java | 16 +
.../ql/optimizer/SetReducerParallelism.java | 2 +
.../physical/LlapClusterStateForCompile.java | 123 +
.../hive/ql/optimizer/physical/LlapDecider.java | 56 +-
.../hadoop/hive/ql/parse/GenTezUtils.java | 4 +
.../hadoop/hive/ql/parse/TezCompiler.java | 5 +-
.../apache/hadoop/hive/ql/plan/ReduceWork.java | 21 +
.../hadoop/hive/ql/plan/TezEdgeProperty.java | 6 +
.../queries/clientpositive/subquery_scalar.q | 1 +
.../queries/clientpositive/unionDistinct_1.q | 4 +-
.../clientpositive/vector_binary_join_groupby.q | 16 +-
.../vector_groupby_grouping_sets_limit.q | 25 +-
.../clientpositive/vector_groupby_reduce.q | 2 +
.../clientpositive/llap/bucket_groupby.q.out | 422 +-
.../clientpositive/llap/cbo_rp_lineage2.q.out | 58 +-
.../clientpositive/llap/cbo_rp_views.q.out | 2 +-
.../results/clientpositive/llap/cbo_views.q.out | 2 +-
.../results/clientpositive/llap/cluster.q.out | 688 +-
.../llap/correlationoptimizer1.q.out | 32 +-
.../results/clientpositive/llap/cte_1.q.out | 37900 ++++++++---------
.../clientpositive/llap/except_distinct.q.out | 2 +-
.../clientpositive/llap/explainuser_4.q.out | 4 +-
.../llap/hybridgrace_hashjoin_2.q.out | 4 +-
.../clientpositive/llap/intersect_all.q.out | 1470 +-
.../llap/intersect_distinct.q.out | 924 +-
.../clientpositive/llap/lateral_view.q.out | 8 +-
.../results/clientpositive/llap/lineage2.q.out | 58 +-
.../clientpositive/llap/multiMapJoin1.q.out | 4 +-
.../clientpositive/llap/multi_column_in.q.out | 2 +-
.../llap/multi_count_distinct_null.q.out | 24 +-
.../clientpositive/llap/parquet_types.q.out | 2 +-
.../llap/partition_multilevels.q.out | 244 +-
.../llap/reduce_deduplicate_extended.q.out | 32 +-
.../clientpositive/llap/skiphf_aggr.q.out | 4 +-
.../clientpositive/llap/subquery_multi.q.out | 450 +-
.../clientpositive/llap/subquery_notin.q.out | 106 +-
.../clientpositive/llap/subquery_scalar.q.out | 408 +-
.../llap/tez_dynpart_hashjoin_1.q.out | 2 +-
.../clientpositive/llap/tez_union2.q.out | 924 +-
.../llap/tez_vector_dynpart_hashjoin_1.q.out | 4 +-
.../clientpositive/llap/unionDistinct_1.q.out | 4160 +-
.../llap/vector_binary_join_groupby.q.out | 95 +-
.../clientpositive/llap/vector_coalesce_2.q.out | 4 +-
.../llap/vector_complex_all.q.out | 6 +-
.../llap/vector_decimal_udf.q.out | 32 +-
.../vector_groupby_grouping_sets_limit.q.out | 222 +-
.../llap/vector_groupby_reduce.q.out | 33 +
.../llap/vector_grouping_sets.q.out | 8 +-
.../llap/vector_partitioned_date_time.q.out | 52 +-
.../llap/vectorized_distinct_gby.q.out | 2 +-
.../vectorized_dynamic_semijoin_reduction.q.out | 72 +-
.../llap/vectorized_parquet.q.out | 164 +-
.../vector_binary_join_groupby.q.out | 115 +-
.../clientpositive/vector_groupby_reduce.q.out | 49 +-
56 files changed, 24817 insertions(+), 24266 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 457f7af..771f588 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2858,6 +2858,9 @@ public class HiveConf extends Configuration {
"Turn on Tez' auto reducer parallelism feature. When enabled, Hive will still estimate data sizes\n" +
"and set parallelism estimates. Tez will sample source vertices' output sizes and adjust the estimates at runtime as\n" +
"necessary."),
+ TEZ_LLAP_MIN_REDUCER_PER_EXECUTOR("hive.tez.llap.min.reducer.per.executor", 0.95f,
+ "If above 0, the min number of reducers for auto-parallelism for LLAP scheduling will\n" +
+ "be set to this fraction of the number of executors."),
TEZ_MAX_PARTITION_FACTOR("hive.tez.max.partition.factor", 2f,
"When auto reducer parallelism is enabled this factor will be used to over-partition data in shuffle edges."),
TEZ_MIN_PARTITION_FACTOR("hive.tez.min.partition.factor", 0.25f,
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java
index 610c0a5..76fc9c7 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java
@@ -61,7 +61,7 @@ public class LlapRegistryService extends AbstractService {
if (hosts.startsWith("@")) {
// Caching instances only in case of the YARN registry. Each host based list will get it's own copy.
String appName = hosts.substring(1);
- String userName = HiveConf.getVar(conf, ConfVars.LLAP_ZK_REGISTRY_USER, RegistryUtils.currentUser());
+ String userName = HiveConf.getVar(conf, ConfVars.LLAP_ZK_REGISTRY_USER, currentUser());
String key = appName + "-" + userName;
registry = yarnRegistries.get(key);
if (registry == null || !registry.isInState(STATE.STARTED)) {
@@ -79,6 +79,9 @@ public class LlapRegistryService extends AbstractService {
return registry;
}
+ public static String currentUser() {
+ return RegistryUtils.currentUser();
+ }
@Override
public void serviceInit(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
index 004bb2f..13750cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -378,6 +379,9 @@ public class FetchOperator implements Serializable {
inputSplits = splitSampling(work.getSplitSample(), inputSplits);
}
if (inputSplits.length > 0) {
+ if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_IN_TEST)) {
+ Arrays.sort(inputSplits, new FetchInputFormatSplitComparator());
+ }
return inputSplits;
}
}
@@ -695,4 +699,16 @@ public class FetchOperator implements Serializable {
return inputFormat.getRecordReader(getInputSplit(), job, Reporter.NULL);
}
}
+
+ private static class FetchInputFormatSplitComparator implements Comparator<FetchInputFormatSplit> {
+ @Override
+ public int compare(FetchInputFormatSplit a, FetchInputFormatSplit b) {
+ final Path ap = a.getPath();
+ final Path bp = b.getPath();
+ if (ap != null) {
+ return (ap.compareTo(bp));
+ }
+ return Long.signum(a.getLength() - b.getLength());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java
index 60a8604..b51af55 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.stats.StatsUtils;
import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.AUTOPARALLEL;
import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.UNIFORM;
+import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.FIXED;
/**
* SetReducerParallelism determines how many reducers should
@@ -106,6 +107,7 @@ public class SetReducerParallelism implements NodeProcessor {
}
} else {
LOG.info("Number of reducers determined to be: "+desc.getNumReducers());
+ desc.setReducerTraits(EnumSet.of(FIXED)); // usually controlled by bucketing
}
return false;
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
new file mode 100644
index 0000000..b2e8614
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.physical;
+
+import java.util.concurrent.ExecutionException;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.registry.ServiceInstance;
+import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
+import org.apache.hadoop.hive.llap.registry.impl.InactiveServiceInstance;
+import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapClusterStateForCompile {
+ protected static final Logger LOG = LoggerFactory.getLogger(LlapClusterStateForCompile.class);
+
+ private static final long CLUSTER_UPDATE_INTERVAL_NS = 120 * 1000000000L; // 2 minutes.
+ private Long lastClusterUpdateNs;
+ private Integer noConfigNodeCount, executorCount;
+ private LlapRegistryService svc;
+ private final Configuration conf;
+
+ // It's difficult to impossible to pass global things to compilation, so we have a static cache.
+ private static final Cache<String, LlapClusterStateForCompile> CACHE =
+ CacheBuilder.newBuilder().initialCapacity(10).maximumSize(100).build();
+
+ public static LlapClusterStateForCompile getClusterInfo(final Configuration conf) {
+ final String nodes = HiveConf.getTrimmedVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
+ final String userName = HiveConf.getVar(
+ conf, ConfVars.LLAP_ZK_REGISTRY_USER, LlapRegistryService.currentUser());
+ Callable<LlapClusterStateForCompile> generator = new Callable<LlapClusterStateForCompile>() {
+ @Override
+ public LlapClusterStateForCompile call() throws Exception {
+ LOG.info("Creating cluster info for " + userName + ":" + nodes);
+ return new LlapClusterStateForCompile(conf);
+ }
+ };
+ try {
+ return CACHE.get(userName + ":" + nodes, generator);
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e); // Should never happen... ctor is just assignments.
+ }
+ }
+
+ private LlapClusterStateForCompile(Configuration conf) {
+ this.conf = conf;
+ }
+
+ public boolean hasClusterInfo() {
+ return lastClusterUpdateNs != null;
+ }
+
+ public int getKnownExecutorCount() {
+ return executorCount;
+ }
+
+ public int getNodeCountWithUnknownExecutors() {
+ return noConfigNodeCount;
+ }
+
+ public synchronized void initClusterInfo() {
+ if (lastClusterUpdateNs != null) {
+ long elapsed = System.nanoTime() - lastClusterUpdateNs;
+ if (elapsed < CLUSTER_UPDATE_INTERVAL_NS) return;
+ }
+ if (svc == null) {
+ try {
+ svc = LlapRegistryService.getClient(conf);
+ } catch (Throwable t) {
+ LOG.info("Cannot create the client; ignoring", t);
+ return; // Don't fail; this is best-effort.
+ }
+ }
+ ServiceInstanceSet instances;
+ try {
+ instances = svc.getInstances(10);
+ } catch (IOException e) {
+ LOG.info("Cannot update cluster information; ignoring", e);
+ return; // Don't wait for the cluster if not started; this is best-effort.
+ }
+ int executorsLocal = 0, noConfigNodesLocal = 0;
+ for (ServiceInstance si : instances.getAll()) {
+ if (si instanceof InactiveServiceInstance) continue; // Shouldn't happen in getAll.
+ Map<String, String> props = si.getProperties();
+ if (props == null) {
+ ++noConfigNodesLocal;
+ continue;
+ }
+ try {
+ executorsLocal += Integer.parseInt(props.get(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname));
+ } catch (NumberFormatException e) {
+ ++noConfigNodesLocal;
+ }
+ }
+ lastClusterUpdateNs = System.nanoTime();
+ noConfigNodeCount = noConfigNodesLocal;
+ executorCount = executorsLocal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
index a694cf8..c81131e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
@@ -26,7 +26,6 @@ import static org.apache.hadoop.hive.ql.optimizer.physical.LlapDecider.LlapMode.
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Deque;
import java.util.EnumSet;
@@ -51,7 +50,6 @@ import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.tez.TezTask;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
@@ -106,12 +104,20 @@ public class LlapDecider implements PhysicalPlanResolver {
}
private LlapMode mode;
+ private final LlapClusterStateForCompile clusterState;
+
+ public LlapDecider(LlapClusterStateForCompile clusterState) {
+ this.clusterState = clusterState;
+ }
+
class LlapDecisionDispatcher implements Dispatcher {
private final HiveConf conf;
private final boolean doSkipUdfCheck;
private final boolean arePermanentFnsAllowed;
private final boolean shouldUber;
+ private final float minReducersPerExec;
+ private final int executorsPerNode;
private List<MapJoinOperator> mapJoinOpList;
private final Map<Rule, NodeProcessor> rules;
@@ -121,6 +127,9 @@ public class LlapDecider implements PhysicalPlanResolver {
arePermanentFnsAllowed = HiveConf.getBoolVar(conf, ConfVars.LLAP_ALLOW_PERMANENT_FNS);
// Don't user uber in "all" mode - everything can go into LLAP, which is better than uber.
shouldUber = HiveConf.getBoolVar(conf, ConfVars.LLAP_AUTO_ALLOW_UBER) && (mode != all);
+ minReducersPerExec = HiveConf.getFloatVar(
+ conf, ConfVars.TEZ_LLAP_MIN_REDUCER_PER_EXECUTOR);
+ executorsPerNode = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS); // TODO# hmm
mapJoinOpList = new ArrayList<MapJoinOperator>();
rules = getRules();
}
@@ -139,22 +148,57 @@ public class LlapDecider implements PhysicalPlanResolver {
return null;
}
- private void handleWork(TezWork tezWork, BaseWork work)
- throws SemanticException {
+ private void handleWork(TezWork tezWork, BaseWork work) throws SemanticException {
boolean workCanBeDoneInLlap = evaluateWork(tezWork, work);
LOG.debug(
"Work " + work + " " + (workCanBeDoneInLlap ? "can" : "cannot") + " be done in LLAP");
if (workCanBeDoneInLlap) {
for (MapJoinOperator graceMapJoinOp : mapJoinOpList) {
- LOG.debug(
- "Disabling hybrid grace hash join in case of LLAP and non-dynamic partition hash join.");
+ LOG.debug("Disabling hybrid grace hash join in case of LLAP "
+ + "and non-dynamic partition hash join.");
graceMapJoinOp.getConf().setHybridHashJoin(false);
}
+ adjustAutoParallelism(work);
+
convertWork(tezWork, work);
}
mapJoinOpList.clear();
}
+ private void adjustAutoParallelism(BaseWork work) {
+ if (minReducersPerExec <= 0 || !(work instanceof ReduceWork)) return;
+ ReduceWork reduceWork = (ReduceWork)work;
+ if (reduceWork.isAutoReduceParallelism() == false && reduceWork.isUniformDistribution() == false) {
+ return; // Not based on ARP and cannot assume uniform distribution, bail.
+ }
+ clusterState.initClusterInfo();
+ int targetCount = 0;
+ if (!clusterState.hasClusterInfo()) {
+ LOG.warn("Cannot determine LLAP cluster information");
+ targetCount = (int)Math.ceil(minReducersPerExec * 1 * executorsPerNode);
+ } else {
+ targetCount = (int)Math.ceil(minReducersPerExec * (clusterState.getKnownExecutorCount()
+ + clusterState.getNodeCountWithUnknownExecutors() * executorsPerNode));
+ }
+ // We only increase the targets here.
+ if (reduceWork.isAutoReduceParallelism()) {
+ int newMin = Math.max(reduceWork.getMinReduceTasks(), targetCount);
+ if (newMin < reduceWork.getMaxReduceTasks()) {
+ reduceWork.setMinReduceTasks(newMin);
+ reduceWork.getEdgePropRef().setAutoReduce(conf, true, newMin,
+ reduceWork.getMaxReduceTasks(), conf.getLongVar(HiveConf.ConfVars.BYTESPERREDUCER));
+ } else {
+ reduceWork.setAutoReduceParallelism(false);
+ reduceWork.setNumReduceTasks(newMin);
+ // TODO: is this correct? based on the same logic as HIVE-14200
+ reduceWork.getEdgePropRef().setAutoReduce(null, false, 0, 0, 0);
+ }
+ } else {
+ // UNIFORM || AUTOPARALLEL (maxed out)
+ reduceWork.setNumReduceTasks(Math.max(reduceWork.getNumReduceTasks(), targetCount));
+ }
+ }
+
private void convertWork(TezWork tezWork, BaseWork work)
throws SemanticException {
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index 83e89af..a9c1e61 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.parse;
import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.AUTOPARALLEL;
+import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.UNIFORM;
import java.util.*;
@@ -96,6 +97,7 @@ public class GenTezUtils {
reduceWork.setNumReduceTasks(reduceSink.getConf().getNumReducers());
reduceWork.setSlowStart(reduceSink.getConf().isSlowStart());
+ reduceWork.setUniformDistribution(reduceSink.getConf().getReducerTraits().contains(UNIFORM));
if (isAutoReduceParallelism && reduceSink.getConf().getReducerTraits().contains(AUTOPARALLEL)) {
@@ -103,6 +105,7 @@ public class GenTezUtils {
final int maxReducers = context.conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS);
// estimated number of reducers
final int nReducers = reduceSink.getConf().getNumReducers();
+ // TODO# HERE
// min we allow tez to pick
int minPartition = Math.max(1, (int) (nReducers * minPartitionFactor));
@@ -139,6 +142,7 @@ public class GenTezUtils {
edgeProp = new TezEdgeProperty(edgeType);
edgeProp.setSlowStart(reduceWork.isSlowStart());
}
+ reduceWork.setEdgePropRef(edgeProp);
tezWork.connect(
context.preceedingWork,
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index 26eda04..f87ca28 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.ql.parse;
+import org.apache.hadoop.hive.ql.optimizer.physical.LlapClusterStateForCompile;
+
import com.google.common.base.Preconditions;
import java.io.Serializable;
import java.util.*;
@@ -597,7 +599,8 @@ public class TezCompiler extends TaskCompiler {
}
if ("llap".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_MODE))) {
- physicalCtx = new LlapDecider().resolve(physicalCtx);
+ LlapClusterStateForCompile llapInfo = LlapClusterStateForCompile.getClusterInfo(conf);
+ physicalCtx = new LlapDecider(llapInfo).resolve(physicalCtx);
} else {
LOG.debug("Skipping llap decider");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
index 0cb1e57..50eeb99 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
@@ -85,6 +85,8 @@ public class ReduceWork extends BaseWork {
// boolean that says whether tez auto reduce parallelism should be used
private boolean isAutoReduceParallelism;
+ // boolean that says whether the data distribution is uniform hash (not java HashCode)
+ private transient boolean isUniformDistribution = false;
// boolean that says whether to slow start or not
private boolean isSlowStart = true;
@@ -104,6 +106,8 @@ public class ReduceWork extends BaseWork {
private String vectorReduceColumnSortOrder;
private String vectorReduceColumnNullOrder;
+ private transient TezEdgeProperty edgeProp;
+
/**
* If the plan has a reducer and correspondingly a reduce-sink, then store the TableDesc pointing
* to keySerializeInfo of the ReduceSink
@@ -231,6 +235,15 @@ public class ReduceWork extends BaseWork {
this.isSlowStart = isSlowStart;
}
+ // ReducerTraits.UNIFORM
+ public void setUniformDistribution(boolean isUniformDistribution) {
+ this.isUniformDistribution = isUniformDistribution;
+ }
+
+ public boolean isUniformDistribution() {
+ return this.isUniformDistribution;
+ }
+
public void setMinReduceTasks(int minReduceTasks) {
this.minReduceTasks = minReduceTasks;
}
@@ -355,4 +368,12 @@ public class ReduceWork extends BaseWork {
}
return new ReduceExplainVectorization(this);
}
+
+ public void setEdgePropRef(TezEdgeProperty edgeProp) {
+ this.edgeProp = edgeProp;
+ }
+
+ public TezEdgeProperty getEdgePropRef() {
+ return edgeProp;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
index d87bee3..5d7ddc8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
@@ -50,6 +50,12 @@ public class TezEdgeProperty {
public TezEdgeProperty(HiveConf hiveConf, EdgeType edgeType, boolean isAutoReduce,
boolean isSlowStart, int minReducer, int maxReducer, long bytesPerReducer) {
this(hiveConf, edgeType, -1);
+ setAutoReduce(hiveConf, isAutoReduce, minReducer, maxReducer, bytesPerReducer);
+ }
+
+ public void setAutoReduce(HiveConf hiveConf, boolean isAutoReduce, int minReducer,
+ int maxReducer, long bytesPerReducer) {
+ this.hiveConf = hiveConf;
this.minReducer = minReducer;
this.maxReducer = maxReducer;
this.isAutoReduce = isAutoReduce;
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/subquery_scalar.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/subquery_scalar.q b/ql/src/test/queries/clientpositive/subquery_scalar.q
index 99263bc..c7aeb11 100644
--- a/ql/src/test/queries/clientpositive/subquery_scalar.q
+++ b/ql/src/test/queries/clientpositive/subquery_scalar.q
@@ -1,5 +1,6 @@
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
+-- SORT_QUERY_RESULTS
create table tnull(i int, c char(2));
insert into tnull values(NULL, NULL), (NULL, NULL);
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/unionDistinct_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/unionDistinct_1.q b/ql/src/test/queries/clientpositive/unionDistinct_1.q
index 9792267..f2a0dc1 100644
--- a/ql/src/test/queries/clientpositive/unionDistinct_1.q
+++ b/ql/src/test/queries/clientpositive/unionDistinct_1.q
@@ -1,8 +1,8 @@
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
--- union10.q
+-- SORT_QUERY_RESULTS
-
+-- union10.q
-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q b/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q
index f203aef..826918f 100644
--- a/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q
+++ b/ql/src/test/queries/clientpositive/vector_binary_join_groupby.q
@@ -42,20 +42,24 @@ STORED AS ORC;
INSERT INTO TABLE hundredorc SELECT * FROM over1k LIMIT 100;
EXPLAIN VECTORIZATION EXPRESSION
-SELECT sum(hash(*))
-FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin;
+SELECT sum(hash(*)) k
+FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin
+order by k;
-SELECT sum(hash(*))
-FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin;
+SELECT sum(hash(*)) k
+FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin
+order by k;
EXPLAIN VECTORIZATION EXPRESSION
SELECT count(*), bin
FROM hundredorc
-GROUP BY bin;
+GROUP BY bin
+order by bin;
SELECT count(*), bin
FROM hundredorc
-GROUP BY bin;
+GROUP BY bin
+order by bin;
-- HIVE-14045: Involve a binary vector scratch column for small table result (Native Vector MapJoin).
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q b/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q
index fff706b..49cc4ef 100644
--- a/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q
+++ b/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets_limit.q
@@ -3,6 +3,7 @@ SET hive.vectorized.execution.enabled=true;
SET hive.vectorized.execution.reduce.enabled=true;
set hive.fetch.task.conversion=none;
set hive.cli.print.header=true;
+-- SORT_QUERY_RESULTS
CREATE TABLE T1_text(a STRING, b STRING, c STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' STORED AS TEXTFILE;
@@ -13,31 +14,31 @@ CREATE TABLE T1 STORED AS ORC AS SELECT * FROM T1_text;
-- SORT_QUERY_RESULTS
EXPLAIN
-SELECT a, b, count(*) from T1 group by a, b with cube LIMIT 10;
+SELECT a, b, count(*) from T1 group by a, b with cube order by a, b LIMIT 10;
-SELECT a, b, count(*) from T1 group by a, b with cube LIMIT 10;
+SELECT a, b, count(*) from T1 group by a, b with cube order by a, b LIMIT 10;
EXPLAIN
-SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) LIMIT 10;
+SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) order by a, b LIMIT 10;
-SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) LIMIT 10;
+SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) order by a, b LIMIT 10;
EXPLAIN
-SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) LIMIT 10;
+SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) order by a, b LIMIT 10;
-SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) LIMIT 10;
+SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) order by a, b LIMIT 10;
EXPLAIN
-SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) LIMIT 10;
+SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) order by a LIMIT 10;
-SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) LIMIT 10;
+SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) order by a LIMIT 10;
EXPLAIN
-SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) LIMIT 10;
+SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) order by a LIMIT 10;
-SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) LIMIT 10;
+SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) order by a LIMIT 10;
EXPLAIN
-SELECT a + b, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) LIMIT 10;
+SELECT a + b ab, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) order by ab LIMIT 10;
-SELECT a + b, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) LIMIT 10;
+SELECT a + b ab, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) order by ab LIMIT 10;
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/queries/clientpositive/vector_groupby_reduce.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_groupby_reduce.q b/ql/src/test/queries/clientpositive/vector_groupby_reduce.q
index bafb32e..f23b26f 100644
--- a/ql/src/test/queries/clientpositive/vector_groupby_reduce.q
+++ b/ql/src/test/queries/clientpositive/vector_groupby_reduce.q
@@ -98,6 +98,7 @@ select
from
store_sales
group by ss_ticket_number
+order by ss_ticket_number
limit 20;
select
@@ -105,6 +106,7 @@ select
from
store_sales
group by ss_ticket_number
+order by ss_ticket_number
limit 20;
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
index 4ac5ac9..d724131 100644
--- a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
@@ -712,314 +712,314 @@ POSTHOOK: Input: default@clustergroupby
POSTHOOK: Input: default@clustergroupby@ds=100
POSTHOOK: Input: default@clustergroupby@ds=101
#### A masked pattern was here ####
-0 6
10 2
100 4
-103 4
104 4
-105 2
-11 2
111 2
-113 4
114 2
116 2
-118 4
-119 6
-12 4
-120 4
-125 4
126 2
128 6
-129 4
131 2
-133 2
134 4
-136 2
-137 4
-138 8
-143 2
-145 2
-146 4
-149 4
-15 4
-150 2
152 4
-153 2
155 2
-156 2
-157 2
-158 2
-160 2
-162 2
-163 2
-164 4
-165 4
-166 2
-167 6
-168 2
169 8
17 2
170 2
172 4
-174 4
-175 4
-176 4
-177 2
178 2
179 4
-18 4
-180 2
-181 2
-183 2
-186 2
-187 6
189 2
19 2
-190 2
-191 4
192 2
-193 6
194 2
195 4
-196 2
-197 4
199 6
-2 2
20 2
-200 4
-201 2
-202 2
-203 4
-205 4
207 4
-208 6
-209 4
213 4
-214 2
-216 4
-217 4
218 2
-219 4
-221 4
-222 2
223 4
-224 4
-226 2
-228 2
229 4
-230 10
233 4
235 2
-237 4
-238 4
-239 4
24 4
-241 2
-242 4
-244 2
-247 2
-248 2
-249 2
-252 2
-255 4
256 4
257 2
-258 2
26 4
-260 2
-262 2
-263 2
265 4
266 2
27 2
-272 4
-273 6
-274 2
-275 2
-277 8
278 4
-28 2
280 4
-281 4
-282 4
-283 2
284 2
-285 2
-286 2
-287 2
-288 4
-289 2
291 2
-292 2
-296 2
-298 6
30 2
302 2
305 2
-306 2
-307 4
-308 2
-309 4
-310 2
-311 6
-315 2
316 6
-317 4
-318 6
321 4
-322 4
323 2
-325 4
-327 6
33 2
-331 4
-332 2
333 4
-335 2
-336 2
338 2
-339 2
-34 2
-341 2
-342 4
344 4
-345 2
-348 10
-35 6
-351 2
353 4
356 2
-360 2
-362 2
364 2
-365 2
-366 2
-367 4
368 2
369 6
37 4
-373 2
-374 2
-375 2
377 2
378 2
-379 2
-382 4
-384 6
386 2
-389 2
392 2
-393 2
-394 2
-395 4
-396 6
397 4
-399 4
-4 2
-400 2
-401 10
-402 2
-403 6
404 4
-406 8
407 2
-409 6
-41 2
411 2
-413 4
-414 4
-417 6
418 2
419 2
-42 4
-421 2
424 4
-427 2
-429 4
-43 2
430 6
-431 6
432 2
-435 2
436 2
437 2
-438 6
-439 4
-44 2
-443 2
-444 2
-446 2
448 2
-449 2
-452 2
-453 2
454 6
-455 2
457 2
-458 4
-459 4
-460 2
-462 4
-463 4
-466 6
-467 2
468 8
-469 10
-47 2
-470 2
-472 2
-475 2
477 2
-478 4
479 2
-480 6
-481 2
-482 2
-483 2
-484 2
-485 2
-487 2
-489 8
490 2
-491 2
492 4
493 2
-494 2
495 2
496 2
497 2
-498 6
-5 6
-51 4
-53 2
-54 2
57 2
+67 4
+8 2
+80 2
+85 2
+9 2
+92 2
+97 4
+105 2
+11 2
+113 4
+136 2
+137 4
+138 8
+143 2
+150 2
+158 2
+160 2
+162 2
+165 4
+166 2
+174 4
+177 2
+18 4
+180 2
+186 2
+196 2
+197 4
+217 4
+219 4
+226 2
+230 10
+238 4
+242 4
+248 2
+252 2
+260 2
+263 2
+274 2
+277 8
+296 2
+307 4
+308 2
+325 4
+327 6
+342 4
+348 10
+35 6
+351 2
+367 4
+389 2
+393 2
+394 2
+4 2
+403 6
+409 6
+414 4
+421 2
+427 2
+429 4
+435 2
+438 6
+439 4
+455 2
+458 4
+460 2
+462 4
+475 2
+478 4
+480 6
+484 2
+487 2
+53 2
58 4
-64 2
-65 2
66 2
-67 4
69 2
-70 6
+77 2
+78 2
+83 4
+90 6
+95 4
+103 4
+120 4
+129 4
+133 2
+145 2
+146 4
+149 4
+15 4
+157 2
+181 2
+187 6
+191 4
+2 2
+200 4
+201 2
+202 2
+203 4
+208 6
+214 2
+216 4
+222 2
+224 4
+228 2
+237 4
+239 4
+241 2
+244 2
+255 4
+258 2
+262 2
+273 6
+275 2
+28 2
+282 4
+285 2
+286 2
+287 2
+288 4
+292 2
+298 6
+310 2
+311 6
+332 2
+335 2
+336 2
+339 2
+360 2
+362 2
+373 2
+384 6
+395 4
+396 6
+402 2
+417 6
+42 4
+43 2
+444 2
+449 2
+453 2
+459 4
+47 2
+481 2
+482 2
+483 2
+485 2
+489 8
+494 2
+65 2
72 4
74 2
76 4
-77 2
-78 2
-8 2
-80 2
82 2
-83 4
-84 4
-85 2
86 2
87 2
-9 2
-90 6
-92 2
-95 4
96 2
-97 4
+0 6
+118 4
+119 6
+12 4
+125 4
+153 2
+156 2
+163 2
+164 4
+167 6
+168 2
+175 4
+176 4
+183 2
+190 2
+193 6
+205 4
+209 4
+221 4
+247 2
+249 2
+272 4
+281 4
+283 2
+289 2
+306 2
+309 4
+315 2
+317 4
+318 6
+322 4
+331 4
+34 2
+341 2
+345 2
+365 2
+366 2
+374 2
+375 2
+379 2
+382 4
+399 4
+400 2
+401 10
+406 8
+41 2
+413 4
+431 6
+44 2
+443 2
+446 2
+452 2
+463 4
+466 6
+467 2
+469 10
+470 2
+472 2
+491 2
+498 6
+5 6
+51 4
+54 2
+64 2
+70 6
+84 4
98 4
PREHOOK: query: explain
select key, count(1) from clustergroupby group by key, 3
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out b/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out
index 4bc28ce..44f9d68 100644
--- a/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out
+++ b/ql/src/test/results/clientpositive/llap/cbo_rp_lineage2.q.out
@@ -141,87 +141,87 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
#### A masked pattern was here ####
{"version":"1.0","engine":"tez","database":"default","hash":"3901b5e3a164064736b3234355046340","queryText":"select key, count(1) a from dest1 group by key","edges":[],"vertices":[]}
- 20
128 2
-146 2
-150 2
213 2
-224 2
+278 2
+369 2
+ 20
+150 2
238 2
+66 2
+146 2
+224 2
255 2
273 2
-278 2
311 2
-369 2
401 2
406 2
-66 2
98 2
PREHOOK: query: select key k, count(*) from dest1 group by key
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
#### A masked pattern was here ####
{"version":"1.0","engine":"tez","database":"default","hash":"0d5a212f10847aeaab31e8c31121e6d4","queryText":"select key k, count(*) from dest1 group by key","edges":[],"vertices":[]}
- 20
128 2
-146 2
-150 2
213 2
-224 2
+278 2
+369 2
+ 20
+150 2
238 2
+66 2
+146 2
+224 2
255 2
273 2
-278 2
311 2
-369 2
401 2
406 2
-66 2
98 2
PREHOOK: query: select key k, count(value) from dest1 group by key
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
#### A masked pattern was here ####
{"version":"1.0","engine":"tez","database":"default","hash":"56429eccb04ded722f5bd9d9d8cf7260","queryText":"select key k, count(value) from dest1 group by key","edges":[],"vertices":[]}
- 20
128 2
-146 2
-150 2
213 2
-224 2
+278 2
+369 2
+ 20
+150 2
238 2
+66 2
+146 2
+224 2
255 2
273 2
-278 2
311 2
-369 2
401 2
406 2
-66 2
98 2
PREHOOK: query: select value, max(length(key)) from dest1 group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
#### A masked pattern was here ####
{"version":"1.0","engine":"tez","database":"default","hash":"7e1cfc3dece85b41b6f7c46365580cde","queryText":"select value, max(length(key)) from dest1 group by value","edges":[],"vertices":[]}
- 3
val_146 3
-val_150 3
+val_27 0
+val_409 0
+val_484 0
+val_66 2
+ 3
val_165 0
+val_238 3
val_193 0
val_213 3
-val_238 3
val_255 3
val_265 0
-val_27 0
val_273 3
val_278 3
-val_311 3
val_401 3
+val_150 3
+val_311 3
val_406 3
-val_409 0
-val_484 0
-val_66 2
val_98 2
PREHOOK: query: select value, max(length(key)) from dest1 group by value order by value limit 5
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out b/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out
index 2ac2d3a..d603957 100644
--- a/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out
+++ b/ql/src/test/results/clientpositive/llap/cbo_rp_views.q.out
@@ -50,8 +50,8 @@ POSTHOOK: Input: default@cbo_t1
POSTHOOK: Input: default@cbo_t1@dt=2014
POSTHOOK: Input: default@v1
#### A masked pattern was here ####
-NULL
1
+NULL
1
PREHOOK: query: select count(v1.c_int) from v1 join cbo_t2 on v1.c_int = cbo_t2.c_int
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/cbo_views.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/cbo_views.q.out b/ql/src/test/results/clientpositive/llap/cbo_views.q.out
index 44e5501..25ea4cf 100644
--- a/ql/src/test/results/clientpositive/llap/cbo_views.q.out
+++ b/ql/src/test/results/clientpositive/llap/cbo_views.q.out
@@ -50,8 +50,8 @@ POSTHOOK: Input: default@cbo_t1
POSTHOOK: Input: default@cbo_t1@dt=2014
POSTHOOK: Input: default@v1
#### A masked pattern was here ####
-NULL
1
+NULL
1
PREHOOK: query: select count(v1.c_int) from v1 join cbo_t2 on v1.c_int = cbo_t2.c_int
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/cluster.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/cluster.q.out b/ql/src/test/results/clientpositive/llap/cluster.q.out
index 8c89ee3..2fa976b 100644
--- a/ql/src/test/results/clientpositive/llap/cluster.q.out
+++ b/ql/src/test/results/clientpositive/llap/cluster.q.out
@@ -1063,77 +1063,22 @@ CLUSTER BY unioninput.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0 val_0
-0 val_0
-0 val_0
10 val_10
-103 val_103
-103 val_103
104 val_104
104 val_104
-105 val_105
-11 val_11
111 val_111
-113 val_113
-113 val_113
114 val_114
116 val_116
-118 val_118
-118 val_118
-119 val_119
-119 val_119
-119 val_119
-12 val_12
-12 val_12
-120 val_120
-120 val_120
-125 val_125
-125 val_125
126 val_126
128 val_128
128 val_128
128 val_128
-129 val_129
-129 val_129
131 val_131
-133 val_133
134 val_134
134 val_134
-136 val_136
-137 val_137
-137 val_137
-138 val_138
-138 val_138
-138 val_138
-138 val_138
-143 val_143
-145 val_145
-146 val_146
-146 val_146
-149 val_149
-149 val_149
-15 val_15
-15 val_15
-150 val_150
152 val_152
152 val_152
-153 val_153
155 val_155
-156 val_156
-157 val_157
-158 val_158
-160 val_160
-162 val_162
-163 val_163
-164 val_164
-164 val_164
-165 val_165
-165 val_165
-166 val_166
-167 val_167
-167 val_167
-167 val_167
-168 val_168
169 val_169
169 val_169
169 val_169
@@ -1142,374 +1087,463 @@ POSTHOOK: Input: default@src
170 val_170
172 val_172
172 val_172
-174 val_174
-174 val_174
-175 val_175
-175 val_175
-176 val_176
-176 val_176
-177 val_177
178 val_178
179 val_179
179 val_179
-18 val_18
-18 val_18
-180 val_180
-181 val_181
-183 val_183
-186 val_186
-187 val_187
-187 val_187
-187 val_187
189 val_189
19 val_19
-190 val_190
-191 val_191
-191 val_191
192 val_192
-193 val_193
-193 val_193
-193 val_193
194 val_194
195 val_195
195 val_195
-196 val_196
-197 val_197
-197 val_197
199 val_199
199 val_199
199 val_199
-2 val_2
20 val_20
-200 val_200
-200 val_200
-201 val_201
-202 val_202
-203 val_203
-203 val_203
-205 val_205
-205 val_205
207 val_207
207 val_207
-208 val_208
-208 val_208
-208 val_208
-209 val_209
-209 val_209
213 val_213
213 val_213
-214 val_214
-216 val_216
-216 val_216
-217 val_217
-217 val_217
218 val_218
-219 val_219
-219 val_219
-221 val_221
-221 val_221
-222 val_222
223 val_223
223 val_223
-224 val_224
-224 val_224
-226 val_226
-228 val_228
229 val_229
229 val_229
-230 val_230
-230 val_230
-230 val_230
-230 val_230
-230 val_230
233 val_233
233 val_233
235 val_235
-237 val_237
-237 val_237
-238 val_238
-238 val_238
-239 val_239
-239 val_239
24 val_24
24 val_24
-241 val_241
-242 val_242
-242 val_242
-244 val_244
-247 val_247
-248 val_248
-249 val_249
-252 val_252
-255 val_255
-255 val_255
256 val_256
256 val_256
257 val_257
-258 val_258
26 val_26
26 val_26
-260 val_260
-262 val_262
-263 val_263
265 val_265
265 val_265
266 val_266
27 val_27
-272 val_272
-272 val_272
-273 val_273
-273 val_273
-273 val_273
-274 val_274
-275 val_275
-277 val_277
-277 val_277
-277 val_277
-277 val_277
278 val_278
278 val_278
-28 val_28
280 val_280
280 val_280
-281 val_281
-281 val_281
-282 val_282
-282 val_282
-283 val_283
284 val_284
-285 val_285
-286 val_286
-287 val_287
-288 val_288
-288 val_288
-289 val_289
291 val_291
-292 val_292
-296 val_296
-298 val_298
-298 val_298
-298 val_298
30 val_30
302 val_302
305 val_305
-306 val_306
-307 val_307
-307 val_307
-308 val_308
-309 val_309
-309 val_309
-310 val_310
-311 val_311
-311 val_311
-311 val_311
-315 val_315
316 val_316
316 val_316
316 val_316
-317 val_317
-317 val_317
-318 val_318
-318 val_318
-318 val_318
321 val_321
321 val_321
-322 val_322
-322 val_322
323 val_323
-325 val_325
-325 val_325
-327 val_327
-327 val_327
-327 val_327
33 val_33
-331 val_331
-331 val_331
-332 val_332
333 val_333
333 val_333
-335 val_335
-336 val_336
338 val_338
-339 val_339
-34 val_34
-341 val_341
-342 val_342
-342 val_342
344 val_344
344 val_344
-345 val_345
-348 val_348
-348 val_348
-348 val_348
-348 val_348
-348 val_348
-35 val_35
-35 val_35
-35 val_35
-351 val_351
353 val_353
353 val_353
356 val_356
-360 val_360
-362 val_362
364 val_364
-365 val_365
-366 val_366
-367 val_367
-367 val_367
368 val_368
369 val_369
369 val_369
369 val_369
37 val_37
37 val_37
-373 val_373
-374 val_374
-375 val_375
377 val_377
378 val_378
-379 val_379
-382 val_382
-382 val_382
-384 val_384
-384 val_384
-384 val_384
386 val_386
-389 val_389
392 val_392
-393 val_393
-394 val_394
-395 val_395
-395 val_395
-396 val_396
-396 val_396
-396 val_396
397 val_397
397 val_397
-399 val_399
-399 val_399
-4 val_4
-400 val_400
-401 val_401
-401 val_401
-401 val_401
-401 val_401
-401 val_401
-402 val_402
-403 val_403
-403 val_403
-403 val_403
404 val_404
404 val_404
-406 val_406
-406 val_406
-406 val_406
-406 val_406
407 val_407
-409 val_409
-409 val_409
-409 val_409
-41 val_41
411 val_411
-413 val_413
-413 val_413
-414 val_414
-414 val_414
-417 val_417
-417 val_417
-417 val_417
418 val_418
419 val_419
-42 val_42
-42 val_42
-421 val_421
424 val_424
424 val_424
-427 val_427
-429 val_429
-429 val_429
-43 val_43
430 val_430
430 val_430
430 val_430
-431 val_431
-431 val_431
-431 val_431
432 val_432
-435 val_435
436 val_436
437 val_437
+448 val_448
+454 val_454
+454 val_454
+454 val_454
+457 val_457
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+477 val_477
+479 val_479
+490 val_490
+492 val_492
+492 val_492
+493 val_493
+495 val_495
+496 val_496
+497 val_497
+57 val_57
+67 val_67
+67 val_67
+8 val_8
+80 val_80
+85 val_85
+9 val_9
+92 val_92
+97 val_97
+97 val_97
+105 val_105
+11 val_11
+113 val_113
+113 val_113
+136 val_136
+137 val_137
+137 val_137
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+143 val_143
+150 val_150
+158 val_158
+160 val_160
+162 val_162
+165 val_165
+165 val_165
+166 val_166
+174 val_174
+174 val_174
+177 val_177
+18 val_18
+18 val_18
+180 val_180
+186 val_186
+196 val_196
+197 val_197
+197 val_197
+217 val_217
+217 val_217
+219 val_219
+219 val_219
+226 val_226
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+238 val_238
+238 val_238
+242 val_242
+242 val_242
+248 val_248
+252 val_252
+260 val_260
+263 val_263
+274 val_274
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+296 val_296
+307 val_307
+307 val_307
+308 val_308
+325 val_325
+325 val_325
+327 val_327
+327 val_327
+327 val_327
+342 val_342
+342 val_342
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+35 val_35
+35 val_35
+35 val_35
+351 val_351
+367 val_367
+367 val_367
+389 val_389
+393 val_393
+394 val_394
+4 val_4
+403 val_403
+403 val_403
+403 val_403
+409 val_409
+409 val_409
+409 val_409
+414 val_414
+414 val_414
+421 val_421
+427 val_427
+429 val_429
+429 val_429
+435 val_435
438 val_438
438 val_438
438 val_438
439 val_439
439 val_439
-44 val_44
-443 val_443
-444 val_444
-446 val_446
-448 val_448
-449 val_449
-452 val_452
-453 val_453
-454 val_454
-454 val_454
-454 val_454
455 val_455
-457 val_457
458 val_458
458 val_458
-459 val_459
-459 val_459
460 val_460
462 val_462
462 val_462
-463 val_463
-463 val_463
-466 val_466
-466 val_466
-466 val_466
-467 val_467
-468 val_468
-468 val_468
-468 val_468
-468 val_468
-469 val_469
-469 val_469
-469 val_469
-469 val_469
-469 val_469
-47 val_47
-470 val_470
-472 val_472
475 val_475
-477 val_477
478 val_478
478 val_478
-479 val_479
480 val_480
480 val_480
480 val_480
+484 val_484
+487 val_487
+53 val_53
+58 val_58
+58 val_58
+66 val_66
+69 val_69
+77 val_77
+78 val_78
+83 val_83
+83 val_83
+90 val_90
+90 val_90
+90 val_90
+95 val_95
+95 val_95
+103 val_103
+103 val_103
+120 val_120
+120 val_120
+129 val_129
+129 val_129
+133 val_133
+145 val_145
+146 val_146
+146 val_146
+149 val_149
+149 val_149
+15 val_15
+15 val_15
+157 val_157
+181 val_181
+187 val_187
+187 val_187
+187 val_187
+191 val_191
+191 val_191
+2 val_2
+200 val_200
+200 val_200
+201 val_201
+202 val_202
+203 val_203
+203 val_203
+208 val_208
+208 val_208
+208 val_208
+214 val_214
+216 val_216
+216 val_216
+222 val_222
+224 val_224
+224 val_224
+228 val_228
+237 val_237
+237 val_237
+239 val_239
+239 val_239
+241 val_241
+244 val_244
+255 val_255
+255 val_255
+258 val_258
+262 val_262
+273 val_273
+273 val_273
+273 val_273
+275 val_275
+28 val_28
+282 val_282
+282 val_282
+285 val_285
+286 val_286
+287 val_287
+288 val_288
+288 val_288
+292 val_292
+298 val_298
+298 val_298
+298 val_298
+310 val_310
+311 val_311
+311 val_311
+311 val_311
+332 val_332
+335 val_335
+336 val_336
+339 val_339
+360 val_360
+362 val_362
+373 val_373
+384 val_384
+384 val_384
+384 val_384
+395 val_395
+395 val_395
+396 val_396
+396 val_396
+396 val_396
+402 val_402
+417 val_417
+417 val_417
+417 val_417
+42 val_42
+42 val_42
+43 val_43
+444 val_444
+449 val_449
+453 val_453
+459 val_459
+459 val_459
+47 val_47
481 val_481
482 val_482
483 val_483
-484 val_484
485 val_485
-487 val_487
489 val_489
489 val_489
489 val_489
489 val_489
-490 val_490
-491 val_491
-492 val_492
-492 val_492
-493 val_493
494 val_494
-495 val_495
-496 val_496
-497 val_497
+65 val_65
+72 val_72
+72 val_72
+74 val_74
+76 val_76
+76 val_76
+82 val_82
+86 val_86
+87 val_87
+96 val_96
+0 val_0
+0 val_0
+0 val_0
+118 val_118
+118 val_118
+119 val_119
+119 val_119
+119 val_119
+12 val_12
+12 val_12
+125 val_125
+125 val_125
+153 val_153
+156 val_156
+163 val_163
+164 val_164
+164 val_164
+167 val_167
+167 val_167
+167 val_167
+168 val_168
+175 val_175
+175 val_175
+176 val_176
+176 val_176
+183 val_183
+190 val_190
+193 val_193
+193 val_193
+193 val_193
+205 val_205
+205 val_205
+209 val_209
+209 val_209
+221 val_221
+221 val_221
+247 val_247
+249 val_249
+272 val_272
+272 val_272
+281 val_281
+281 val_281
+283 val_283
+289 val_289
+306 val_306
+309 val_309
+309 val_309
+315 val_315
+317 val_317
+317 val_317
+318 val_318
+318 val_318
+318 val_318
+322 val_322
+322 val_322
+331 val_331
+331 val_331
+34 val_34
+341 val_341
+345 val_345
+365 val_365
+366 val_366
+374 val_374
+375 val_375
+379 val_379
+382 val_382
+382 val_382
+399 val_399
+399 val_399
+400 val_400
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+41 val_41
+413 val_413
+413 val_413
+431 val_431
+431 val_431
+431 val_431
+44 val_44
+443 val_443
+446 val_446
+452 val_452
+463 val_463
+463 val_463
+466 val_466
+466 val_466
+466 val_466
+467 val_467
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+470 val_470
+472 val_472
+491 val_491
498 val_498
498 val_498
498 val_498
@@ -1518,46 +1552,12 @@ POSTHOOK: Input: default@src
5 val_5
51 val_51
51 val_51
-53 val_53
54 val_54
-57 val_57
-58 val_58
-58 val_58
64 val_64
-65 val_65
-66 val_66
-67 val_67
-67 val_67
-69 val_69
70 val_70
70 val_70
70 val_70
-72 val_72
-72 val_72
-74 val_74
-76 val_76
-76 val_76
-77 val_77
-78 val_78
-8 val_8
-80 val_80
-82 val_82
-83 val_83
-83 val_83
84 val_84
84 val_84
-85 val_85
-86 val_86
-87 val_87
-9 val_9
-90 val_90
-90 val_90
-90 val_90
-92 val_92
-95 val_95
-95 val_95
-96 val_96
-97 val_97
-97 val_97
98 val_98
98 val_98
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out
index 86d94b4..cc86d15 100644
--- a/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out
+++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out
@@ -1438,20 +1438,20 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@src1
#### A masked pattern was here ####
NULL 10
-128 NULL 1
-146 val_146 2
-150 val_150 1
213 val_213 2
-224 NULL 1
238 val_238 2
+406 val_406 4
+66 val_66 1
+224 NULL 1
+278 val_278 2
+128 NULL 1
+150 val_150 1
255 val_255 2
273 val_273 3
-278 val_278 2
+401 val_401 5
+146 val_146 2
311 val_311 3
369 NULL 1
-401 val_401 5
-406 val_406 4
-66 val_66 1
98 val_98 2
PREHOOK: query: EXPLAIN
SELECT x.key, y.value, count(1) AS cnt
@@ -1573,20 +1573,20 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@src1
#### A masked pattern was here ####
NULL 10
-128 NULL 1
-146 val_146 2
-150 val_150 1
213 val_213 2
-224 NULL 1
238 val_238 2
+406 val_406 4
+66 val_66 1
+224 NULL 1
+278 val_278 2
+128 NULL 1
+150 val_150 1
255 val_255 2
273 val_273 3
-278 val_278 2
+401 val_401 5
+146 val_146 2
311 val_311 3
369 NULL 1
-401 val_401 5
-406 val_406 4
-66 val_66 1
98 val_98 2
PREHOOK: query: EXPLAIN
SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt))